From 7b229b13d78d112e2c5d4a60a3c6f602289959fa Mon Sep 17 00:00:00 2001 From: Dmitry Torokhov Date: Sat, 10 Apr 2021 19:56:05 -0700 Subject: HID: hid-input: add mapping for emoji picker key HUTRR101 added a new usage code for a key that is supposed to invoke and dismiss an emoji picker widget to assist users to locate and enter emojis. This patch adds a new key definition KEY_EMOJI_PICKER and maps 0x0c/0x0d9 usage code to this new keycode. Additionally hid-debug is adjusted to recognize this new usage code as well. Signed-off-by: Dmitry Torokhov Signed-off-by: Jiri Kosina --- include/uapi/linux/input-event-codes.h | 1 + 1 file changed, 1 insertion(+) (limited to 'include/uapi') diff --git a/include/uapi/linux/input-event-codes.h b/include/uapi/linux/input-event-codes.h index ee93428ced9a..225ec87d4f22 100644 --- a/include/uapi/linux/input-event-codes.h +++ b/include/uapi/linux/input-event-codes.h @@ -611,6 +611,7 @@ #define KEY_VOICECOMMAND 0x246 /* Listening Voice Command */ #define KEY_ASSISTANT 0x247 /* AL Context-aware desktop assistant */ #define KEY_KBD_LAYOUT_NEXT 0x248 /* AC Next Keyboard Layout Select */ +#define KEY_EMOJI_PICKER 0x249 /* Show/hide emoji picker (HUTRR101) */ #define KEY_BRIGHTNESS_MIN 0x250 /* Set Brightness to Minimum */ #define KEY_BRIGHTNESS_MAX 0x251 /* Set Brightness to Maximum */ -- cgit v1.2.3 From 7ac592aa35a684ff1858fb9ec282886b9e3575ac Mon Sep 17 00:00:00 2001 From: Chris Hyser Date: Wed, 24 Mar 2021 17:40:15 -0400 Subject: sched: prctl() core-scheduling interface This patch provides support for setting and copying core scheduling 'task cookies' between threads (PID), processes (TGID), and process groups (PGID). The value of core scheduling isn't that tasks don't share a core, 'nosmt' can do that. The value lies in exploiting all the sharing opportunities that exist to recover possible lost performance and that requires a degree of flexibility in the API. From a security perspective (and there are others), the thread, process and process group distinction is an existent hierarchal categorization of tasks that reflects many of the security concerns about 'data sharing'. For example, protecting against cache-snooping by a thread that can just read the memory directly isn't all that useful. With this in mind, subcommands to CREATE/SHARE (TO/FROM) provide a mechanism to create and share cookies. CREATE/SHARE_TO specify a target pid with enum pidtype used to specify the scope of the targeted tasks. For example, PIDTYPE_TGID will share the cookie with the process and all of it's threads as typically desired in a security scenario. API: prctl(PR_SCHED_CORE, PR_SCHED_CORE_GET, tgtpid, pidtype, &cookie) prctl(PR_SCHED_CORE, PR_SCHED_CORE_CREATE, tgtpid, pidtype, NULL) prctl(PR_SCHED_CORE, PR_SCHED_CORE_SHARE_TO, tgtpid, pidtype, NULL) prctl(PR_SCHED_CORE, PR_SCHED_CORE_SHARE_FROM, srcpid, pidtype, NULL) where 'tgtpid/srcpid == 0' implies the current process and pidtype is kernel enum pid_type {PIDTYPE_PID, PIDTYPE_TGID, PIDTYPE_PGID, ...}. For return values, EINVAL, ENOMEM are what they say. ESRCH means the tgtpid/srcpid was not found. EPERM indicates lack of PTRACE permission access to tgtpid/srcpid. ENODEV indicates your machines lacks SMT. [peterz: complete rewrite] Signed-off-by: Chris Hyser Signed-off-by: Peter Zijlstra (Intel) Tested-by: Don Hiatt Tested-by: Hongyu Ning Tested-by: Vincent Guittot Link: https://lkml.kernel.org/r/20210422123309.039845339@infradead.org --- include/linux/sched.h | 2 + include/uapi/linux/prctl.h | 8 +++ kernel/sched/core_sched.c | 114 +++++++++++++++++++++++++++++++++++++++ kernel/sys.c | 5 ++ tools/include/uapi/linux/prctl.h | 8 +++ 5 files changed, 137 insertions(+) (limited to 'include/uapi') diff --git a/include/linux/sched.h b/include/linux/sched.h index fba47e52e482..c7e7d50e2fdc 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -2182,6 +2182,8 @@ const struct cpumask *sched_trace_rd_span(struct root_domain *rd); #ifdef CONFIG_SCHED_CORE extern void sched_core_free(struct task_struct *tsk); extern void sched_core_fork(struct task_struct *p); +extern int sched_core_share_pid(unsigned int cmd, pid_t pid, enum pid_type type, + unsigned long uaddr); #else static inline void sched_core_free(struct task_struct *tsk) { } static inline void sched_core_fork(struct task_struct *p) { } diff --git a/include/uapi/linux/prctl.h b/include/uapi/linux/prctl.h index 18a9f59dc067..967d9c55323d 100644 --- a/include/uapi/linux/prctl.h +++ b/include/uapi/linux/prctl.h @@ -259,4 +259,12 @@ struct prctl_mm_map { #define PR_PAC_SET_ENABLED_KEYS 60 #define PR_PAC_GET_ENABLED_KEYS 61 +/* Request the scheduler to share a core */ +#define PR_SCHED_CORE 62 +# define PR_SCHED_CORE_GET 0 +# define PR_SCHED_CORE_CREATE 1 /* create unique core_sched cookie */ +# define PR_SCHED_CORE_SHARE_TO 2 /* push core_sched cookie to pid */ +# define PR_SCHED_CORE_SHARE_FROM 3 /* pull core_sched cookie to pid */ +# define PR_SCHED_CORE_MAX 4 + #endif /* _LINUX_PRCTL_H */ diff --git a/kernel/sched/core_sched.c b/kernel/sched/core_sched.c index dcbbeaefaaa3..9a80e9a474c0 100644 --- a/kernel/sched/core_sched.c +++ b/kernel/sched/core_sched.c @@ -1,5 +1,6 @@ // SPDX-License-Identifier: GPL-2.0-only +#include #include "sched.h" /* @@ -113,3 +114,116 @@ void sched_core_free(struct task_struct *p) { sched_core_put_cookie(p->core_cookie); } + +static void __sched_core_set(struct task_struct *p, unsigned long cookie) +{ + cookie = sched_core_get_cookie(cookie); + cookie = sched_core_update_cookie(p, cookie); + sched_core_put_cookie(cookie); +} + +/* Called from prctl interface: PR_SCHED_CORE */ +int sched_core_share_pid(unsigned int cmd, pid_t pid, enum pid_type type, + unsigned long uaddr) +{ + unsigned long cookie = 0, id = 0; + struct task_struct *task, *p; + struct pid *grp; + int err = 0; + + if (!static_branch_likely(&sched_smt_present)) + return -ENODEV; + + if (type > PIDTYPE_PGID || cmd >= PR_SCHED_CORE_MAX || pid < 0 || + (cmd != PR_SCHED_CORE_GET && uaddr)) + return -EINVAL; + + rcu_read_lock(); + if (pid == 0) { + task = current; + } else { + task = find_task_by_vpid(pid); + if (!task) { + rcu_read_unlock(); + return -ESRCH; + } + } + get_task_struct(task); + rcu_read_unlock(); + + /* + * Check if this process has the right to modify the specified + * process. Use the regular "ptrace_may_access()" checks. + */ + if (!ptrace_may_access(task, PTRACE_MODE_READ_REALCREDS)) { + err = -EPERM; + goto out; + } + + switch (cmd) { + case PR_SCHED_CORE_GET: + if (type != PIDTYPE_PID || uaddr & 7) { + err = -EINVAL; + goto out; + } + cookie = sched_core_clone_cookie(task); + if (cookie) { + /* XXX improve ? */ + ptr_to_hashval((void *)cookie, &id); + } + err = put_user(id, (u64 __user *)uaddr); + goto out; + + case PR_SCHED_CORE_CREATE: + cookie = sched_core_alloc_cookie(); + if (!cookie) { + err = -ENOMEM; + goto out; + } + break; + + case PR_SCHED_CORE_SHARE_TO: + cookie = sched_core_clone_cookie(current); + break; + + case PR_SCHED_CORE_SHARE_FROM: + if (type != PIDTYPE_PID) { + err = -EINVAL; + goto out; + } + cookie = sched_core_clone_cookie(task); + __sched_core_set(current, cookie); + goto out; + + default: + err = -EINVAL; + goto out; + }; + + if (type == PIDTYPE_PID) { + __sched_core_set(task, cookie); + goto out; + } + + read_lock(&tasklist_lock); + grp = task_pid_type(task, type); + + do_each_pid_thread(grp, type, p) { + if (!ptrace_may_access(p, PTRACE_MODE_READ_REALCREDS)) { + err = -EPERM; + goto out_tasklist; + } + } while_each_pid_thread(grp, type, p); + + do_each_pid_thread(grp, type, p) { + __sched_core_set(p, cookie); + } while_each_pid_thread(grp, type, p); +out_tasklist: + read_unlock(&tasklist_lock); + +out: + sched_core_put_cookie(cookie); + put_task_struct(task); + return err; +} + diff --git a/kernel/sys.c b/kernel/sys.c index 3a583a29815f..9de46a4bf492 100644 --- a/kernel/sys.c +++ b/kernel/sys.c @@ -2550,6 +2550,11 @@ SYSCALL_DEFINE5(prctl, int, option, unsigned long, arg2, unsigned long, arg3, error = set_syscall_user_dispatch(arg2, arg3, arg4, (char __user *) arg5); break; +#ifdef CONFIG_SCHED_CORE + case PR_SCHED_CORE: + error = sched_core_share_pid(arg2, arg3, arg4, arg5); + break; +#endif default: error = -EINVAL; break; diff --git a/tools/include/uapi/linux/prctl.h b/tools/include/uapi/linux/prctl.h index 18a9f59dc067..967d9c55323d 100644 --- a/tools/include/uapi/linux/prctl.h +++ b/tools/include/uapi/linux/prctl.h @@ -259,4 +259,12 @@ struct prctl_mm_map { #define PR_PAC_SET_ENABLED_KEYS 60 #define PR_PAC_GET_ENABLED_KEYS 61 +/* Request the scheduler to share a core */ +#define PR_SCHED_CORE 62 +# define PR_SCHED_CORE_GET 0 +# define PR_SCHED_CORE_CREATE 1 /* create unique core_sched cookie */ +# define PR_SCHED_CORE_SHARE_TO 2 /* push core_sched cookie to pid */ +# define PR_SCHED_CORE_SHARE_FROM 3 /* pull core_sched cookie to pid */ +# define PR_SCHED_CORE_MAX 4 + #endif /* _LINUX_PRCTL_H */ -- cgit v1.2.3 From ed5aecd3da2eabd8a6c9f5593df2c4f00985fca2 Mon Sep 17 00:00:00 2001 From: Jiri Slaby Date: Wed, 5 May 2021 11:18:54 +0200 Subject: tty: remove broken r3964 line discipline Noone stepped up in the past two years since it was marked as BROKEN by commit c7084edc3f6d (tty: mark Siemens R3964 line discipline as BROKEN). Remove the line discipline for good. Three remarks: * we remove also the uapi header (as noone is able to use that interface anyway) * we do *not* remove the N_R3964 constant definition from tty.h, so it remains reserved. * in_interrupt() check is now removed from vt's con_put_char. Noone else calls tty_operations::put_char from interrupt context. Signed-off-by: Jiri Slaby Link: https://lore.kernel.org/r/20210505091928.22010-2-jslaby@suse.cz Signed-off-by: Greg Kroah-Hartman --- arch/powerpc/configs/ppc6xx_defconfig | 1 - drivers/char/Kconfig | 13 - drivers/tty/Makefile | 1 - drivers/tty/n_r3964.c | 1283 --------------------------------- drivers/tty/vt/vt.c | 2 - include/linux/n_r3964.h | 175 ----- include/uapi/linux/n_r3964.h | 99 --- 7 files changed, 1574 deletions(-) delete mode 100644 drivers/tty/n_r3964.c delete mode 100644 include/linux/n_r3964.h delete mode 100644 include/uapi/linux/n_r3964.h (limited to 'include/uapi') diff --git a/arch/powerpc/configs/ppc6xx_defconfig b/arch/powerpc/configs/ppc6xx_defconfig index 1fd9d1260f9e..ee09f7bb6ea9 100644 --- a/arch/powerpc/configs/ppc6xx_defconfig +++ b/arch/powerpc/configs/ppc6xx_defconfig @@ -621,7 +621,6 @@ CONFIG_HW_RANDOM=y CONFIG_HW_RANDOM_VIRTIO=m CONFIG_NVRAM=y CONFIG_DTLK=m -CONFIG_R3964=m CONFIG_CARDMAN_4000=m CONFIG_CARDMAN_4040=m CONFIG_IPWIRELESS=m diff --git a/drivers/char/Kconfig b/drivers/char/Kconfig index b151e0fcdeb5..52d0dd49a683 100644 --- a/drivers/char/Kconfig +++ b/drivers/char/Kconfig @@ -218,19 +218,6 @@ config XILINX_HWICAP If unsure, say N. -config R3964 - tristate "Siemens R3964 line discipline" - depends on TTY && BROKEN - help - This driver allows synchronous communication with devices using the - Siemens R3964 packet protocol. Unless you are dealing with special - hardware like PLCs, you are unlikely to need this. - - To compile this driver as a module, choose M here: the - module will be called n_r3964. - - If unsure, say N. - config APPLICOM tristate "Applicom intelligent fieldbus card support" depends on PCI diff --git a/drivers/tty/Makefile b/drivers/tty/Makefile index c7054f5117c3..a2bd75fbaaa4 100644 --- a/drivers/tty/Makefile +++ b/drivers/tty/Makefile @@ -9,7 +9,6 @@ obj-$(CONFIG_AUDIT) += tty_audit.o obj-$(CONFIG_MAGIC_SYSRQ) += sysrq.o obj-$(CONFIG_N_HDLC) += n_hdlc.o obj-$(CONFIG_N_GSM) += n_gsm.o -obj-$(CONFIG_R3964) += n_r3964.o obj-y += vt/ obj-$(CONFIG_HVC_DRIVER) += hvc/ diff --git a/drivers/tty/n_r3964.c b/drivers/tty/n_r3964.c deleted file mode 100644 index 2eb76ea1d88d..000000000000 --- a/drivers/tty/n_r3964.c +++ /dev/null @@ -1,1283 +0,0 @@ -// SPDX-License-Identifier: GPL-1.0+ -/* r3964 linediscipline for linux - * - * ----------------------------------------------------------- - * Copyright by - * Philips Automation Projects - * Kassel (Germany) - * ----------------------------------------------------------- - * Author: - * L. Haag - * - * $Log: n_r3964.c,v $ - * Revision 1.10 2001/03/18 13:02:24 dwmw2 - * Fix timer usage, use spinlocks properly. - * - * Revision 1.9 2001/03/18 12:52:14 dwmw2 - * Merge changes in 2.4.2 - * - * Revision 1.8 2000/03/23 14:14:54 dwmw2 - * Fix race in sleeping in r3964_read() - * - * Revision 1.7 1999/28/08 11:41:50 dwmw2 - * Port to 2.3 kernel - * - * Revision 1.6 1998/09/30 00:40:40 dwmw2 - * Fixed compilation on 2.0.x kernels - * Updated to newly registered tty-ldisc number 9 - * - * Revision 1.5 1998/09/04 21:57:36 dwmw2 - * Signal handling bug fixes, port to 2.1.x. - * - * Revision 1.4 1998/04/02 20:26:59 lhaag - * select, blocking, ... - * - * Revision 1.3 1998/02/12 18:58:43 root - * fixed some memory leaks - * calculation of checksum characters - * - * Revision 1.2 1998/02/07 13:03:34 root - * ioctl read_telegram - * - * Revision 1.1 1998/02/06 19:21:03 root - * Initial revision - * - * - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include /* used in new tty drivers */ -#include /* used in new tty drivers */ -#include -#include -#include -#include -#include - -/*#define DEBUG_QUEUE*/ - -/* Log successful handshake and protocol operations */ -/*#define DEBUG_PROTO_S*/ - -/* Log handshake and protocol errors: */ -/*#define DEBUG_PROTO_E*/ - -/* Log Linediscipline operations (open, close, read, write...): */ -/*#define DEBUG_LDISC*/ - -/* Log module and memory operations (init, cleanup; kmalloc, kfree): */ -/*#define DEBUG_MODUL*/ - -/* Macro helpers for debug output: */ -#define TRACE(format, args...) printk("r3964: " format "\n" , ## args) - -#ifdef DEBUG_MODUL -#define TRACE_M(format, args...) printk("r3964: " format "\n" , ## args) -#else -#define TRACE_M(fmt, arg...) do {} while (0) -#endif -#ifdef DEBUG_PROTO_S -#define TRACE_PS(format, args...) printk("r3964: " format "\n" , ## args) -#else -#define TRACE_PS(fmt, arg...) do {} while (0) -#endif -#ifdef DEBUG_PROTO_E -#define TRACE_PE(format, args...) printk("r3964: " format "\n" , ## args) -#else -#define TRACE_PE(fmt, arg...) do {} while (0) -#endif -#ifdef DEBUG_LDISC -#define TRACE_L(format, args...) printk("r3964: " format "\n" , ## args) -#else -#define TRACE_L(fmt, arg...) do {} while (0) -#endif -#ifdef DEBUG_QUEUE -#define TRACE_Q(format, args...) printk("r3964: " format "\n" , ## args) -#else -#define TRACE_Q(fmt, arg...) do {} while (0) -#endif -static void add_tx_queue(struct r3964_info *, struct r3964_block_header *); -static void remove_from_tx_queue(struct r3964_info *pInfo, int error_code); -static void put_char(struct r3964_info *pInfo, unsigned char ch); -static void trigger_transmit(struct r3964_info *pInfo); -static void retry_transmit(struct r3964_info *pInfo); -static void transmit_block(struct r3964_info *pInfo); -static void receive_char(struct r3964_info *pInfo, const unsigned char c); -static void receive_error(struct r3964_info *pInfo, const char flag); -static void on_timeout(struct timer_list *t); -static int enable_signals(struct r3964_info *pInfo, struct pid *pid, int arg); -static int read_telegram(struct r3964_info *pInfo, struct pid *pid, - unsigned char __user * buf); -static void add_msg(struct r3964_client_info *pClient, int msg_id, int arg, - int error_code, struct r3964_block_header *pBlock); -static struct r3964_message *remove_msg(struct r3964_info *pInfo, - struct r3964_client_info *pClient); -static void remove_client_block(struct r3964_info *pInfo, - struct r3964_client_info *pClient); - -static int r3964_open(struct tty_struct *tty); -static void r3964_close(struct tty_struct *tty); -static ssize_t r3964_read(struct tty_struct *tty, struct file *file, - void *cookie, unsigned char *buf, size_t nr); -static ssize_t r3964_write(struct tty_struct *tty, struct file *file, - const unsigned char *buf, size_t nr); -static int r3964_ioctl(struct tty_struct *tty, struct file *file, - unsigned int cmd, unsigned long arg); -#ifdef CONFIG_COMPAT -static int r3964_compat_ioctl(struct tty_struct *tty, struct file *file, - unsigned int cmd, unsigned long arg); -#endif -static void r3964_set_termios(struct tty_struct *tty, struct ktermios *old); -static __poll_t r3964_poll(struct tty_struct *tty, struct file *file, - struct poll_table_struct *wait); -static void r3964_receive_buf(struct tty_struct *tty, const unsigned char *cp, - char *fp, int count); - -static struct tty_ldisc_ops tty_ldisc_N_R3964 = { - .owner = THIS_MODULE, - .name = "R3964", - .open = r3964_open, - .close = r3964_close, - .read = r3964_read, - .write = r3964_write, - .ioctl = r3964_ioctl, -#ifdef CONFIG_COMPAT - .compat_ioctl = r3964_compat_ioctl, -#endif - .set_termios = r3964_set_termios, - .poll = r3964_poll, - .receive_buf = r3964_receive_buf, -}; - -static void dump_block(const unsigned char *block, unsigned int length) -{ - unsigned int i, j; - char linebuf[16 * 3 + 1]; - - for (i = 0; i < length; i += 16) { - for (j = 0; (j < 16) && (j + i < length); j++) { - sprintf(linebuf + 3 * j, "%02x ", block[i + j]); - } - linebuf[3 * j] = '\0'; - TRACE_PS("%s", linebuf); - } -} - -/************************************************************* - * Driver initialisation - *************************************************************/ - -/************************************************************* - * Module support routines - *************************************************************/ - -static void __exit r3964_exit(void) -{ - int status; - - TRACE_M("cleanup_module()"); - - status = tty_unregister_ldisc(N_R3964); - - if (status != 0) { - printk(KERN_ERR "r3964: error unregistering linediscipline: " - "%d\n", status); - } else { - TRACE_L("linediscipline successfully unregistered"); - } -} - -static int __init r3964_init(void) -{ - int status; - - printk("r3964: Philips r3964 Driver $Revision: 1.10 $\n"); - - /* - * Register the tty line discipline - */ - - status = tty_register_ldisc(N_R3964, &tty_ldisc_N_R3964); - if (status == 0) { - TRACE_L("line discipline %d registered", N_R3964); - TRACE_L("flags=%x num=%x", tty_ldisc_N_R3964.flags, - tty_ldisc_N_R3964.num); - TRACE_L("open=%p", tty_ldisc_N_R3964.open); - TRACE_L("tty_ldisc_N_R3964 = %p", &tty_ldisc_N_R3964); - } else { - printk(KERN_ERR "r3964: error registering line discipline: " - "%d\n", status); - } - return status; -} - -module_init(r3964_init); -module_exit(r3964_exit); - -/************************************************************* - * Protocol implementation routines - *************************************************************/ - -static void add_tx_queue(struct r3964_info *pInfo, - struct r3964_block_header *pHeader) -{ - unsigned long flags; - - spin_lock_irqsave(&pInfo->lock, flags); - - pHeader->next = NULL; - - if (pInfo->tx_last == NULL) { - pInfo->tx_first = pInfo->tx_last = pHeader; - } else { - pInfo->tx_last->next = pHeader; - pInfo->tx_last = pHeader; - } - - spin_unlock_irqrestore(&pInfo->lock, flags); - - TRACE_Q("add_tx_queue %p, length %d, tx_first = %p", - pHeader, pHeader->length, pInfo->tx_first); -} - -static void remove_from_tx_queue(struct r3964_info *pInfo, int error_code) -{ - struct r3964_block_header *pHeader; - unsigned long flags; -#ifdef DEBUG_QUEUE - struct r3964_block_header *pDump; -#endif - - pHeader = pInfo->tx_first; - - if (pHeader == NULL) - return; - -#ifdef DEBUG_QUEUE - printk("r3964: remove_from_tx_queue: %p, length %u - ", - pHeader, pHeader->length); - for (pDump = pHeader; pDump; pDump = pDump->next) - printk("%p ", pDump); - printk("\n"); -#endif - - if (pHeader->owner) { - if (error_code) { - add_msg(pHeader->owner, R3964_MSG_ACK, 0, - error_code, NULL); - } else { - add_msg(pHeader->owner, R3964_MSG_ACK, pHeader->length, - error_code, NULL); - } - wake_up_interruptible(&pInfo->tty->read_wait); - } - - spin_lock_irqsave(&pInfo->lock, flags); - - pInfo->tx_first = pHeader->next; - if (pInfo->tx_first == NULL) { - pInfo->tx_last = NULL; - } - - spin_unlock_irqrestore(&pInfo->lock, flags); - - kfree(pHeader); - TRACE_M("remove_from_tx_queue - kfree %p", pHeader); - - TRACE_Q("remove_from_tx_queue: tx_first = %p, tx_last = %p", - pInfo->tx_first, pInfo->tx_last); -} - -static void add_rx_queue(struct r3964_info *pInfo, - struct r3964_block_header *pHeader) -{ - unsigned long flags; - - spin_lock_irqsave(&pInfo->lock, flags); - - pHeader->next = NULL; - - if (pInfo->rx_last == NULL) { - pInfo->rx_first = pInfo->rx_last = pHeader; - } else { - pInfo->rx_last->next = pHeader; - pInfo->rx_last = pHeader; - } - pInfo->blocks_in_rx_queue++; - - spin_unlock_irqrestore(&pInfo->lock, flags); - - TRACE_Q("add_rx_queue: %p, length = %d, rx_first = %p, count = %d", - pHeader, pHeader->length, - pInfo->rx_first, pInfo->blocks_in_rx_queue); -} - -static void remove_from_rx_queue(struct r3964_info *pInfo, - struct r3964_block_header *pHeader) -{ - unsigned long flags; - struct r3964_block_header *pFind; - - if (pHeader == NULL) - return; - - TRACE_Q("remove_from_rx_queue: rx_first = %p, rx_last = %p, count = %d", - pInfo->rx_first, pInfo->rx_last, pInfo->blocks_in_rx_queue); - TRACE_Q("remove_from_rx_queue: %p, length %u", - pHeader, pHeader->length); - - spin_lock_irqsave(&pInfo->lock, flags); - - if (pInfo->rx_first == pHeader) { - /* Remove the first block in the linked list: */ - pInfo->rx_first = pHeader->next; - - if (pInfo->rx_first == NULL) { - pInfo->rx_last = NULL; - } - pInfo->blocks_in_rx_queue--; - } else { - /* Find block to remove: */ - for (pFind = pInfo->rx_first; pFind; pFind = pFind->next) { - if (pFind->next == pHeader) { - /* Got it. */ - pFind->next = pHeader->next; - pInfo->blocks_in_rx_queue--; - if (pFind->next == NULL) { - /* Oh, removed the last one! */ - pInfo->rx_last = pFind; - } - break; - } - } - } - - spin_unlock_irqrestore(&pInfo->lock, flags); - - kfree(pHeader); - TRACE_M("remove_from_rx_queue - kfree %p", pHeader); - - TRACE_Q("remove_from_rx_queue: rx_first = %p, rx_last = %p, count = %d", - pInfo->rx_first, pInfo->rx_last, pInfo->blocks_in_rx_queue); -} - -static void put_char(struct r3964_info *pInfo, unsigned char ch) -{ - struct tty_struct *tty = pInfo->tty; - /* FIXME: put_char should not be called from an IRQ */ - tty_put_char(tty, ch); - pInfo->bcc ^= ch; -} - -static void flush(struct r3964_info *pInfo) -{ - struct tty_struct *tty = pInfo->tty; - - if (tty == NULL || tty->ops->flush_chars == NULL) - return; - tty->ops->flush_chars(tty); -} - -static void trigger_transmit(struct r3964_info *pInfo) -{ - unsigned long flags; - - spin_lock_irqsave(&pInfo->lock, flags); - - if ((pInfo->state == R3964_IDLE) && (pInfo->tx_first != NULL)) { - pInfo->state = R3964_TX_REQUEST; - pInfo->nRetry = 0; - pInfo->flags &= ~R3964_ERROR; - mod_timer(&pInfo->tmr, jiffies + R3964_TO_QVZ); - - spin_unlock_irqrestore(&pInfo->lock, flags); - - TRACE_PS("trigger_transmit - sent STX"); - - put_char(pInfo, STX); - flush(pInfo); - - pInfo->bcc = 0; - } else { - spin_unlock_irqrestore(&pInfo->lock, flags); - } -} - -static void retry_transmit(struct r3964_info *pInfo) -{ - if (pInfo->nRetry < R3964_MAX_RETRIES) { - TRACE_PE("transmission failed. Retry #%d", pInfo->nRetry); - pInfo->bcc = 0; - put_char(pInfo, STX); - flush(pInfo); - pInfo->state = R3964_TX_REQUEST; - pInfo->nRetry++; - mod_timer(&pInfo->tmr, jiffies + R3964_TO_QVZ); - } else { - TRACE_PE("transmission failed after %d retries", - R3964_MAX_RETRIES); - - remove_from_tx_queue(pInfo, R3964_TX_FAIL); - - put_char(pInfo, NAK); - flush(pInfo); - pInfo->state = R3964_IDLE; - - trigger_transmit(pInfo); - } -} - -static void transmit_block(struct r3964_info *pInfo) -{ - struct tty_struct *tty = pInfo->tty; - struct r3964_block_header *pBlock = pInfo->tx_first; - int room = 0; - - if (tty == NULL || pBlock == NULL) { - return; - } - - room = tty_write_room(tty); - - TRACE_PS("transmit_block %p, room %d, length %d", - pBlock, room, pBlock->length); - - while (pInfo->tx_position < pBlock->length) { - if (room < 2) - break; - - if (pBlock->data[pInfo->tx_position] == DLE) { - /* send additional DLE char: */ - put_char(pInfo, DLE); - } - put_char(pInfo, pBlock->data[pInfo->tx_position++]); - - room--; - } - - if ((pInfo->tx_position == pBlock->length) && (room >= 3)) { - put_char(pInfo, DLE); - put_char(pInfo, ETX); - if (pInfo->flags & R3964_BCC) { - put_char(pInfo, pInfo->bcc); - } - pInfo->state = R3964_WAIT_FOR_TX_ACK; - mod_timer(&pInfo->tmr, jiffies + R3964_TO_QVZ); - } - flush(pInfo); -} - -static void on_receive_block(struct r3964_info *pInfo) -{ - unsigned int length; - struct r3964_client_info *pClient; - struct r3964_block_header *pBlock; - - length = pInfo->rx_position; - - /* compare byte checksum characters: */ - if (pInfo->flags & R3964_BCC) { - if (pInfo->bcc != pInfo->last_rx) { - TRACE_PE("checksum error - got %x but expected %x", - pInfo->last_rx, pInfo->bcc); - pInfo->flags |= R3964_CHECKSUM; - } - } - - /* check for errors (parity, overrun,...): */ - if (pInfo->flags & R3964_ERROR) { - TRACE_PE("on_receive_block - transmission failed error %x", - pInfo->flags & R3964_ERROR); - - put_char(pInfo, NAK); - flush(pInfo); - if (pInfo->nRetry < R3964_MAX_RETRIES) { - pInfo->state = R3964_WAIT_FOR_RX_REPEAT; - pInfo->nRetry++; - mod_timer(&pInfo->tmr, jiffies + R3964_TO_RX_PANIC); - } else { - TRACE_PE("on_receive_block - failed after max retries"); - pInfo->state = R3964_IDLE; - } - return; - } - - /* received block; submit DLE: */ - put_char(pInfo, DLE); - flush(pInfo); - del_timer_sync(&pInfo->tmr); - TRACE_PS(" rx success: got %d chars", length); - - /* prepare struct r3964_block_header: */ - pBlock = kmalloc(length + sizeof(struct r3964_block_header), - GFP_KERNEL); - TRACE_M("on_receive_block - kmalloc %p", pBlock); - - if (pBlock == NULL) - return; - - pBlock->length = length; - pBlock->data = ((unsigned char *)pBlock) + - sizeof(struct r3964_block_header); - pBlock->locks = 0; - pBlock->next = NULL; - pBlock->owner = NULL; - - memcpy(pBlock->data, pInfo->rx_buf, length); - - /* queue block into rx_queue: */ - add_rx_queue(pInfo, pBlock); - - /* notify attached client processes: */ - for (pClient = pInfo->firstClient; pClient; pClient = pClient->next) { - if (pClient->sig_flags & R3964_SIG_DATA) { - add_msg(pClient, R3964_MSG_DATA, length, R3964_OK, - pBlock); - } - } - wake_up_interruptible(&pInfo->tty->read_wait); - - pInfo->state = R3964_IDLE; - - trigger_transmit(pInfo); -} - -static void receive_char(struct r3964_info *pInfo, const unsigned char c) -{ - switch (pInfo->state) { - case R3964_TX_REQUEST: - if (c == DLE) { - TRACE_PS("TX_REQUEST - got DLE"); - - pInfo->state = R3964_TRANSMITTING; - pInfo->tx_position = 0; - - transmit_block(pInfo); - } else if (c == STX) { - if (pInfo->nRetry == 0) { - TRACE_PE("TX_REQUEST - init conflict"); - if (pInfo->priority == R3964_SLAVE) { - goto start_receiving; - } - } else { - TRACE_PE("TX_REQUEST - secondary init " - "conflict!? Switching to SLAVE mode " - "for next rx."); - goto start_receiving; - } - } else { - TRACE_PE("TX_REQUEST - char != DLE: %x", c); - retry_transmit(pInfo); - } - break; - case R3964_TRANSMITTING: - if (c == NAK) { - TRACE_PE("TRANSMITTING - got NAK"); - retry_transmit(pInfo); - } else { - TRACE_PE("TRANSMITTING - got invalid char"); - - pInfo->state = R3964_WAIT_ZVZ_BEFORE_TX_RETRY; - mod_timer(&pInfo->tmr, jiffies + R3964_TO_ZVZ); - } - break; - case R3964_WAIT_FOR_TX_ACK: - if (c == DLE) { - TRACE_PS("WAIT_FOR_TX_ACK - got DLE"); - remove_from_tx_queue(pInfo, R3964_OK); - - pInfo->state = R3964_IDLE; - trigger_transmit(pInfo); - } else { - retry_transmit(pInfo); - } - break; - case R3964_WAIT_FOR_RX_REPEAT: - case R3964_IDLE: - if (c == STX) { - /* Prevent rx_queue from overflow: */ - if (pInfo->blocks_in_rx_queue >= - R3964_MAX_BLOCKS_IN_RX_QUEUE) { - TRACE_PE("IDLE - got STX but no space in " - "rx_queue!"); - pInfo->state = R3964_WAIT_FOR_RX_BUF; - mod_timer(&pInfo->tmr, - jiffies + R3964_TO_NO_BUF); - break; - } -start_receiving: - /* Ok, start receiving: */ - TRACE_PS("IDLE - got STX"); - pInfo->rx_position = 0; - pInfo->last_rx = 0; - pInfo->flags &= ~R3964_ERROR; - pInfo->state = R3964_RECEIVING; - mod_timer(&pInfo->tmr, jiffies + R3964_TO_ZVZ); - pInfo->nRetry = 0; - put_char(pInfo, DLE); - flush(pInfo); - pInfo->bcc = 0; - } - break; - case R3964_RECEIVING: - if (pInfo->rx_position < RX_BUF_SIZE) { - pInfo->bcc ^= c; - - if (c == DLE) { - if (pInfo->last_rx == DLE) { - pInfo->last_rx = 0; - goto char_to_buf; - } - pInfo->last_rx = DLE; - break; - } else if ((c == ETX) && (pInfo->last_rx == DLE)) { - if (pInfo->flags & R3964_BCC) { - pInfo->state = R3964_WAIT_FOR_BCC; - mod_timer(&pInfo->tmr, - jiffies + R3964_TO_ZVZ); - } else { - on_receive_block(pInfo); - } - } else { - pInfo->last_rx = c; -char_to_buf: - pInfo->rx_buf[pInfo->rx_position++] = c; - mod_timer(&pInfo->tmr, jiffies + R3964_TO_ZVZ); - } - } - /* else: overflow-msg? BUF_SIZE>MTU; should not happen? */ - break; - case R3964_WAIT_FOR_BCC: - pInfo->last_rx = c; - on_receive_block(pInfo); - break; - } -} - -static void receive_error(struct r3964_info *pInfo, const char flag) -{ - switch (flag) { - case TTY_NORMAL: - break; - case TTY_BREAK: - TRACE_PE("received break"); - pInfo->flags |= R3964_BREAK; - break; - case TTY_PARITY: - TRACE_PE("parity error"); - pInfo->flags |= R3964_PARITY; - break; - case TTY_FRAME: - TRACE_PE("frame error"); - pInfo->flags |= R3964_FRAME; - break; - case TTY_OVERRUN: - TRACE_PE("frame overrun"); - pInfo->flags |= R3964_OVERRUN; - break; - default: - TRACE_PE("receive_error - unknown flag %d", flag); - pInfo->flags |= R3964_UNKNOWN; - break; - } -} - -static void on_timeout(struct timer_list *t) -{ - struct r3964_info *pInfo = from_timer(pInfo, t, tmr); - - switch (pInfo->state) { - case R3964_TX_REQUEST: - TRACE_PE("TX_REQUEST - timeout"); - retry_transmit(pInfo); - break; - case R3964_WAIT_ZVZ_BEFORE_TX_RETRY: - put_char(pInfo, NAK); - flush(pInfo); - retry_transmit(pInfo); - break; - case R3964_WAIT_FOR_TX_ACK: - TRACE_PE("WAIT_FOR_TX_ACK - timeout"); - retry_transmit(pInfo); - break; - case R3964_WAIT_FOR_RX_BUF: - TRACE_PE("WAIT_FOR_RX_BUF - timeout"); - put_char(pInfo, NAK); - flush(pInfo); - pInfo->state = R3964_IDLE; - break; - case R3964_RECEIVING: - TRACE_PE("RECEIVING - timeout after %d chars", - pInfo->rx_position); - put_char(pInfo, NAK); - flush(pInfo); - pInfo->state = R3964_IDLE; - break; - case R3964_WAIT_FOR_RX_REPEAT: - TRACE_PE("WAIT_FOR_RX_REPEAT - timeout"); - pInfo->state = R3964_IDLE; - break; - case R3964_WAIT_FOR_BCC: - TRACE_PE("WAIT_FOR_BCC - timeout"); - put_char(pInfo, NAK); - flush(pInfo); - pInfo->state = R3964_IDLE; - break; - } -} - -static struct r3964_client_info *findClient(struct r3964_info *pInfo, - struct pid *pid) -{ - struct r3964_client_info *pClient; - - for (pClient = pInfo->firstClient; pClient; pClient = pClient->next) { - if (pClient->pid == pid) { - return pClient; - } - } - return NULL; -} - -static int enable_signals(struct r3964_info *pInfo, struct pid *pid, int arg) -{ - struct r3964_client_info *pClient; - struct r3964_client_info **ppClient; - struct r3964_message *pMsg; - - if ((arg & R3964_SIG_ALL) == 0) { - /* Remove client from client list */ - for (ppClient = &pInfo->firstClient; *ppClient; - ppClient = &(*ppClient)->next) { - pClient = *ppClient; - - if (pClient->pid == pid) { - TRACE_PS("removing client %d from client list", - pid_nr(pid)); - *ppClient = pClient->next; - while (pClient->msg_count) { - pMsg = remove_msg(pInfo, pClient); - if (pMsg) { - kfree(pMsg); - TRACE_M("enable_signals - msg " - "kfree %p", pMsg); - } - } - put_pid(pClient->pid); - kfree(pClient); - TRACE_M("enable_signals - kfree %p", pClient); - return 0; - } - } - return -EINVAL; - } else { - pClient = findClient(pInfo, pid); - if (pClient) { - /* update signal options */ - pClient->sig_flags = arg; - } else { - /* add client to client list */ - pClient = kmalloc(sizeof(struct r3964_client_info), - GFP_KERNEL); - TRACE_M("enable_signals - kmalloc %p", pClient); - if (pClient == NULL) - return -ENOMEM; - - TRACE_PS("add client %d to client list", pid_nr(pid)); - spin_lock_init(&pClient->lock); - pClient->sig_flags = arg; - pClient->pid = get_pid(pid); - pClient->next = pInfo->firstClient; - pClient->first_msg = NULL; - pClient->last_msg = NULL; - pClient->next_block_to_read = NULL; - pClient->msg_count = 0; - pInfo->firstClient = pClient; - } - } - - return 0; -} - -static int read_telegram(struct r3964_info *pInfo, struct pid *pid, - unsigned char __user * buf) -{ - struct r3964_client_info *pClient; - struct r3964_block_header *block; - - if (!buf) { - return -EINVAL; - } - - pClient = findClient(pInfo, pid); - if (pClient == NULL) { - return -EINVAL; - } - - block = pClient->next_block_to_read; - if (!block) { - return 0; - } else { - if (copy_to_user(buf, block->data, block->length)) - return -EFAULT; - - remove_client_block(pInfo, pClient); - return block->length; - } - - return -EINVAL; -} - -static void add_msg(struct r3964_client_info *pClient, int msg_id, int arg, - int error_code, struct r3964_block_header *pBlock) -{ - struct r3964_message *pMsg; - unsigned long flags; - - if (pClient->msg_count < R3964_MAX_MSG_COUNT - 1) { -queue_the_message: - - pMsg = kmalloc(sizeof(struct r3964_message), - error_code ? GFP_ATOMIC : GFP_KERNEL); - TRACE_M("add_msg - kmalloc %p", pMsg); - if (pMsg == NULL) { - return; - } - - spin_lock_irqsave(&pClient->lock, flags); - - pMsg->msg_id = msg_id; - pMsg->arg = arg; - pMsg->error_code = error_code; - pMsg->block = pBlock; - pMsg->next = NULL; - - if (pClient->last_msg == NULL) { - pClient->first_msg = pClient->last_msg = pMsg; - } else { - pClient->last_msg->next = pMsg; - pClient->last_msg = pMsg; - } - - pClient->msg_count++; - - if (pBlock != NULL) { - pBlock->locks++; - } - spin_unlock_irqrestore(&pClient->lock, flags); - } else { - if ((pClient->last_msg->msg_id == R3964_MSG_ACK) - && (pClient->last_msg->error_code == R3964_OVERFLOW)) { - pClient->last_msg->arg++; - TRACE_PE("add_msg - inc prev OVERFLOW-msg"); - } else { - msg_id = R3964_MSG_ACK; - arg = 0; - error_code = R3964_OVERFLOW; - pBlock = NULL; - TRACE_PE("add_msg - queue OVERFLOW-msg"); - goto queue_the_message; - } - } - /* Send SIGIO signal to client process: */ - if (pClient->sig_flags & R3964_USE_SIGIO) { - kill_pid(pClient->pid, SIGIO, 1); - } -} - -static struct r3964_message *remove_msg(struct r3964_info *pInfo, - struct r3964_client_info *pClient) -{ - struct r3964_message *pMsg = NULL; - unsigned long flags; - - if (pClient->first_msg) { - spin_lock_irqsave(&pClient->lock, flags); - - pMsg = pClient->first_msg; - pClient->first_msg = pMsg->next; - if (pClient->first_msg == NULL) { - pClient->last_msg = NULL; - } - - pClient->msg_count--; - if (pMsg->block) { - remove_client_block(pInfo, pClient); - pClient->next_block_to_read = pMsg->block; - } - spin_unlock_irqrestore(&pClient->lock, flags); - } - return pMsg; -} - -static void remove_client_block(struct r3964_info *pInfo, - struct r3964_client_info *pClient) -{ - struct r3964_block_header *block; - - TRACE_PS("remove_client_block PID %d", pid_nr(pClient->pid)); - - block = pClient->next_block_to_read; - if (block) { - block->locks--; - if (block->locks == 0) { - remove_from_rx_queue(pInfo, block); - } - } - pClient->next_block_to_read = NULL; -} - -/************************************************************* - * Line discipline routines - *************************************************************/ - -static int r3964_open(struct tty_struct *tty) -{ - struct r3964_info *pInfo; - - TRACE_L("open"); - TRACE_L("tty=%p, PID=%d, disc_data=%p", - tty, current->pid, tty->disc_data); - - pInfo = kmalloc(sizeof(struct r3964_info), GFP_KERNEL); - TRACE_M("r3964_open - info kmalloc %p", pInfo); - - if (!pInfo) { - printk(KERN_ERR "r3964: failed to alloc info structure\n"); - return -ENOMEM; - } - - pInfo->rx_buf = kmalloc(RX_BUF_SIZE, GFP_KERNEL); - TRACE_M("r3964_open - rx_buf kmalloc %p", pInfo->rx_buf); - - if (!pInfo->rx_buf) { - printk(KERN_ERR "r3964: failed to alloc receive buffer\n"); - kfree(pInfo); - TRACE_M("r3964_open - info kfree %p", pInfo); - return -ENOMEM; - } - - pInfo->tx_buf = kmalloc(TX_BUF_SIZE, GFP_KERNEL); - TRACE_M("r3964_open - tx_buf kmalloc %p", pInfo->tx_buf); - - if (!pInfo->tx_buf) { - printk(KERN_ERR "r3964: failed to alloc transmit buffer\n"); - kfree(pInfo->rx_buf); - TRACE_M("r3964_open - rx_buf kfree %p", pInfo->rx_buf); - kfree(pInfo); - TRACE_M("r3964_open - info kfree %p", pInfo); - return -ENOMEM; - } - - spin_lock_init(&pInfo->lock); - mutex_init(&pInfo->read_lock); - pInfo->tty = tty; - pInfo->priority = R3964_MASTER; - pInfo->rx_first = pInfo->rx_last = NULL; - pInfo->tx_first = pInfo->tx_last = NULL; - pInfo->rx_position = 0; - pInfo->tx_position = 0; - pInfo->last_rx = 0; - pInfo->blocks_in_rx_queue = 0; - pInfo->firstClient = NULL; - pInfo->state = R3964_IDLE; - pInfo->flags = R3964_DEBUG; - pInfo->nRetry = 0; - - tty->disc_data = pInfo; - tty->receive_room = 65536; - - timer_setup(&pInfo->tmr, on_timeout, 0); - - return 0; -} - -static void r3964_close(struct tty_struct *tty) -{ - struct r3964_info *pInfo = tty->disc_data; - struct r3964_client_info *pClient, *pNext; - struct r3964_message *pMsg; - struct r3964_block_header *pHeader, *pNextHeader; - unsigned long flags; - - TRACE_L("close"); - - /* - * Make sure that our task queue isn't activated. If it - * is, take it out of the linked list. - */ - del_timer_sync(&pInfo->tmr); - - /* Remove client-structs and message queues: */ - pClient = pInfo->firstClient; - while (pClient) { - pNext = pClient->next; - while (pClient->msg_count) { - pMsg = remove_msg(pInfo, pClient); - if (pMsg) { - kfree(pMsg); - TRACE_M("r3964_close - msg kfree %p", pMsg); - } - } - put_pid(pClient->pid); - kfree(pClient); - TRACE_M("r3964_close - client kfree %p", pClient); - pClient = pNext; - } - /* Remove jobs from tx_queue: */ - spin_lock_irqsave(&pInfo->lock, flags); - pHeader = pInfo->tx_first; - pInfo->tx_first = pInfo->tx_last = NULL; - spin_unlock_irqrestore(&pInfo->lock, flags); - - while (pHeader) { - pNextHeader = pHeader->next; - kfree(pHeader); - pHeader = pNextHeader; - } - - /* Free buffers: */ - kfree(pInfo->rx_buf); - TRACE_M("r3964_close - rx_buf kfree %p", pInfo->rx_buf); - kfree(pInfo->tx_buf); - TRACE_M("r3964_close - tx_buf kfree %p", pInfo->tx_buf); - kfree(pInfo); - TRACE_M("r3964_close - info kfree %p", pInfo); -} - -static ssize_t r3964_read(struct tty_struct *tty, struct file *file, - unsigned char *kbuf, size_t nr, - void **cookie, unsigned long offset) -{ - struct r3964_info *pInfo = tty->disc_data; - struct r3964_client_info *pClient; - struct r3964_message *pMsg; - struct r3964_client_message theMsg; - int ret; - - TRACE_L("read()"); - - /* - * Internal serialization of reads. - */ - if (file->f_flags & O_NONBLOCK) { - if (!mutex_trylock(&pInfo->read_lock)) - return -EAGAIN; - } else { - if (mutex_lock_interruptible(&pInfo->read_lock)) - return -ERESTARTSYS; - } - - pClient = findClient(pInfo, task_pid(current)); - if (pClient) { - pMsg = remove_msg(pInfo, pClient); - if (pMsg == NULL) { - /* no messages available. */ - if (tty_io_nonblock(tty, file)) { - ret = -EAGAIN; - goto unlock; - } - /* block until there is a message: */ - wait_event_interruptible(tty->read_wait, - (pMsg = remove_msg(pInfo, pClient))); - } - - /* If we still haven't got a message, we must have been signalled */ - - if (!pMsg) { - ret = -EINTR; - goto unlock; - } - - /* deliver msg to client process: */ - theMsg.msg_id = pMsg->msg_id; - theMsg.arg = pMsg->arg; - theMsg.error_code = pMsg->error_code; - ret = sizeof(struct r3964_client_message); - - kfree(pMsg); - TRACE_M("r3964_read - msg kfree %p", pMsg); - - memcpy(kbuf, &theMsg, ret); - - TRACE_PS("read - return %d", ret); - goto unlock; - } - ret = -EPERM; -unlock: - mutex_unlock(&pInfo->read_lock); - return ret; -} - -static ssize_t r3964_write(struct tty_struct *tty, struct file *file, - const unsigned char *data, size_t count) -{ - struct r3964_info *pInfo = tty->disc_data; - struct r3964_block_header *pHeader; - struct r3964_client_info *pClient; - unsigned char *new_data; - - TRACE_L("write request, %d characters", count); -/* - * Verify the pointers - */ - - if (!pInfo) - return -EIO; - -/* - * Ensure that the caller does not wish to send too much. - */ - if (count > R3964_MTU) { - if (pInfo->flags & R3964_DEBUG) { - TRACE_L(KERN_WARNING "r3964_write: truncating user " - "packet from %u to mtu %d", count, R3964_MTU); - } - count = R3964_MTU; - } -/* - * Allocate a buffer for the data and copy it from the buffer with header prepended - */ - new_data = kmalloc(count + sizeof(struct r3964_block_header), - GFP_KERNEL); - TRACE_M("r3964_write - kmalloc %p", new_data); - if (new_data == NULL) { - if (pInfo->flags & R3964_DEBUG) { - printk(KERN_ERR "r3964_write: no memory\n"); - } - return -ENOSPC; - } - - pHeader = (struct r3964_block_header *)new_data; - pHeader->data = new_data + sizeof(struct r3964_block_header); - pHeader->length = count; - pHeader->locks = 0; - pHeader->owner = NULL; - - pClient = findClient(pInfo, task_pid(current)); - if (pClient) { - pHeader->owner = pClient; - } - - memcpy(pHeader->data, data, count); /* We already verified this */ - - if (pInfo->flags & R3964_DEBUG) { - dump_block(pHeader->data, count); - } - -/* - * Add buffer to transmit-queue: - */ - add_tx_queue(pInfo, pHeader); - trigger_transmit(pInfo); - - return 0; -} - -static int r3964_ioctl(struct tty_struct *tty, struct file *file, - unsigned int cmd, unsigned long arg) -{ - struct r3964_info *pInfo = tty->disc_data; - if (pInfo == NULL) - return -EINVAL; - switch (cmd) { - case R3964_ENABLE_SIGNALS: - return enable_signals(pInfo, task_pid(current), arg); - case R3964_SETPRIORITY: - if (arg < R3964_MASTER || arg > R3964_SLAVE) - return -EINVAL; - pInfo->priority = arg & 0xff; - return 0; - case R3964_USE_BCC: - if (arg) - pInfo->flags |= R3964_BCC; - else - pInfo->flags &= ~R3964_BCC; - return 0; - case R3964_READ_TELEGRAM: - return read_telegram(pInfo, task_pid(current), - (unsigned char __user *)arg); - default: - return -ENOIOCTLCMD; - } -} - -#ifdef CONFIG_COMPAT -static int r3964_compat_ioctl(struct tty_struct *tty, struct file *file, - unsigned int cmd, unsigned long arg) -{ - switch (cmd) { - case R3964_ENABLE_SIGNALS: - case R3964_SETPRIORITY: - case R3964_USE_BCC: - return r3964_ioctl(tty, file, cmd, arg); - default: - return -ENOIOCTLCMD; - } -} -#endif - -static void r3964_set_termios(struct tty_struct *tty, struct ktermios *old) -{ - TRACE_L("set_termios"); -} - -/* Called without the kernel lock held - fine */ -static __poll_t r3964_poll(struct tty_struct *tty, struct file *file, - struct poll_table_struct *wait) -{ - struct r3964_info *pInfo = tty->disc_data; - struct r3964_client_info *pClient; - struct r3964_message *pMsg = NULL; - unsigned long flags; - __poll_t result = EPOLLOUT; - - TRACE_L("POLL"); - - pClient = findClient(pInfo, task_pid(current)); - if (pClient) { - poll_wait(file, &tty->read_wait, wait); - spin_lock_irqsave(&pInfo->lock, flags); - pMsg = pClient->first_msg; - spin_unlock_irqrestore(&pInfo->lock, flags); - if (pMsg) - result |= EPOLLIN | EPOLLRDNORM; - } else { - result = -EINVAL; - } - return result; -} - -static void r3964_receive_buf(struct tty_struct *tty, const unsigned char *cp, - char *fp, int count) -{ - struct r3964_info *pInfo = tty->disc_data; - const unsigned char *p; - char *f, flags = TTY_NORMAL; - int i; - - for (i = count, p = cp, f = fp; i; i--, p++) { - if (f) - flags = *f++; - if (flags == TTY_NORMAL) { - receive_char(pInfo, *p); - } else { - receive_error(pInfo, flags); - } - - } -} - -MODULE_LICENSE("GPL"); -MODULE_ALIAS_LDISC(N_R3964); diff --git a/drivers/tty/vt/vt.c b/drivers/tty/vt/vt.c index 01645e87b3d5..e5040bdadcd6 100644 --- a/drivers/tty/vt/vt.c +++ b/drivers/tty/vt/vt.c @@ -3260,8 +3260,6 @@ static int con_write(struct tty_struct *tty, const unsigned char *buf, int count static int con_put_char(struct tty_struct *tty, unsigned char ch) { - if (in_interrupt()) - return 0; /* n_r3964 calls put_char() from interrupt context */ return do_con_write(tty, &ch, 1); } diff --git a/include/linux/n_r3964.h b/include/linux/n_r3964.h deleted file mode 100644 index 90a803aa42e8..000000000000 --- a/include/linux/n_r3964.h +++ /dev/null @@ -1,175 +0,0 @@ -/* r3964 linediscipline for linux - * - * ----------------------------------------------------------- - * Copyright by - * Philips Automation Projects - * Kassel (Germany) - * ----------------------------------------------------------- - * This software may be used and distributed according to the terms of - * the GNU General Public License, incorporated herein by reference. - * - * Author: - * L. Haag - * - * $Log: r3964.h,v $ - * Revision 1.4 2005/12/21 19:54:24 Kurt Huwig - * Fixed HZ usage on 2.6 kernels - * Removed unnecessary include - * - * Revision 1.3 2001/03/18 13:02:24 dwmw2 - * Fix timer usage, use spinlocks properly. - * - * Revision 1.2 2001/03/18 12:53:15 dwmw2 - * Merge changes in 2.4.2 - * - * Revision 1.1.1.1 1998/10/13 16:43:14 dwmw2 - * This'll screw the version control - * - * Revision 1.6 1998/09/30 00:40:38 dwmw2 - * Updated to use kernel's N_R3964 if available - * - * Revision 1.4 1998/04/02 20:29:44 lhaag - * select, blocking, ... - * - * Revision 1.3 1998/02/12 18:58:43 root - * fixed some memory leaks - * calculation of checksum characters - * - * Revision 1.2 1998/02/07 13:03:17 root - * ioctl read_telegram - * - * Revision 1.1 1998/02/06 19:19:43 root - * Initial revision - * - * - */ -#ifndef __LINUX_N_R3964_H__ -#define __LINUX_N_R3964_H__ - - -#include -#include - -/* - * Common ascii handshake characters: - */ - -#define STX 0x02 -#define ETX 0x03 -#define DLE 0x10 -#define NAK 0x15 - -/* - * Timeouts (from milliseconds to jiffies) - */ - -#define R3964_TO_QVZ ((550)*HZ/1000) -#define R3964_TO_ZVZ ((220)*HZ/1000) -#define R3964_TO_NO_BUF ((400)*HZ/1000) -#define R3964_NO_TX_ROOM ((100)*HZ/1000) -#define R3964_TO_RX_PANIC ((4000)*HZ/1000) -#define R3964_MAX_RETRIES 5 - - -enum { R3964_IDLE, - R3964_TX_REQUEST, R3964_TRANSMITTING, - R3964_WAIT_ZVZ_BEFORE_TX_RETRY, R3964_WAIT_FOR_TX_ACK, - R3964_WAIT_FOR_RX_BUF, - R3964_RECEIVING, R3964_WAIT_FOR_BCC, R3964_WAIT_FOR_RX_REPEAT - }; - -/* - * All open file-handles are 'clients' and are stored in a linked list: - */ - -struct r3964_message; - -struct r3964_client_info { - spinlock_t lock; - struct pid *pid; - unsigned int sig_flags; - - struct r3964_client_info *next; - - struct r3964_message *first_msg; - struct r3964_message *last_msg; - struct r3964_block_header *next_block_to_read; - int msg_count; -}; - - - -struct r3964_block_header; - -/* internal version of client_message: */ -struct r3964_message { - int msg_id; - int arg; - int error_code; - struct r3964_block_header *block; - struct r3964_message *next; -}; - -/* - * Header of received block in rx_buf/tx_buf: - */ - -struct r3964_block_header -{ - unsigned int length; /* length in chars without header */ - unsigned char *data; /* usually data is located - immediately behind this struct */ - unsigned int locks; /* only used in rx_buffer */ - - struct r3964_block_header *next; - struct r3964_client_info *owner; /* =NULL in rx_buffer */ -}; - -/* - * If rx_buf hasn't enough space to store R3964_MTU chars, - * we will reject all incoming STX-requests by sending NAK. - */ - -#define RX_BUF_SIZE 4000 -#define TX_BUF_SIZE 4000 -#define R3964_MAX_BLOCKS_IN_RX_QUEUE 100 - -#define R3964_PARITY 0x0001 -#define R3964_FRAME 0x0002 -#define R3964_OVERRUN 0x0004 -#define R3964_UNKNOWN 0x0008 -#define R3964_BREAK 0x0010 -#define R3964_CHECKSUM 0x0020 -#define R3964_ERROR 0x003f -#define R3964_BCC 0x4000 -#define R3964_DEBUG 0x8000 - - -struct r3964_info { - spinlock_t lock; - struct tty_struct *tty; - unsigned char priority; - unsigned char *rx_buf; /* ring buffer */ - unsigned char *tx_buf; - - struct r3964_block_header *rx_first; - struct r3964_block_header *rx_last; - struct r3964_block_header *tx_first; - struct r3964_block_header *tx_last; - unsigned int tx_position; - unsigned int rx_position; - unsigned char last_rx; - unsigned char bcc; - unsigned int blocks_in_rx_queue; - - struct mutex read_lock; /* serialize r3964_read */ - - struct r3964_client_info *firstClient; - unsigned int state; - unsigned int flags; - - struct timer_list tmr; - int nRetry; -}; - -#endif diff --git a/include/uapi/linux/n_r3964.h b/include/uapi/linux/n_r3964.h deleted file mode 100644 index 6bbd18520f30..000000000000 --- a/include/uapi/linux/n_r3964.h +++ /dev/null @@ -1,99 +0,0 @@ -/* SPDX-License-Identifier: GPL-1.0+ WITH Linux-syscall-note */ -/* r3964 linediscipline for linux - * - * ----------------------------------------------------------- - * Copyright by - * Philips Automation Projects - * Kassel (Germany) - * ----------------------------------------------------------- - * This software may be used and distributed according to the terms of - * the GNU General Public License, incorporated herein by reference. - * - * Author: - * L. Haag - * - * $Log: r3964.h,v $ - * Revision 1.4 2005/12/21 19:54:24 Kurt Huwig - * Fixed HZ usage on 2.6 kernels - * Removed unnecessary include - * - * Revision 1.3 2001/03/18 13:02:24 dwmw2 - * Fix timer usage, use spinlocks properly. - * - * Revision 1.2 2001/03/18 12:53:15 dwmw2 - * Merge changes in 2.4.2 - * - * Revision 1.1.1.1 1998/10/13 16:43:14 dwmw2 - * This'll screw the version control - * - * Revision 1.6 1998/09/30 00:40:38 dwmw2 - * Updated to use kernel's N_R3964 if available - * - * Revision 1.4 1998/04/02 20:29:44 lhaag - * select, blocking, ... - * - * Revision 1.3 1998/02/12 18:58:43 root - * fixed some memory leaks - * calculation of checksum characters - * - * Revision 1.2 1998/02/07 13:03:17 root - * ioctl read_telegram - * - * Revision 1.1 1998/02/06 19:19:43 root - * Initial revision - * - * - */ - -#ifndef _UAPI__LINUX_N_R3964_H__ -#define _UAPI__LINUX_N_R3964_H__ - -/* line disciplines for r3964 protocol */ - - -/* - * Ioctl-commands - */ - -#define R3964_ENABLE_SIGNALS 0x5301 -#define R3964_SETPRIORITY 0x5302 -#define R3964_USE_BCC 0x5303 -#define R3964_READ_TELEGRAM 0x5304 - -/* Options for R3964_SETPRIORITY */ -#define R3964_MASTER 0 -#define R3964_SLAVE 1 - -/* Options for R3964_ENABLE_SIGNALS */ -#define R3964_SIG_ACK 0x0001 -#define R3964_SIG_DATA 0x0002 -#define R3964_SIG_ALL 0x000f -#define R3964_SIG_NONE 0x0000 -#define R3964_USE_SIGIO 0x1000 - -/* - * r3964 operation states: - */ - -/* types for msg_id: */ -enum {R3964_MSG_ACK=1, R3964_MSG_DATA }; - -#define R3964_MAX_MSG_COUNT 32 - -/* error codes for client messages */ -#define R3964_OK 0 /* no error. */ -#define R3964_TX_FAIL -1 /* transmission error, block NOT sent */ -#define R3964_OVERFLOW -2 /* msg queue overflow */ - -/* the client gets this struct when calling read(fd,...): */ -struct r3964_client_message { - int msg_id; - int arg; - int error_code; -}; - -#define R3964_MTU 256 - - - -#endif /* _UAPI__LINUX_N_R3964_H__ */ -- cgit v1.2.3 From b7fb0916544de44ce099d9f3b6129c86b484de25 Mon Sep 17 00:00:00 2001 From: Linus Lüssing Date: Thu, 13 May 2021 15:20:52 +0200 Subject: net: bridge: mcast: add ip4+ip6 mcast router timers to mdb netlink MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Now that we have split the multicast router state into two, one for IPv4 and one for IPv6, also add individual timers to the mdb netlink router port dump. Leaving the old timer attribute for backwards compatibility. Signed-off-by: Linus Lüssing Signed-off-by: David S. Miller --- include/uapi/linux/if_bridge.h | 2 ++ net/bridge/br_mdb.c | 8 +++++++- 2 files changed, 9 insertions(+), 1 deletion(-) (limited to 'include/uapi') diff --git a/include/uapi/linux/if_bridge.h b/include/uapi/linux/if_bridge.h index 13d59c51ef5b..6b56a7549531 100644 --- a/include/uapi/linux/if_bridge.h +++ b/include/uapi/linux/if_bridge.h @@ -627,6 +627,8 @@ enum { MDBA_ROUTER_PATTR_UNSPEC, MDBA_ROUTER_PATTR_TIMER, MDBA_ROUTER_PATTR_TYPE, + MDBA_ROUTER_PATTR_INET_TIMER, + MDBA_ROUTER_PATTR_INET6_TIMER, __MDBA_ROUTER_PATTR_MAX }; #define MDBA_ROUTER_PATTR_MAX (__MDBA_ROUTER_PATTR_MAX - 1) diff --git a/net/bridge/br_mdb.c b/net/bridge/br_mdb.c index 10c416c7bf47..3f839a8cc9fb 100644 --- a/net/bridge/br_mdb.c +++ b/net/bridge/br_mdb.c @@ -79,7 +79,13 @@ static int br_rports_fill_info(struct sk_buff *skb, struct netlink_callback *cb, nla_put_u32(skb, MDBA_ROUTER_PATTR_TIMER, max(ip4_timer, ip6_timer)) || nla_put_u8(skb, MDBA_ROUTER_PATTR_TYPE, - p->multicast_router)) { + p->multicast_router) || + (have_ip4_mc_rtr && + nla_put_u32(skb, MDBA_ROUTER_PATTR_INET_TIMER, + ip4_timer)) || + (have_ip6_mc_rtr && + nla_put_u32(skb, MDBA_ROUTER_PATTR_INET6_TIMER, + ip6_timer))) { nla_nest_cancel(skb, port_nest); goto fail; } -- cgit v1.2.3 From 08fdced60ca08e34e316a3ab945636fcdfcbc973 Mon Sep 17 00:00:00 2001 From: David Henningsson Date: Sat, 15 May 2021 09:15:33 +0200 Subject: ALSA: rawmidi: Add framing mode This commit adds a new framing mode that frames all MIDI data into 32-byte frames with a timestamp. The main benefit is that we can get accurate timestamps even if userspace wakeup and processing is not immediate. Testing on a Celeron N3150 with this mode has a max jitter of 2.8 ms, compared to the in-kernel seq implementation which has a max jitter of 5 ms during idle and much worse when running scheduler stress tests in parallel. Signed-off-by: David Henningsson Reviewed-by: Jaroslav Kysela Link: https://lore.kernel.org/r/20210515071533.55332-1-coding@diwic.se Signed-off-by: Takashi Iwai --- include/sound/rawmidi.h | 2 + include/uapi/sound/asound.h | 30 ++++++++++++++- sound/core/rawmidi.c | 93 ++++++++++++++++++++++++++++++++++++++++++++- sound/core/rawmidi_compat.c | 4 +- 4 files changed, 124 insertions(+), 5 deletions(-) (limited to 'include/uapi') diff --git a/include/sound/rawmidi.h b/include/sound/rawmidi.h index 334842daa904..989e1517332d 100644 --- a/include/sound/rawmidi.h +++ b/include/sound/rawmidi.h @@ -81,6 +81,8 @@ struct snd_rawmidi_substream { bool opened; /* open flag */ bool append; /* append flag (merge more streams) */ bool active_sensing; /* send active sensing when close */ + unsigned int framing; /* whether to frame input data */ + unsigned int clock_type; /* clock source to use for input framing */ int use_count; /* use counter (for output) */ size_t bytes; struct snd_rawmidi *rmidi; diff --git a/include/uapi/sound/asound.h b/include/uapi/sound/asound.h index 535a7229e1d9..d17c061950df 100644 --- a/include/uapi/sound/asound.h +++ b/include/uapi/sound/asound.h @@ -710,7 +710,7 @@ enum { * Raw MIDI section - /dev/snd/midi?? */ -#define SNDRV_RAWMIDI_VERSION SNDRV_PROTOCOL_VERSION(2, 0, 1) +#define SNDRV_RAWMIDI_VERSION SNDRV_PROTOCOL_VERSION(2, 0, 2) enum { SNDRV_RAWMIDI_STREAM_OUTPUT = 0, @@ -736,12 +736,38 @@ struct snd_rawmidi_info { unsigned char reserved[64]; /* reserved for future use */ }; +#define SNDRV_RAWMIDI_MODE_FRAMING_MASK (7<<0) +#define SNDRV_RAWMIDI_MODE_FRAMING_SHIFT 0 +#define SNDRV_RAWMIDI_MODE_FRAMING_NONE (0<<0) +#define SNDRV_RAWMIDI_MODE_FRAMING_TSTAMP (1<<0) +#define SNDRV_RAWMIDI_MODE_CLOCK_MASK (7<<3) +#define SNDRV_RAWMIDI_MODE_CLOCK_SHIFT 3 +#define SNDRV_RAWMIDI_MODE_CLOCK_NONE (0<<3) +#define SNDRV_RAWMIDI_MODE_CLOCK_REALTIME (1<<3) +#define SNDRV_RAWMIDI_MODE_CLOCK_MONOTONIC (2<<3) +#define SNDRV_RAWMIDI_MODE_CLOCK_MONOTONIC_RAW (3<<3) + +#define SNDRV_RAWMIDI_FRAMING_DATA_LENGTH 16 + +struct snd_rawmidi_framing_tstamp { + /* For now, frame_type is always 0. Midi 2.0 is expected to add new + * types here. Applications are expected to skip unknown frame types. + */ + __u8 frame_type; + __u8 length; /* number of valid bytes in data field */ + __u8 reserved[2]; + __u32 tv_nsec; /* nanoseconds */ + __u64 tv_sec; /* seconds */ + __u8 data[SNDRV_RAWMIDI_FRAMING_DATA_LENGTH]; +} __packed; + struct snd_rawmidi_params { int stream; size_t buffer_size; /* queue size in bytes */ size_t avail_min; /* minimum avail bytes for wakeup */ unsigned int no_active_sensing: 1; /* do not send active sensing byte in close() */ - unsigned char reserved[16]; /* reserved for future use */ + unsigned int mode; /* For input data only, frame incoming data */ + unsigned char reserved[12]; /* reserved for future use */ }; #ifndef __KERNEL__ diff --git a/sound/core/rawmidi.c b/sound/core/rawmidi.c index aca00af93afe..4a6534db77d6 100644 --- a/sound/core/rawmidi.c +++ b/sound/core/rawmidi.c @@ -680,9 +680,12 @@ static int resize_runtime_buffer(struct snd_rawmidi_runtime *runtime, bool is_input) { char *newbuf, *oldbuf; + unsigned int framing = params->mode & SNDRV_RAWMIDI_MODE_FRAMING_MASK; if (params->buffer_size < 32 || params->buffer_size > 1024L * 1024L) return -EINVAL; + if (framing == SNDRV_RAWMIDI_MODE_FRAMING_TSTAMP && (params->buffer_size & 0x1f) != 0) + return -EINVAL; if (params->avail_min < 1 || params->avail_min > params->buffer_size) return -EINVAL; if (params->buffer_size != runtime->buffer_size) { @@ -720,8 +723,24 @@ EXPORT_SYMBOL(snd_rawmidi_output_params); int snd_rawmidi_input_params(struct snd_rawmidi_substream *substream, struct snd_rawmidi_params *params) { + unsigned int framing = params->mode & SNDRV_RAWMIDI_MODE_FRAMING_MASK; + unsigned int clock_type = params->mode & SNDRV_RAWMIDI_MODE_CLOCK_MASK; + int err; + + if (framing == SNDRV_RAWMIDI_MODE_FRAMING_NONE && clock_type != SNDRV_RAWMIDI_MODE_CLOCK_NONE) + return -EINVAL; + else if (clock_type > SNDRV_RAWMIDI_MODE_CLOCK_MONOTONIC_RAW) + return -EINVAL; + if (framing > SNDRV_RAWMIDI_MODE_FRAMING_TSTAMP) + return -EINVAL; snd_rawmidi_drain_input(substream); - return resize_runtime_buffer(substream->runtime, params, true); + err = resize_runtime_buffer(substream->runtime, params, true); + if (err < 0) + return err; + + substream->framing = framing; + substream->clock_type = clock_type; + return 0; } EXPORT_SYMBOL(snd_rawmidi_input_params); @@ -963,6 +982,62 @@ static int snd_rawmidi_control_ioctl(struct snd_card *card, return -ENOIOCTLCMD; } +static int receive_with_tstamp_framing(struct snd_rawmidi_substream *substream, + const unsigned char *buffer, int src_count, const struct timespec64 *tstamp) +{ + struct snd_rawmidi_runtime *runtime = substream->runtime; + struct snd_rawmidi_framing_tstamp *dest_ptr; + struct snd_rawmidi_framing_tstamp frame = { .tv_sec = tstamp->tv_sec, .tv_nsec = tstamp->tv_nsec }; + int dest_frames = 0; + int orig_count = src_count; + int frame_size = sizeof(struct snd_rawmidi_framing_tstamp); + + BUILD_BUG_ON(frame_size != 0x20); + if (snd_BUG_ON((runtime->hw_ptr & 0x1f) != 0)) + return -EINVAL; + + while (src_count > 0) { + if ((int)(runtime->buffer_size - runtime->avail) < frame_size) { + runtime->xruns += src_count; + break; + } + if (src_count >= SNDRV_RAWMIDI_FRAMING_DATA_LENGTH) + frame.length = SNDRV_RAWMIDI_FRAMING_DATA_LENGTH; + else { + frame.length = src_count; + memset(frame.data, 0, SNDRV_RAWMIDI_FRAMING_DATA_LENGTH); + } + memcpy(frame.data, buffer, frame.length); + buffer += frame.length; + src_count -= frame.length; + dest_ptr = (struct snd_rawmidi_framing_tstamp *) (runtime->buffer + runtime->hw_ptr); + *dest_ptr = frame; + runtime->avail += frame_size; + runtime->hw_ptr += frame_size; + runtime->hw_ptr %= runtime->buffer_size; + dest_frames++; + } + return orig_count - src_count; +} + +static struct timespec64 get_framing_tstamp(struct snd_rawmidi_substream *substream) +{ + struct timespec64 ts64 = {0, 0}; + + switch (substream->clock_type) { + case SNDRV_RAWMIDI_MODE_CLOCK_MONOTONIC_RAW: + ktime_get_raw_ts64(&ts64); + break; + case SNDRV_RAWMIDI_MODE_CLOCK_MONOTONIC: + ktime_get_ts64(&ts64); + break; + case SNDRV_RAWMIDI_MODE_CLOCK_REALTIME: + ktime_get_real_ts64(&ts64); + break; + } + return ts64; +} + /** * snd_rawmidi_receive - receive the input data from the device * @substream: the rawmidi substream @@ -977,6 +1052,7 @@ int snd_rawmidi_receive(struct snd_rawmidi_substream *substream, const unsigned char *buffer, int count) { unsigned long flags; + struct timespec64 ts64 = get_framing_tstamp(substream); int result = 0, count1; struct snd_rawmidi_runtime *runtime = substream->runtime; @@ -987,8 +1063,11 @@ int snd_rawmidi_receive(struct snd_rawmidi_substream *substream, "snd_rawmidi_receive: input is not active!!!\n"); return -EINVAL; } + spin_lock_irqsave(&runtime->lock, flags); - if (count == 1) { /* special case, faster code */ + if (substream->framing == SNDRV_RAWMIDI_MODE_FRAMING_TSTAMP) { + result = receive_with_tstamp_framing(substream, buffer, count, &ts64); + } else if (count == 1) { /* special case, faster code */ substream->bytes++; if (runtime->avail < runtime->buffer_size) { runtime->buffer[runtime->hw_ptr++] = buffer[0]; @@ -1541,6 +1620,8 @@ static void snd_rawmidi_proc_info_read(struct snd_info_entry *entry, struct snd_rawmidi_substream *substream; struct snd_rawmidi_runtime *runtime; unsigned long buffer_size, avail, xruns; + unsigned int clock_type; + static const char *clock_names[4] = { "none", "realtime", "monotonic", "monotonic raw" }; rmidi = entry->private_data; snd_iprintf(buffer, "%s\n\n", rmidi->name); @@ -1596,6 +1677,14 @@ static void snd_rawmidi_proc_info_read(struct snd_info_entry *entry, " Avail : %lu\n" " Overruns : %lu\n", buffer_size, avail, xruns); + if (substream->framing == SNDRV_RAWMIDI_MODE_FRAMING_TSTAMP) { + clock_type = substream->clock_type >> SNDRV_RAWMIDI_MODE_CLOCK_SHIFT; + if (!snd_BUG_ON(clock_type >= sizeof(clock_names))) + snd_iprintf(buffer, + " Framing : tstamp\n" + " Clock type : %s\n", + clock_names[clock_type]); + } } } } diff --git a/sound/core/rawmidi_compat.c b/sound/core/rawmidi_compat.c index 7397130976d0..68a93443583c 100644 --- a/sound/core/rawmidi_compat.c +++ b/sound/core/rawmidi_compat.c @@ -13,7 +13,8 @@ struct snd_rawmidi_params32 { u32 buffer_size; u32 avail_min; unsigned int no_active_sensing; /* avoid bit-field */ - unsigned char reserved[16]; + unsigned int mode; + unsigned char reserved[12]; } __attribute__((packed)); static int snd_rawmidi_ioctl_params_compat(struct snd_rawmidi_file *rfile, @@ -25,6 +26,7 @@ static int snd_rawmidi_ioctl_params_compat(struct snd_rawmidi_file *rfile, if (get_user(params.stream, &src->stream) || get_user(params.buffer_size, &src->buffer_size) || get_user(params.avail_min, &src->avail_min) || + get_user(params.mode, &src->mode) || get_user(val, &src->no_active_sensing)) return -EFAULT; params.no_active_sensing = val; -- cgit v1.2.3 From 79a7f8bdb159d9914b58740f3d31d602a6e4aca8 Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Thu, 13 May 2021 17:36:03 -0700 Subject: bpf: Introduce bpf_sys_bpf() helper and program type. Add placeholders for bpf_sys_bpf() helper and new program type. Make sure to check that expected_attach_type is zero for future extensibility. Allow tracing helper functions to be used in this program type, since they will only execute from user context via bpf_prog_test_run. Signed-off-by: Alexei Starovoitov Signed-off-by: Daniel Borkmann Acked-by: John Fastabend Acked-by: Andrii Nakryiko Link: https://lore.kernel.org/bpf/20210514003623.28033-2-alexei.starovoitov@gmail.com --- include/linux/bpf.h | 10 ++++++++ include/linux/bpf_types.h | 2 ++ include/uapi/linux/bpf.h | 8 +++++++ kernel/bpf/syscall.c | 53 ++++++++++++++++++++++++++++++++++++++++++ kernel/bpf/verifier.c | 8 +++++++ net/bpf/test_run.c | 43 ++++++++++++++++++++++++++++++++++ tools/include/uapi/linux/bpf.h | 8 +++++++ 7 files changed, 132 insertions(+) (limited to 'include/uapi') diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 02b02cb29ce2..04a2bf41ae72 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -1826,6 +1826,9 @@ static inline bool bpf_map_is_dev_bound(struct bpf_map *map) struct bpf_map *bpf_map_offload_map_alloc(union bpf_attr *attr); void bpf_map_offload_map_free(struct bpf_map *map); +int bpf_prog_test_run_syscall(struct bpf_prog *prog, + const union bpf_attr *kattr, + union bpf_attr __user *uattr); #else static inline int bpf_prog_offload_init(struct bpf_prog *prog, union bpf_attr *attr) @@ -1851,6 +1854,13 @@ static inline struct bpf_map *bpf_map_offload_map_alloc(union bpf_attr *attr) static inline void bpf_map_offload_map_free(struct bpf_map *map) { } + +static inline int bpf_prog_test_run_syscall(struct bpf_prog *prog, + const union bpf_attr *kattr, + union bpf_attr __user *uattr) +{ + return -ENOTSUPP; +} #endif /* CONFIG_NET && CONFIG_BPF_SYSCALL */ #if defined(CONFIG_INET) && defined(CONFIG_BPF_SYSCALL) diff --git a/include/linux/bpf_types.h b/include/linux/bpf_types.h index f883f01a5061..a9db1eae6796 100644 --- a/include/linux/bpf_types.h +++ b/include/linux/bpf_types.h @@ -77,6 +77,8 @@ BPF_PROG_TYPE(BPF_PROG_TYPE_LSM, lsm, void *, void *) #endif /* CONFIG_BPF_LSM */ #endif +BPF_PROG_TYPE(BPF_PROG_TYPE_SYSCALL, bpf_syscall, + void *, void *) BPF_MAP_TYPE(BPF_MAP_TYPE_ARRAY, array_map_ops) BPF_MAP_TYPE(BPF_MAP_TYPE_PERCPU_ARRAY, percpu_array_map_ops) diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h index ec6d85a81744..c92648f38144 100644 --- a/include/uapi/linux/bpf.h +++ b/include/uapi/linux/bpf.h @@ -937,6 +937,7 @@ enum bpf_prog_type { BPF_PROG_TYPE_EXT, BPF_PROG_TYPE_LSM, BPF_PROG_TYPE_SK_LOOKUP, + BPF_PROG_TYPE_SYSCALL, /* a program that can execute syscalls */ }; enum bpf_attach_type { @@ -4735,6 +4736,12 @@ union bpf_attr { * be zero-terminated except when **str_size** is 0. * * Or **-EBUSY** if the per-CPU memory copy buffer is busy. + * + * long bpf_sys_bpf(u32 cmd, void *attr, u32 attr_size) + * Description + * Execute bpf syscall with given arguments. + * Return + * A syscall result. */ #define __BPF_FUNC_MAPPER(FN) \ FN(unspec), \ @@ -4903,6 +4910,7 @@ union bpf_attr { FN(check_mtu), \ FN(for_each_map_elem), \ FN(snprintf), \ + FN(sys_bpf), \ /* */ /* integer value in 'imm' field of BPF_CALL instruction selects which helper diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index 941ca06d9dfa..b1e7352919cb 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -2014,6 +2014,7 @@ bpf_prog_load_check_attach(enum bpf_prog_type prog_type, if (expected_attach_type == BPF_SK_LOOKUP) return 0; return -EINVAL; + case BPF_PROG_TYPE_SYSCALL: case BPF_PROG_TYPE_EXT: if (expected_attach_type) return -EINVAL; @@ -4508,3 +4509,55 @@ SYSCALL_DEFINE3(bpf, int, cmd, union bpf_attr __user *, uattr, unsigned int, siz return err; } + +static bool syscall_prog_is_valid_access(int off, int size, + enum bpf_access_type type, + const struct bpf_prog *prog, + struct bpf_insn_access_aux *info) +{ + if (off < 0 || off >= U16_MAX) + return false; + if (off % size != 0) + return false; + return true; +} + +BPF_CALL_3(bpf_sys_bpf, int, cmd, void *, attr, u32, attr_size) +{ + return -EINVAL; +} + +const struct bpf_func_proto bpf_sys_bpf_proto = { + .func = bpf_sys_bpf, + .gpl_only = false, + .ret_type = RET_INTEGER, + .arg1_type = ARG_ANYTHING, + .arg2_type = ARG_PTR_TO_MEM, + .arg3_type = ARG_CONST_SIZE, +}; + +const struct bpf_func_proto * __weak +tracing_prog_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) +{ + return bpf_base_func_proto(func_id); +} + +static const struct bpf_func_proto * +syscall_prog_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) +{ + switch (func_id) { + case BPF_FUNC_sys_bpf: + return &bpf_sys_bpf_proto; + default: + return tracing_prog_func_proto(func_id, prog); + } +} + +const struct bpf_verifier_ops bpf_syscall_verifier_ops = { + .get_func_proto = syscall_prog_func_proto, + .is_valid_access = syscall_prog_is_valid_access, +}; + +const struct bpf_prog_ops bpf_syscall_prog_ops = { + .test_run = bpf_prog_test_run_syscall, +}; diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index bdfdb54676ea..37407d8fbca4 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -13196,6 +13196,14 @@ static int check_attach_btf_id(struct bpf_verifier_env *env) int ret; u64 key; + if (prog->type == BPF_PROG_TYPE_SYSCALL) { + if (prog->aux->sleepable) + /* attach_btf_id checked to be zero already */ + return 0; + verbose(env, "Syscall programs can only be sleepable\n"); + return -EINVAL; + } + if (prog->aux->sleepable && prog->type != BPF_PROG_TYPE_TRACING && prog->type != BPF_PROG_TYPE_LSM) { verbose(env, "Only fentry/fexit/fmod_ret and lsm programs can be sleepable\n"); diff --git a/net/bpf/test_run.c b/net/bpf/test_run.c index a5d72c48fb66..a6972d7ddf80 100644 --- a/net/bpf/test_run.c +++ b/net/bpf/test_run.c @@ -918,3 +918,46 @@ out: kfree(user_ctx); return ret; } + +int bpf_prog_test_run_syscall(struct bpf_prog *prog, + const union bpf_attr *kattr, + union bpf_attr __user *uattr) +{ + void __user *ctx_in = u64_to_user_ptr(kattr->test.ctx_in); + __u32 ctx_size_in = kattr->test.ctx_size_in; + void *ctx = NULL; + u32 retval; + int err = 0; + + /* doesn't support data_in/out, ctx_out, duration, or repeat or flags */ + if (kattr->test.data_in || kattr->test.data_out || + kattr->test.ctx_out || kattr->test.duration || + kattr->test.repeat || kattr->test.flags) + return -EINVAL; + + if (ctx_size_in < prog->aux->max_ctx_offset || + ctx_size_in > U16_MAX) + return -EINVAL; + + if (ctx_size_in) { + ctx = kzalloc(ctx_size_in, GFP_USER); + if (!ctx) + return -ENOMEM; + if (copy_from_user(ctx, ctx_in, ctx_size_in)) { + err = -EFAULT; + goto out; + } + } + retval = bpf_prog_run_pin_on_cpu(prog, ctx); + + if (copy_to_user(&uattr->test.retval, &retval, sizeof(u32))) { + err = -EFAULT; + goto out; + } + if (ctx_size_in) + if (copy_to_user(ctx_in, ctx, ctx_size_in)) + err = -EFAULT; +out: + kfree(ctx); + return err; +} diff --git a/tools/include/uapi/linux/bpf.h b/tools/include/uapi/linux/bpf.h index ec6d85a81744..c92648f38144 100644 --- a/tools/include/uapi/linux/bpf.h +++ b/tools/include/uapi/linux/bpf.h @@ -937,6 +937,7 @@ enum bpf_prog_type { BPF_PROG_TYPE_EXT, BPF_PROG_TYPE_LSM, BPF_PROG_TYPE_SK_LOOKUP, + BPF_PROG_TYPE_SYSCALL, /* a program that can execute syscalls */ }; enum bpf_attach_type { @@ -4735,6 +4736,12 @@ union bpf_attr { * be zero-terminated except when **str_size** is 0. * * Or **-EBUSY** if the per-CPU memory copy buffer is busy. + * + * long bpf_sys_bpf(u32 cmd, void *attr, u32 attr_size) + * Description + * Execute bpf syscall with given arguments. + * Return + * A syscall result. */ #define __BPF_FUNC_MAPPER(FN) \ FN(unspec), \ @@ -4903,6 +4910,7 @@ union bpf_attr { FN(check_mtu), \ FN(for_each_map_elem), \ FN(snprintf), \ + FN(sys_bpf), \ /* */ /* integer value in 'imm' field of BPF_CALL instruction selects which helper -- cgit v1.2.3 From 387544bfa291a22383d60b40f887360e2b931ec6 Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Thu, 13 May 2021 17:36:10 -0700 Subject: bpf: Introduce fd_idx Typical program loading sequence involves creating bpf maps and applying map FDs into bpf instructions in various places in the bpf program. This job is done by libbpf that is using compiler generated ELF relocations to patch certain instruction after maps are created and BTFs are loaded. The goal of fd_idx is to allow bpf instructions to stay immutable after compilation. At load time the libbpf would still create maps as usual, but it wouldn't need to patch instructions. It would store map_fds into __u32 fd_array[] and would pass that pointer to sys_bpf(BPF_PROG_LOAD). Signed-off-by: Alexei Starovoitov Signed-off-by: Daniel Borkmann Acked-by: Andrii Nakryiko Link: https://lore.kernel.org/bpf/20210514003623.28033-9-alexei.starovoitov@gmail.com --- include/linux/bpf_verifier.h | 1 + include/uapi/linux/bpf.h | 16 +++++++++----- kernel/bpf/syscall.c | 2 +- kernel/bpf/verifier.c | 47 +++++++++++++++++++++++++++++++++--------- tools/include/uapi/linux/bpf.h | 16 +++++++++----- 5 files changed, 61 insertions(+), 21 deletions(-) (limited to 'include/uapi') diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index d4632aa3ca50..e774ecc1cd1f 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -450,6 +450,7 @@ struct bpf_verifier_env { u32 peak_states; /* longest register parentage chain walked for liveness marking */ u32 longest_mark_read_walk; + bpfptr_t fd_array; }; __printf(2, 0) void bpf_verifier_vlog(struct bpf_verifier_log *log, diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h index c92648f38144..de58a714ed36 100644 --- a/include/uapi/linux/bpf.h +++ b/include/uapi/linux/bpf.h @@ -1098,8 +1098,8 @@ enum bpf_link_type { /* When BPF ldimm64's insn[0].src_reg != 0 then this can have * the following extensions: * - * insn[0].src_reg: BPF_PSEUDO_MAP_FD - * insn[0].imm: map fd + * insn[0].src_reg: BPF_PSEUDO_MAP_[FD|IDX] + * insn[0].imm: map fd or fd_idx * insn[1].imm: 0 * insn[0].off: 0 * insn[1].off: 0 @@ -1107,15 +1107,19 @@ enum bpf_link_type { * verifier type: CONST_PTR_TO_MAP */ #define BPF_PSEUDO_MAP_FD 1 -/* insn[0].src_reg: BPF_PSEUDO_MAP_VALUE - * insn[0].imm: map fd +#define BPF_PSEUDO_MAP_IDX 5 + +/* insn[0].src_reg: BPF_PSEUDO_MAP_[IDX_]VALUE + * insn[0].imm: map fd or fd_idx * insn[1].imm: offset into value * insn[0].off: 0 * insn[1].off: 0 * ldimm64 rewrite: address of map[0]+offset * verifier type: PTR_TO_MAP_VALUE */ -#define BPF_PSEUDO_MAP_VALUE 2 +#define BPF_PSEUDO_MAP_VALUE 2 +#define BPF_PSEUDO_MAP_IDX_VALUE 6 + /* insn[0].src_reg: BPF_PSEUDO_BTF_ID * insn[0].imm: kernel btd id of VAR * insn[1].imm: 0 @@ -1315,6 +1319,8 @@ union bpf_attr { /* or valid module BTF object fd or 0 to attach to vmlinux */ __u32 attach_btf_obj_fd; }; + __u32 :32; /* pad */ + __aligned_u64 fd_array; /* array of FDs */ }; struct { /* anonymous struct used by BPF_OBJ_* commands */ diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index 415865c49dd4..da7dc2406470 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -2089,7 +2089,7 @@ static bool is_perfmon_prog_type(enum bpf_prog_type prog_type) } /* last field in 'union bpf_attr' used by this command */ -#define BPF_PROG_LOAD_LAST_FIELD attach_prog_fd +#define BPF_PROG_LOAD_LAST_FIELD fd_array static int bpf_prog_load(union bpf_attr *attr, bpfptr_t uattr) { diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index e63c7d60e00d..9189eecb26dd 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -8915,12 +8915,14 @@ static int check_ld_imm(struct bpf_verifier_env *env, struct bpf_insn *insn) mark_reg_known_zero(env, regs, insn->dst_reg); dst_reg->map_ptr = map; - if (insn->src_reg == BPF_PSEUDO_MAP_VALUE) { + if (insn->src_reg == BPF_PSEUDO_MAP_VALUE || + insn->src_reg == BPF_PSEUDO_MAP_IDX_VALUE) { dst_reg->type = PTR_TO_MAP_VALUE; dst_reg->off = aux->map_off; if (map_value_has_spin_lock(map)) dst_reg->id = ++env->id_gen; - } else if (insn->src_reg == BPF_PSEUDO_MAP_FD) { + } else if (insn->src_reg == BPF_PSEUDO_MAP_FD || + insn->src_reg == BPF_PSEUDO_MAP_IDX) { dst_reg->type = CONST_PTR_TO_MAP; } else { verbose(env, "bpf verifier is misconfigured\n"); @@ -11173,6 +11175,7 @@ static int resolve_pseudo_ldimm64(struct bpf_verifier_env *env) struct bpf_map *map; struct fd f; u64 addr; + u32 fd; if (i == insn_cnt - 1 || insn[1].code != 0 || insn[1].dst_reg != 0 || insn[1].src_reg != 0 || @@ -11202,16 +11205,38 @@ static int resolve_pseudo_ldimm64(struct bpf_verifier_env *env) /* In final convert_pseudo_ld_imm64() step, this is * converted into regular 64-bit imm load insn. */ - if ((insn[0].src_reg != BPF_PSEUDO_MAP_FD && - insn[0].src_reg != BPF_PSEUDO_MAP_VALUE) || - (insn[0].src_reg == BPF_PSEUDO_MAP_FD && - insn[1].imm != 0)) { - verbose(env, - "unrecognized bpf_ld_imm64 insn\n"); + switch (insn[0].src_reg) { + case BPF_PSEUDO_MAP_VALUE: + case BPF_PSEUDO_MAP_IDX_VALUE: + break; + case BPF_PSEUDO_MAP_FD: + case BPF_PSEUDO_MAP_IDX: + if (insn[1].imm == 0) + break; + fallthrough; + default: + verbose(env, "unrecognized bpf_ld_imm64 insn\n"); return -EINVAL; } - f = fdget(insn[0].imm); + switch (insn[0].src_reg) { + case BPF_PSEUDO_MAP_IDX_VALUE: + case BPF_PSEUDO_MAP_IDX: + if (bpfptr_is_null(env->fd_array)) { + verbose(env, "fd_idx without fd_array is invalid\n"); + return -EPROTO; + } + if (copy_from_bpfptr_offset(&fd, env->fd_array, + insn[0].imm * sizeof(fd), + sizeof(fd))) + return -EFAULT; + break; + default: + fd = insn[0].imm; + break; + } + + f = fdget(fd); map = __bpf_map_get(f); if (IS_ERR(map)) { verbose(env, "fd %d is not pointing to valid bpf_map\n", @@ -11226,7 +11251,8 @@ static int resolve_pseudo_ldimm64(struct bpf_verifier_env *env) } aux = &env->insn_aux_data[i]; - if (insn->src_reg == BPF_PSEUDO_MAP_FD) { + if (insn[0].src_reg == BPF_PSEUDO_MAP_FD || + insn[0].src_reg == BPF_PSEUDO_MAP_IDX) { addr = (unsigned long)map; } else { u32 off = insn[1].imm; @@ -13308,6 +13334,7 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr) env->insn_aux_data[i].orig_idx = i; env->prog = *prog; env->ops = bpf_verifier_ops[env->prog->type]; + env->fd_array = make_bpfptr(attr->fd_array, uattr.is_kernel); is_priv = bpf_capable(); bpf_get_btf_vmlinux(); diff --git a/tools/include/uapi/linux/bpf.h b/tools/include/uapi/linux/bpf.h index c92648f38144..de58a714ed36 100644 --- a/tools/include/uapi/linux/bpf.h +++ b/tools/include/uapi/linux/bpf.h @@ -1098,8 +1098,8 @@ enum bpf_link_type { /* When BPF ldimm64's insn[0].src_reg != 0 then this can have * the following extensions: * - * insn[0].src_reg: BPF_PSEUDO_MAP_FD - * insn[0].imm: map fd + * insn[0].src_reg: BPF_PSEUDO_MAP_[FD|IDX] + * insn[0].imm: map fd or fd_idx * insn[1].imm: 0 * insn[0].off: 0 * insn[1].off: 0 @@ -1107,15 +1107,19 @@ enum bpf_link_type { * verifier type: CONST_PTR_TO_MAP */ #define BPF_PSEUDO_MAP_FD 1 -/* insn[0].src_reg: BPF_PSEUDO_MAP_VALUE - * insn[0].imm: map fd +#define BPF_PSEUDO_MAP_IDX 5 + +/* insn[0].src_reg: BPF_PSEUDO_MAP_[IDX_]VALUE + * insn[0].imm: map fd or fd_idx * insn[1].imm: offset into value * insn[0].off: 0 * insn[1].off: 0 * ldimm64 rewrite: address of map[0]+offset * verifier type: PTR_TO_MAP_VALUE */ -#define BPF_PSEUDO_MAP_VALUE 2 +#define BPF_PSEUDO_MAP_VALUE 2 +#define BPF_PSEUDO_MAP_IDX_VALUE 6 + /* insn[0].src_reg: BPF_PSEUDO_BTF_ID * insn[0].imm: kernel btd id of VAR * insn[1].imm: 0 @@ -1315,6 +1319,8 @@ union bpf_attr { /* or valid module BTF object fd or 0 to attach to vmlinux */ __u32 attach_btf_obj_fd; }; + __u32 :32; /* pad */ + __aligned_u64 fd_array; /* array of FDs */ }; struct { /* anonymous struct used by BPF_OBJ_* commands */ -- cgit v1.2.3 From 3d78417b60fba249cc555468cb72d96f5cde2964 Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Thu, 13 May 2021 17:36:11 -0700 Subject: bpf: Add bpf_btf_find_by_name_kind() helper. Add new helper: long bpf_btf_find_by_name_kind(char *name, int name_sz, u32 kind, int flags) Description Find BTF type with given name and kind in vmlinux BTF or in module's BTFs. Return Returns btf_id and btf_obj_fd in lower and upper 32 bits. It will be used by loader program to find btf_id to attach the program to and to find btf_ids of ksyms. Signed-off-by: Alexei Starovoitov Signed-off-by: Daniel Borkmann Acked-by: Andrii Nakryiko Link: https://lore.kernel.org/bpf/20210514003623.28033-10-alexei.starovoitov@gmail.com --- include/linux/bpf.h | 1 + include/uapi/linux/bpf.h | 7 +++++ kernel/bpf/btf.c | 62 ++++++++++++++++++++++++++++++++++++++++++ kernel/bpf/syscall.c | 2 ++ tools/include/uapi/linux/bpf.h | 7 +++++ 5 files changed, 79 insertions(+) (limited to 'include/uapi') diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 7fd53380c981..9dc44ba97584 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -1974,6 +1974,7 @@ extern const struct bpf_func_proto bpf_get_socket_ptr_cookie_proto; extern const struct bpf_func_proto bpf_task_storage_get_proto; extern const struct bpf_func_proto bpf_task_storage_delete_proto; extern const struct bpf_func_proto bpf_for_each_map_elem_proto; +extern const struct bpf_func_proto bpf_btf_find_by_name_kind_proto; const struct bpf_func_proto *bpf_tracing_func_proto( enum bpf_func_id func_id, const struct bpf_prog *prog); diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h index de58a714ed36..3cc07351c1cf 100644 --- a/include/uapi/linux/bpf.h +++ b/include/uapi/linux/bpf.h @@ -4748,6 +4748,12 @@ union bpf_attr { * Execute bpf syscall with given arguments. * Return * A syscall result. + * + * long bpf_btf_find_by_name_kind(char *name, int name_sz, u32 kind, int flags) + * Description + * Find BTF type with given name and kind in vmlinux BTF or in module's BTFs. + * Return + * Returns btf_id and btf_obj_fd in lower and upper 32 bits. */ #define __BPF_FUNC_MAPPER(FN) \ FN(unspec), \ @@ -4917,6 +4923,7 @@ union bpf_attr { FN(for_each_map_elem), \ FN(snprintf), \ FN(sys_bpf), \ + FN(btf_find_by_name_kind), \ /* */ /* integer value in 'imm' field of BPF_CALL instruction selects which helper diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index fbf6c06a9d62..85716327c375 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -6085,3 +6085,65 @@ struct module *btf_try_get_module(const struct btf *btf) return res; } + +BPF_CALL_4(bpf_btf_find_by_name_kind, char *, name, int, name_sz, u32, kind, int, flags) +{ + struct btf *btf; + long ret; + + if (flags) + return -EINVAL; + + if (name_sz <= 1 || name[name_sz - 1]) + return -EINVAL; + + btf = bpf_get_btf_vmlinux(); + if (IS_ERR(btf)) + return PTR_ERR(btf); + + ret = btf_find_by_name_kind(btf, name, kind); + /* ret is never zero, since btf_find_by_name_kind returns + * positive btf_id or negative error. + */ + if (ret < 0) { + struct btf *mod_btf; + int id; + + /* If name is not found in vmlinux's BTF then search in module's BTFs */ + spin_lock_bh(&btf_idr_lock); + idr_for_each_entry(&btf_idr, mod_btf, id) { + if (!btf_is_module(mod_btf)) + continue; + /* linear search could be slow hence unlock/lock + * the IDR to avoiding holding it for too long + */ + btf_get(mod_btf); + spin_unlock_bh(&btf_idr_lock); + ret = btf_find_by_name_kind(mod_btf, name, kind); + if (ret > 0) { + int btf_obj_fd; + + btf_obj_fd = __btf_new_fd(mod_btf); + if (btf_obj_fd < 0) { + btf_put(mod_btf); + return btf_obj_fd; + } + return ret | (((u64)btf_obj_fd) << 32); + } + spin_lock_bh(&btf_idr_lock); + btf_put(mod_btf); + } + spin_unlock_bh(&btf_idr_lock); + } + return ret; +} + +const struct bpf_func_proto bpf_btf_find_by_name_kind_proto = { + .func = bpf_btf_find_by_name_kind, + .gpl_only = false, + .ret_type = RET_INTEGER, + .arg1_type = ARG_PTR_TO_MEM, + .arg2_type = ARG_CONST_SIZE, + .arg3_type = ARG_ANYTHING, + .arg4_type = ARG_ANYTHING, +}; diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index da7dc2406470..f93ff2ebf96d 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -4584,6 +4584,8 @@ syscall_prog_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) switch (func_id) { case BPF_FUNC_sys_bpf: return &bpf_sys_bpf_proto; + case BPF_FUNC_btf_find_by_name_kind: + return &bpf_btf_find_by_name_kind_proto; default: return tracing_prog_func_proto(func_id, prog); } diff --git a/tools/include/uapi/linux/bpf.h b/tools/include/uapi/linux/bpf.h index de58a714ed36..3cc07351c1cf 100644 --- a/tools/include/uapi/linux/bpf.h +++ b/tools/include/uapi/linux/bpf.h @@ -4748,6 +4748,12 @@ union bpf_attr { * Execute bpf syscall with given arguments. * Return * A syscall result. + * + * long bpf_btf_find_by_name_kind(char *name, int name_sz, u32 kind, int flags) + * Description + * Find BTF type with given name and kind in vmlinux BTF or in module's BTFs. + * Return + * Returns btf_id and btf_obj_fd in lower and upper 32 bits. */ #define __BPF_FUNC_MAPPER(FN) \ FN(unspec), \ @@ -4917,6 +4923,7 @@ union bpf_attr { FN(for_each_map_elem), \ FN(snprintf), \ FN(sys_bpf), \ + FN(btf_find_by_name_kind), \ /* */ /* integer value in 'imm' field of BPF_CALL instruction selects which helper -- cgit v1.2.3 From 3abea089246f76c1517b054ddb5946f3f1dbd2c0 Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Thu, 13 May 2021 17:36:12 -0700 Subject: bpf: Add bpf_sys_close() helper. Add bpf_sys_close() helper to be used by the syscall/loader program to close intermediate FDs and other cleanup. Note this helper must never be allowed inside fdget/fdput bracketing. Signed-off-by: Alexei Starovoitov Signed-off-by: Daniel Borkmann Acked-by: Andrii Nakryiko Link: https://lore.kernel.org/bpf/20210514003623.28033-11-alexei.starovoitov@gmail.com --- include/uapi/linux/bpf.h | 7 +++++++ kernel/bpf/syscall.c | 19 +++++++++++++++++++ tools/include/uapi/linux/bpf.h | 7 +++++++ 3 files changed, 33 insertions(+) (limited to 'include/uapi') diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h index 3cc07351c1cf..4cd9a0181f27 100644 --- a/include/uapi/linux/bpf.h +++ b/include/uapi/linux/bpf.h @@ -4754,6 +4754,12 @@ union bpf_attr { * Find BTF type with given name and kind in vmlinux BTF or in module's BTFs. * Return * Returns btf_id and btf_obj_fd in lower and upper 32 bits. + * + * long bpf_sys_close(u32 fd) + * Description + * Execute close syscall for given FD. + * Return + * A syscall result. */ #define __BPF_FUNC_MAPPER(FN) \ FN(unspec), \ @@ -4924,6 +4930,7 @@ union bpf_attr { FN(snprintf), \ FN(sys_bpf), \ FN(btf_find_by_name_kind), \ + FN(sys_close), \ /* */ /* integer value in 'imm' field of BPF_CALL instruction selects which helper diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index f93ff2ebf96d..0f1ce2171f1e 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -4578,6 +4578,23 @@ tracing_prog_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) return bpf_base_func_proto(func_id); } +BPF_CALL_1(bpf_sys_close, u32, fd) +{ + /* When bpf program calls this helper there should not be + * an fdget() without matching completed fdput(). + * This helper is allowed in the following callchain only: + * sys_bpf->prog_test_run->bpf_prog->bpf_sys_close + */ + return close_fd(fd); +} + +const struct bpf_func_proto bpf_sys_close_proto = { + .func = bpf_sys_close, + .gpl_only = false, + .ret_type = RET_INTEGER, + .arg1_type = ARG_ANYTHING, +}; + static const struct bpf_func_proto * syscall_prog_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) { @@ -4586,6 +4603,8 @@ syscall_prog_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) return &bpf_sys_bpf_proto; case BPF_FUNC_btf_find_by_name_kind: return &bpf_btf_find_by_name_kind_proto; + case BPF_FUNC_sys_close: + return &bpf_sys_close_proto; default: return tracing_prog_func_proto(func_id, prog); } diff --git a/tools/include/uapi/linux/bpf.h b/tools/include/uapi/linux/bpf.h index 3cc07351c1cf..4cd9a0181f27 100644 --- a/tools/include/uapi/linux/bpf.h +++ b/tools/include/uapi/linux/bpf.h @@ -4754,6 +4754,12 @@ union bpf_attr { * Find BTF type with given name and kind in vmlinux BTF or in module's BTFs. * Return * Returns btf_id and btf_obj_fd in lower and upper 32 bits. + * + * long bpf_sys_close(u32 fd) + * Description + * Execute close syscall for given FD. + * Return + * A syscall result. */ #define __BPF_FUNC_MAPPER(FN) \ FN(unspec), \ @@ -4924,6 +4930,7 @@ union bpf_attr { FN(snprintf), \ FN(sys_bpf), \ FN(btf_find_by_name_kind), \ + FN(sys_close), \ /* */ /* integer value in 'imm' field of BPF_CALL instruction selects which helper -- cgit v1.2.3 From 7cd60e43a6def40ecb75deb8decc677995970d0b Mon Sep 17 00:00:00 2001 From: "Chang S. Bae" Date: Tue, 18 May 2021 13:03:15 -0700 Subject: uapi/auxvec: Define the aux vector AT_MINSIGSTKSZ Define AT_MINSIGSTKSZ in the generic uapi header. It is already used as generic ABI in glibc's generic elf.h, and this define will prevent future namespace conflicts. In particular, x86 is also using this generic definition. Signed-off-by: Chang S. Bae Signed-off-by: Borislav Petkov Reviewed-by: Len Brown Acked-by: Thomas Gleixner Link: https://lkml.kernel.org/r/20210518200320.17239-2-chang.seok.bae@intel.com --- include/uapi/linux/auxvec.h | 3 +++ 1 file changed, 3 insertions(+) (limited to 'include/uapi') diff --git a/include/uapi/linux/auxvec.h b/include/uapi/linux/auxvec.h index abe5f2b6581b..c7e502bf5a6f 100644 --- a/include/uapi/linux/auxvec.h +++ b/include/uapi/linux/auxvec.h @@ -33,5 +33,8 @@ #define AT_EXECFN 31 /* filename of program */ +#ifndef AT_MINSIGSTKSZ +#define AT_MINSIGSTKSZ 51 /* minimal stack size for signal delivery */ +#endif #endif /* _UAPI_LINUX_AUXVEC_H */ -- cgit v1.2.3 From 5d67f349590ddc94b6d4e25f19085728db9de697 Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Tue, 18 May 2021 18:40:32 -0700 Subject: bpf: Add cmd alias BPF_PROG_RUN Add BPF_PROG_RUN command as an alias to BPF_RPOG_TEST_RUN to better indicate the full range of use cases done by the command. Suggested-by: Daniel Borkmann Signed-off-by: Alexei Starovoitov Signed-off-by: Daniel Borkmann Acked-by: Song Liu Link: https://lore.kernel.org/bpf/20210519014032.20908-1-alexei.starovoitov@gmail.com --- include/uapi/linux/bpf.h | 1 + tools/include/uapi/linux/bpf.h | 1 + tools/lib/bpf/skel_internal.h | 2 +- 3 files changed, 3 insertions(+), 1 deletion(-) (limited to 'include/uapi') diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h index 4cd9a0181f27..418b9b813d65 100644 --- a/include/uapi/linux/bpf.h +++ b/include/uapi/linux/bpf.h @@ -837,6 +837,7 @@ enum bpf_cmd { BPF_PROG_ATTACH, BPF_PROG_DETACH, BPF_PROG_TEST_RUN, + BPF_PROG_RUN = BPF_PROG_TEST_RUN, BPF_PROG_GET_NEXT_ID, BPF_MAP_GET_NEXT_ID, BPF_PROG_GET_FD_BY_ID, diff --git a/tools/include/uapi/linux/bpf.h b/tools/include/uapi/linux/bpf.h index 4cd9a0181f27..418b9b813d65 100644 --- a/tools/include/uapi/linux/bpf.h +++ b/tools/include/uapi/linux/bpf.h @@ -837,6 +837,7 @@ enum bpf_cmd { BPF_PROG_ATTACH, BPF_PROG_DETACH, BPF_PROG_TEST_RUN, + BPF_PROG_RUN = BPF_PROG_TEST_RUN, BPF_PROG_GET_NEXT_ID, BPF_MAP_GET_NEXT_ID, BPF_PROG_GET_FD_BY_ID, diff --git a/tools/lib/bpf/skel_internal.h b/tools/lib/bpf/skel_internal.h index 12a126b452c1..b22b50c1b173 100644 --- a/tools/lib/bpf/skel_internal.h +++ b/tools/lib/bpf/skel_internal.h @@ -102,7 +102,7 @@ static inline int bpf_load_and_run(struct bpf_load_and_run_opts *opts) attr.test.prog_fd = prog_fd; attr.test.ctx_in = (long) opts->ctx; attr.test.ctx_size_in = opts->ctx->sz; - err = skel_sys_bpf(BPF_PROG_TEST_RUN, &attr, sizeof(attr)); + err = skel_sys_bpf(BPF_PROG_RUN, &attr, sizeof(attr)); if (err < 0 || (int)attr.test.retval < 0) { opts->errstr = "failed to execute loader prog"; if (err < 0) -- cgit v1.2.3 From c906b86e9c44946fae505a5996b1f4c24a699fe6 Mon Sep 17 00:00:00 2001 From: Sergey Gorenko Date: Mon, 10 May 2021 13:23:33 +0300 Subject: RDMA/mlx5: Add SQD2RTS bit to the alloc ucontext response The new bit in the comp_mask is needed to mark that kernel supports SQD2RTS transition for the modify QP command. Link: https://lore.kernel.org/r/7ce705fedac1b2b8e3a2f4013e04244dc5946344.1620641808.git.leonro@nvidia.com Reviewed-by: Evgenii Kochetov Signed-off-by: Sergey Gorenko Signed-off-by: Leon Romanovsky Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/mlx5/main.c | 4 ++++ include/uapi/rdma/mlx5-abi.h | 1 + 2 files changed, 5 insertions(+) (limited to 'include/uapi') diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c index 6d1dd09a4388..312aa731860d 100644 --- a/drivers/infiniband/hw/mlx5/main.c +++ b/drivers/infiniband/hw/mlx5/main.c @@ -1817,6 +1817,10 @@ static int set_ucontext_resp(struct ib_ucontext *uctx, resp->comp_mask |= MLX5_IB_ALLOC_UCONTEXT_RESP_MASK_ECE; resp->num_dyn_bfregs = bfregi->num_dyn_bfregs; + + if (MLX5_CAP_GEN(dev->mdev, drain_sigerr)) + resp->comp_mask |= MLX5_IB_ALLOC_UCONTEXT_RESP_MASK_SQD2RTS; + return 0; } diff --git a/include/uapi/rdma/mlx5-abi.h b/include/uapi/rdma/mlx5-abi.h index 27905a0268c9..995faf8f44bd 100644 --- a/include/uapi/rdma/mlx5-abi.h +++ b/include/uapi/rdma/mlx5-abi.h @@ -101,6 +101,7 @@ enum mlx5_ib_alloc_ucontext_resp_mask { MLX5_IB_ALLOC_UCONTEXT_RESP_MASK_CORE_CLOCK_OFFSET = 1UL << 0, MLX5_IB_ALLOC_UCONTEXT_RESP_MASK_DUMP_FILL_MKEY = 1UL << 1, MLX5_IB_ALLOC_UCONTEXT_RESP_MASK_ECE = 1UL << 2, + MLX5_IB_ALLOC_UCONTEXT_RESP_MASK_SQD2RTS = 1UL << 3, }; enum mlx5_user_cmds_supp_uhw { -- cgit v1.2.3 From 12ccb76280f8c0c07794fa68f83286b934981ca5 Mon Sep 17 00:00:00 2001 From: Sean Young Date: Tue, 13 Apr 2021 11:40:17 +0200 Subject: media: lirc: remove out of date comment This file has been updated many times since 2010. Signed-off-by: Sean Young Signed-off-by: Mauro Carvalho Chehab --- include/uapi/linux/lirc.h | 1 - 1 file changed, 1 deletion(-) (limited to 'include/uapi') diff --git a/include/uapi/linux/lirc.h b/include/uapi/linux/lirc.h index c45a4eaea667..9919f2062b14 100644 --- a/include/uapi/linux/lirc.h +++ b/include/uapi/linux/lirc.h @@ -1,7 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * lirc.h - linux infrared remote control header file - * last modified 2010/07/13 by Jarod Wilson */ #ifndef _LINUX_LIRC_H -- cgit v1.2.3 From 2f0968827a48a3b01a0cc9185abd41978d5ce918 Mon Sep 17 00:00:00 2001 From: Ezequiel Garcia Date: Thu, 29 Apr 2021 16:48:16 +0200 Subject: media: uapi: Move the MPEG-2 stateless control type out of staging Move the MPEG-2 stateless control types out of staging, and re-number it to avoid any confusion. Signed-off-by: Ezequiel Garcia Tested-by: Jernej Skrabec Reviewed-by: Jernej Skrabec Tested-by: Daniel Almeida Signed-off-by: Hans Verkuil Signed-off-by: Mauro Carvalho Chehab --- include/media/mpeg2-ctrls.h | 4 ---- include/uapi/linux/videodev2.h | 4 ++++ 2 files changed, 4 insertions(+), 4 deletions(-) (limited to 'include/uapi') diff --git a/include/media/mpeg2-ctrls.h b/include/media/mpeg2-ctrls.h index a84ce088a42e..a3d19de9e53a 100644 --- a/include/media/mpeg2-ctrls.h +++ b/include/media/mpeg2-ctrls.h @@ -16,10 +16,6 @@ #define V4L2_CID_MPEG_VIDEO_MPEG2_PICTURE (V4L2_CID_CODEC_BASE+253) /* enum v4l2_ctrl_type type values */ -#define V4L2_CTRL_TYPE_MPEG2_QUANTISATION 0x0131 -#define V4L2_CTRL_TYPE_MPEG2_SEQUENCE 0x0132 -#define V4L2_CTRL_TYPE_MPEG2_PICTURE 0x0133 - #define V4L2_MPEG2_SEQ_FLAG_PROGRESSIVE 0x01 /** diff --git a/include/uapi/linux/videodev2.h b/include/uapi/linux/videodev2.h index 311a01cc5775..d3bb18a3a51b 100644 --- a/include/uapi/linux/videodev2.h +++ b/include/uapi/linux/videodev2.h @@ -1807,6 +1807,10 @@ enum v4l2_ctrl_type { V4L2_CTRL_TYPE_FWHT_PARAMS = 0x0220, V4L2_CTRL_TYPE_VP8_FRAME = 0x0240, + + V4L2_CTRL_TYPE_MPEG2_QUANTISATION = 0x0250, + V4L2_CTRL_TYPE_MPEG2_SEQUENCE = 0x0251, + V4L2_CTRL_TYPE_MPEG2_PICTURE = 0x0252, }; /* Used in the VIDIOC_QUERYCTRL ioctl for querying controls */ -- cgit v1.2.3 From f4815b399111d992c1118c708f464a847dfd29e2 Mon Sep 17 00:00:00 2001 From: Ezequiel Garcia Date: Thu, 29 Apr 2021 16:48:18 +0200 Subject: media: uapi: move MPEG-2 stateless controls out of staging Until now, the MPEG-2 V4L2 API was not exported as a public API, and only defined in a private media header (media/mpeg2-ctrls.h). After reviewing the MPEG-2 specification in detail, and reworking the controls so they match the MPEG-2 semantics properly, we can consider it ready. Signed-off-by: Ezequiel Garcia Tested-by: Jernej Skrabec Reviewed-by: Jernej Skrabec Tested-by: Daniel Almeida Signed-off-by: Hans Verkuil Signed-off-by: Mauro Carvalho Chehab --- .../media/v4l/ext-ctrls-codec-stateless.rst | 214 ++++++++++++++++++++ .../userspace-api/media/v4l/ext-ctrls-codec.rst | 216 --------------------- .../userspace-api/media/v4l/pixfmt-compressed.rst | 10 +- .../userspace-api/media/v4l/vidioc-g-ext-ctrls.rst | 12 ++ drivers/media/v4l2-core/v4l2-ctrls.c | 12 +- drivers/staging/media/hantro/hantro_drv.c | 6 +- drivers/staging/media/hantro/hantro_g1_mpeg2_dec.c | 6 +- .../staging/media/hantro/rk3399_vpu_hw_mpeg2_dec.c | 6 +- drivers/staging/media/sunxi/cedrus/cedrus.c | 6 +- drivers/staging/media/sunxi/cedrus/cedrus_dec.c | 6 +- include/media/mpeg2-ctrls.h | 126 ------------ include/media/v4l2-ctrls.h | 1 - include/uapi/linux/v4l2-controls.h | 112 +++++++++++ include/uapi/linux/videodev2.h | 3 + 14 files changed, 367 insertions(+), 369 deletions(-) delete mode 100644 include/media/mpeg2-ctrls.h (limited to 'include/uapi') diff --git a/Documentation/userspace-api/media/v4l/ext-ctrls-codec-stateless.rst b/Documentation/userspace-api/media/v4l/ext-ctrls-codec-stateless.rst index 3fc04daa9ffb..2aa508ffb6b9 100644 --- a/Documentation/userspace-api/media/v4l/ext-ctrls-codec-stateless.rst +++ b/Documentation/userspace-api/media/v4l/ext-ctrls-codec-stateless.rst @@ -1244,3 +1244,217 @@ FWHT Flags * - __u8 - ``padding[3]`` - Applications and drivers must set this to zero. + +.. _v4l2-codec-stateless-mpeg2: + +``V4L2_CID_STATELESS_MPEG2_SEQUENCE (struct)`` + Specifies the sequence parameters (as extracted from the bitstream) for the + associated MPEG-2 slice data. This includes fields matching the syntax + elements from the sequence header and sequence extension parts of the + bitstream as specified by :ref:`mpeg2part2`. + +.. c:type:: v4l2_ctrl_mpeg2_sequence + +.. raw:: latex + + \small + +.. cssclass:: longtable + +.. tabularcolumns:: |p{1.4cm}|p{6.5cm}|p{9.4cm}| + +.. flat-table:: struct v4l2_ctrl_mpeg2_sequence + :header-rows: 0 + :stub-columns: 0 + :widths: 1 1 2 + + * - __u16 + - ``horizontal_size`` + - The width of the displayable part of the frame's luminance component. + * - __u16 + - ``vertical_size`` + - The height of the displayable part of the frame's luminance component. + * - __u32 + - ``vbv_buffer_size`` + - Used to calculate the required size of the video buffering verifier, + defined (in bits) as: 16 * 1024 * vbv_buffer_size. + * - __u16 + - ``profile_and_level_indication`` + - The current profile and level indication as extracted from the + bitstream. + * - __u8 + - ``chroma_format`` + - The chrominance sub-sampling format (1: 4:2:0, 2: 4:2:2, 3: 4:4:4). + * - __u8 + - ``flags`` + - See :ref:`MPEG-2 Sequence Flags `. + +.. _mpeg2_sequence_flags: + +``MPEG-2 Sequence Flags`` + +.. cssclass:: longtable + +.. flat-table:: + :header-rows: 0 + :stub-columns: 0 + :widths: 1 1 2 + + * - ``V4L2_MPEG2_SEQ_FLAG_PROGRESSIVE`` + - 0x01 + - Indication that all the frames for the sequence are progressive instead + of interlaced. + +.. raw:: latex + + \normalsize + +``V4L2_CID_STATELESS_MPEG2_PICTURE (struct)`` + Specifies the picture parameters (as extracted from the bitstream) for the + associated MPEG-2 slice data. This includes fields matching the syntax + elements from the picture header and picture coding extension parts of the + bitstream as specified by :ref:`mpeg2part2`. + +.. c:type:: v4l2_ctrl_mpeg2_picture + +.. raw:: latex + + \small + +.. cssclass:: longtable + +.. tabularcolumns:: |p{1.0cm}|p{5.6cm}|p{10.7cm}| + +.. flat-table:: struct v4l2_ctrl_mpeg2_picture + :header-rows: 0 + :stub-columns: 0 + :widths: 1 1 2 + + * - __u64 + - ``backward_ref_ts`` + - Timestamp of the V4L2 capture buffer to use as backward reference, used + with B-coded and P-coded frames. The timestamp refers to the + ``timestamp`` field in struct :c:type:`v4l2_buffer`. Use the + :c:func:`v4l2_timeval_to_ns()` function to convert the struct + :c:type:`timeval` in struct :c:type:`v4l2_buffer` to a __u64. + * - __u64 + - ``forward_ref_ts`` + - Timestamp for the V4L2 capture buffer to use as forward reference, used + with B-coded frames. The timestamp refers to the ``timestamp`` field in + struct :c:type:`v4l2_buffer`. Use the :c:func:`v4l2_timeval_to_ns()` + function to convert the struct :c:type:`timeval` in struct + :c:type:`v4l2_buffer` to a __u64. + * - __u32 + - ``flags`` + - See :ref:`MPEG-2 Picture Flags `. + * - __u8 + - ``f_code[2][2]`` + - Motion vector codes. + * - __u8 + - ``picture_coding_type`` + - Picture coding type for the frame covered by the current slice + (V4L2_MPEG2_PICTURE_CODING_TYPE_I, V4L2_MPEG2_PICTURE_CODING_TYPE_P or + V4L2_MPEG2_PICTURE_CODING_TYPE_B). + * - __u8 + - ``picture_structure`` + - Picture structure (1: interlaced top field, 2: interlaced bottom field, + 3: progressive frame). + * - __u8 + - ``intra_dc_precision`` + - Precision of Discrete Cosine transform (0: 8 bits precision, + 1: 9 bits precision, 2: 10 bits precision, 3: 11 bits precision). + * - __u8 + - ``reserved[5]`` + - Applications and drivers must set this to zero. + +.. _mpeg2_picture_flags: + +``MPEG-2 Picture Flags`` + +.. cssclass:: longtable + +.. flat-table:: + :header-rows: 0 + :stub-columns: 0 + :widths: 1 1 2 + + * - ``V4L2_MPEG2_PIC_FLAG_TOP_FIELD_FIRST`` + - 0x00000001 + - If set and it's an interlaced stream, top field is output first. + * - ``V4L2_MPEG2_PIC_FLAG_FRAME_PRED_DCT`` + - 0x00000002 + - If set only frame-DCT and frame prediction are used. + * - ``V4L2_MPEG2_PIC_FLAG_CONCEALMENT_MV`` + - 0x00000004 + - If set motion vectors are coded for intra macroblocks. + * - ``V4L2_MPEG2_PIC_FLAG_Q_SCALE_TYPE`` + - 0x00000008 + - This flag affects the inverse quantization process. + * - ``V4L2_MPEG2_PIC_FLAG_INTRA_VLC`` + - 0x00000010 + - This flag affects the decoding of transform coefficient data. + * - ``V4L2_MPEG2_PIC_FLAG_ALT_SCAN`` + - 0x00000020 + - This flag affects the decoding of transform coefficient data. + * - ``V4L2_MPEG2_PIC_FLAG_REPEAT_FIRST`` + - 0x00000040 + - This flag affects the decoding process of progressive frames. + * - ``V4L2_MPEG2_PIC_FLAG_PROGRESSIVE`` + - 0x00000080 + - Indicates whether the current frame is progressive. + +.. raw:: latex + + \normalsize + +``V4L2_CID_STATELESS_MPEG2_QUANTISATION (struct)`` + Specifies quantisation matrices, in zigzag scanning order, for the + associated MPEG-2 slice data. This control is initialized by the kernel + to the matrices default values. If a bitstream transmits a user-defined + quantisation matrices load, applications are expected to use this control. + Applications are also expected to set the control loading the default + values, if the quantisation matrices need to be reset, for instance on a + sequence header. This process is specified by section 6.3.7. + "Quant matrix extension" of the specification. + +.. c:type:: v4l2_ctrl_mpeg2_quantisation + +.. tabularcolumns:: |p{0.8cm}|p{8.0cm}|p{8.5cm}| + +.. cssclass:: longtable + +.. raw:: latex + + \small + +.. flat-table:: struct v4l2_ctrl_mpeg2_quantisation + :header-rows: 0 + :stub-columns: 0 + :widths: 1 1 2 + + * - __u8 + - ``intra_quantiser_matrix[64]`` + - The quantisation matrix coefficients for intra-coded frames, in zigzag + scanning order. It is relevant for both luma and chroma components, + although it can be superseded by the chroma-specific matrix for + non-4:2:0 YUV formats. + * - __u8 + - ``non_intra_quantiser_matrix[64]`` + - The quantisation matrix coefficients for non-intra-coded frames, in + zigzag scanning order. It is relevant for both luma and chroma + components, although it can be superseded by the chroma-specific matrix + for non-4:2:0 YUV formats. + * - __u8 + - ``chroma_intra_quantiser_matrix[64]`` + - The quantisation matrix coefficients for the chominance component of + intra-coded frames, in zigzag scanning order. Only relevant for + non-4:2:0 YUV formats. + * - __u8 + - ``chroma_non_intra_quantiser_matrix[64]`` + - The quantisation matrix coefficients for the chrominance component of + non-intra-coded frames, in zigzag scanning order. Only relevant for + non-4:2:0 YUV formats. + +.. raw:: latex + + \normalsize diff --git a/Documentation/userspace-api/media/v4l/ext-ctrls-codec.rst b/Documentation/userspace-api/media/v4l/ext-ctrls-codec.rst index f10b04fba229..0b8061666c57 100644 --- a/Documentation/userspace-api/media/v4l/ext-ctrls-codec.rst +++ b/Documentation/userspace-api/media/v4l/ext-ctrls-codec.rst @@ -1606,222 +1606,6 @@ enum v4l2_mpeg_video_h264_hierarchical_coding_type - ``V4L2_CID_MPEG_VIDEO_H264_HIER_CODING_L6_BR (integer)`` Indicates bit rate (bps) for hierarchical coding layer 6 for H264 encoder. -.. _v4l2-mpeg-mpeg2: - -``V4L2_CID_MPEG_VIDEO_MPEG2_SEQUENCE (struct)`` - Specifies the sequence parameters (as extracted from the bitstream) for the - associated MPEG-2 slice data. This includes fields matching the syntax - elements from the sequence header and sequence extension parts of the - bitstream as specified by :ref:`mpeg2part2`. - - .. note:: - - This compound control is not yet part of the public kernel API and - it is expected to change. - -.. c:type:: v4l2_ctrl_mpeg2_sequence - -.. cssclass:: longtable - -.. tabularcolumns:: |p{1.4cm}|p{6.5cm}|p{9.4cm}| - -.. flat-table:: struct v4l2_ctrl_mpeg2_sequence - :header-rows: 0 - :stub-columns: 0 - :widths: 1 1 2 - - * - __u16 - - ``horizontal_size`` - - The width of the displayable part of the frame's luminance component. - * - __u16 - - ``vertical_size`` - - The height of the displayable part of the frame's luminance component. - * - __u32 - - ``vbv_buffer_size`` - - Used to calculate the required size of the video buffering verifier, - defined (in bits) as: 16 * 1024 * vbv_buffer_size. - * - __u16 - - ``profile_and_level_indication`` - - The current profile and level indication as extracted from the - bitstream. - * - __u8 - - ``chroma_format`` - - The chrominance sub-sampling format (1: 4:2:0, 2: 4:2:2, 3: 4:4:4). - * - __u8 - - ``flags`` - - See :ref:`MPEG-2 Sequence Flags `. - -.. _mpeg2_sequence_flags: - -``MPEG-2 Sequence Flags`` - -.. cssclass:: longtable - -.. flat-table:: - :header-rows: 0 - :stub-columns: 0 - :widths: 1 1 2 - - * - ``V4L2_MPEG2_SEQ_FLAG_PROGRESSIVE`` - - 0x01 - - Indication that all the frames for the sequence are progressive instead - of interlaced. - -``V4L2_CID_MPEG_VIDEO_MPEG2_PICTURE (struct)`` - Specifies the picture parameters (as extracted from the bitstream) for the - associated MPEG-2 slice data. This includes fields matching the syntax - elements from the picture header and picture coding extension parts of the - bitstream as specified by :ref:`mpeg2part2`. - - .. note:: - - This compound control is not yet part of the public kernel API and - it is expected to change. - -.. c:type:: v4l2_ctrl_mpeg2_picture - -.. raw:: latex - - \small - -.. cssclass:: longtable - -.. tabularcolumns:: |p{1.0cm}|p{5.6cm}|p{10.7cm}| - -.. flat-table:: struct v4l2_ctrl_mpeg2_picture - :header-rows: 0 - :stub-columns: 0 - :widths: 1 1 2 - - * - __u64 - - ``backward_ref_ts`` - - Timestamp of the V4L2 capture buffer to use as backward reference, used - with B-coded and P-coded frames. The timestamp refers to the - ``timestamp`` field in struct :c:type:`v4l2_buffer`. Use the - :c:func:`v4l2_timeval_to_ns()` function to convert the struct - :c:type:`timeval` in struct :c:type:`v4l2_buffer` to a __u64. - * - __u64 - - ``forward_ref_ts`` - - Timestamp for the V4L2 capture buffer to use as forward reference, used - with B-coded frames. The timestamp refers to the ``timestamp`` field in - struct :c:type:`v4l2_buffer`. Use the :c:func:`v4l2_timeval_to_ns()` - function to convert the struct :c:type:`timeval` in struct - :c:type:`v4l2_buffer` to a __u64. - * - __u32 - - ``flags`` - - See :ref:`MPEG-2 Picture Flags `. - * - __u8 - - ``f_code[2][2]`` - - Motion vector codes. - * - __u8 - - ``picture_coding_type`` - - Picture coding type for the frame covered by the current slice - (V4L2_MPEG2_PICTURE_CODING_TYPE_I, V4L2_MPEG2_PICTURE_CODING_TYPE_P or - V4L2_MPEG2_PICTURE_CODING_TYPE_B). - * - __u8 - - ``picture_structure`` - - Picture structure (1: interlaced top field, 2: interlaced bottom field, - 3: progressive frame). - * - __u8 - - ``intra_dc_precision`` - - Precision of Discrete Cosine transform (0: 8 bits precision, - 1: 9 bits precision, 2: 10 bits precision, 3: 11 bits precision). - * - __u8 - - ``reserved[5]`` - - Applications and drivers must set this to zero. - - -.. _mpeg2_picture_flags: - -``MPEG-2 Picture Flags`` - -.. cssclass:: longtable - -.. flat-table:: - :header-rows: 0 - :stub-columns: 0 - :widths: 1 1 2 - - * - ``V4L2_MPEG2_PIC_FLAG_TOP_FIELD_FIRST`` - - 0x00000001 - - If set and it's an interlaced stream, top field is output first. - * - ``V4L2_MPEG2_PIC_FLAG_FRAME_PRED_DCT`` - - 0x00000002 - - If set only frame-DCT and frame prediction are used. - * - ``V4L2_MPEG2_PIC_FLAG_CONCEALMENT_MV`` - - 0x00000004 - - If set motion vectors are coded for intra macroblocks. - * - ``V4L2_MPEG2_PIC_FLAG_Q_SCALE_TYPE`` - - 0x00000008 - - This flag affects the inverse quantization process. - * - ``V4L2_MPEG2_PIC_FLAG_INTRA_VLC`` - - 0x00000010 - - This flag affects the decoding of transform coefficient data. - * - ``V4L2_MPEG2_PIC_FLAG_ALT_SCAN`` - - 0x00000020 - - This flag affects the decoding of transform coefficient data. - * - ``V4L2_MPEG2_PIC_FLAG_REPEAT_FIRST`` - - 0x00000040 - - This flag affects the decoding process of progressive frames. - * - ``V4L2_MPEG2_PIC_FLAG_PROGRESSIVE`` - - 0x00000080 - - Indicates whether the current frame is progressive. - -.. raw:: latex - - \normalsize - -``V4L2_CID_MPEG_VIDEO_MPEG2_QUANTISATION (struct)`` - Specifies quantisation matrices (as extracted from the bitstream) for the - associated MPEG-2 slice data. - - .. note:: - - This compound control is not yet part of the public kernel API and - it is expected to change. - -.. c:type:: v4l2_ctrl_mpeg2_quantisation - -.. tabularcolumns:: |p{0.8cm}|p{8.0cm}|p{8.5cm}| - -.. cssclass:: longtable - -.. raw:: latex - - \small - -.. flat-table:: struct v4l2_ctrl_mpeg2_quantisation - :header-rows: 0 - :stub-columns: 0 - :widths: 1 1 2 - - * - __u8 - - ``intra_quantiser_matrix[64]`` - - The quantisation matrix coefficients for intra-coded frames, in zigzag - scanning order. It is relevant for both luma and chroma components, - although it can be superseded by the chroma-specific matrix for - non-4:2:0 YUV formats. - * - __u8 - - ``non_intra_quantiser_matrix[64]`` - - The quantisation matrix coefficients for non-intra-coded frames, in - zigzag scanning order. It is relevant for both luma and chroma - components, although it can be superseded by the chroma-specific matrix - for non-4:2:0 YUV formats. - * - __u8 - - ``chroma_intra_quantiser_matrix[64]`` - - The quantisation matrix coefficients for the chominance component of - intra-coded frames, in zigzag scanning order. Only relevant for - non-4:2:0 YUV formats. - * - __u8 - - ``chroma_non_intra_quantiser_matrix[64]`` - - The quantisation matrix coefficients for the chrominance component of - non-intra-coded frames, in zigzag scanning order. Only relevant for - non-4:2:0 YUV formats. - -.. raw:: latex - - \normalsize - ``V4L2_CID_FWHT_I_FRAME_QP (integer)`` Quantization parameter for an I frame for FWHT. Valid range: from 1 to 31. diff --git a/Documentation/userspace-api/media/v4l/pixfmt-compressed.rst b/Documentation/userspace-api/media/v4l/pixfmt-compressed.rst index 6c10a062adac..0ede39907ee2 100644 --- a/Documentation/userspace-api/media/v4l/pixfmt-compressed.rst +++ b/Documentation/userspace-api/media/v4l/pixfmt-compressed.rst @@ -112,13 +112,13 @@ Compressed Formats - 'MG2S' - MPEG-2 parsed slice data, as extracted from the MPEG-2 bitstream. This format is adapted for stateless video decoders that implement a - MPEG-2 pipeline (using the :ref:`mem2mem` and :ref:`media-request-api`). + MPEG-2 pipeline with the :ref:`stateless_decoder`. Metadata associated with the frame to decode is required to be passed - through the ``V4L2_CID_MPEG_VIDEO_MPEG2_SEQUENCE`` and - ``V4L2_CID_MPEG_VIDEO_MPEG2_PICTURE`` controls. + through the ``V4L2_CID_STATELESS_MPEG2_SEQUENCE`` and + ``V4L2_CID_STATELESS_MPEG2_PICTURE`` controls. Quantisation matrices can optionally be specified through the - ``V4L2_CID_MPEG_VIDEO_MPEG2_QUANTISATION`` control. - See the :ref:`associated Codec Control IDs `. + ``V4L2_CID_STATELESS_MPEG2_QUANTISATION`` control. + See the :ref:`associated Codec Control IDs `. Exactly one output and one capture buffer must be provided for use with this pixel format. The output buffer must contain the appropriate number of macroblocks to decode a full corresponding frame to the matching diff --git a/Documentation/userspace-api/media/v4l/vidioc-g-ext-ctrls.rst b/Documentation/userspace-api/media/v4l/vidioc-g-ext-ctrls.rst index 3ba22983d21f..2d6bc8d94380 100644 --- a/Documentation/userspace-api/media/v4l/vidioc-g-ext-ctrls.rst +++ b/Documentation/userspace-api/media/v4l/vidioc-g-ext-ctrls.rst @@ -221,6 +221,18 @@ still cause this situation. - ``p_vp8_frame`` - A pointer to a struct :c:type:`v4l2_ctrl_vp8_frame`. Valid if this control is of type ``V4L2_CTRL_TYPE_VP8_FRAME``. + * - struct :c:type:`v4l2_ctrl_mpeg2_sequence` * + - ``p_mpeg2_sequence`` + - A pointer to a struct :c:type:`v4l2_ctrl_mpeg2_sequence`. Valid if this control is + of type ``V4L2_CTRL_TYPE_MPEG2_SEQUENCE``. + * - struct :c:type:`v4l2_ctrl_mpeg2_picture` * + - ``p_mpeg2_picture`` + - A pointer to a struct :c:type:`v4l2_ctrl_mpeg2_picture`. Valid if this control is + of type ``V4L2_CTRL_TYPE_MPEG2_PICTURE``. + * - struct :c:type:`v4l2_ctrl_mpeg2_quantisation` * + - ``p_mpeg2_quantisation`` + - A pointer to a struct :c:type:`v4l2_ctrl_mpeg2_quantisation`. Valid if this control is + of type ``V4L2_CTRL_TYPE_MPEG2_QUANTISATION``. * - struct :c:type:`v4l2_ctrl_hdr10_cll_info` * - ``p_hdr10_cll`` - A pointer to a struct :c:type:`v4l2_ctrl_hdr10_cll_info`. Valid if this control is diff --git a/drivers/media/v4l2-core/v4l2-ctrls.c b/drivers/media/v4l2-core/v4l2-ctrls.c index a693ff8dc3dc..d4e2c7318ee6 100644 --- a/drivers/media/v4l2-core/v4l2-ctrls.c +++ b/drivers/media/v4l2-core/v4l2-ctrls.c @@ -977,9 +977,6 @@ const char *v4l2_ctrl_get_name(u32 id) case V4L2_CID_MPEG_VIDEO_LTR_COUNT: return "LTR Count"; case V4L2_CID_MPEG_VIDEO_FRAME_LTR_INDEX: return "Frame LTR Index"; case V4L2_CID_MPEG_VIDEO_USE_LTR_FRAMES: return "Use LTR Frames"; - case V4L2_CID_MPEG_VIDEO_MPEG2_SEQUENCE: return "MPEG-2 Sequence Header"; - case V4L2_CID_MPEG_VIDEO_MPEG2_PICTURE: return "MPEG-2 Picture Header"; - case V4L2_CID_MPEG_VIDEO_MPEG2_QUANTISATION: return "MPEG-2 Quantisation Matrices"; case V4L2_CID_FWHT_I_FRAME_QP: return "FWHT I-Frame QP Value"; case V4L2_CID_FWHT_P_FRAME_QP: return "FWHT P-Frame QP Value"; @@ -1228,6 +1225,9 @@ const char *v4l2_ctrl_get_name(u32 id) case V4L2_CID_STATELESS_H264_DECODE_PARAMS: return "H264 Decode Parameters"; case V4L2_CID_STATELESS_FWHT_PARAMS: return "FWHT Stateless Parameters"; case V4L2_CID_STATELESS_VP8_FRAME: return "VP8 Frame Parameters"; + case V4L2_CID_STATELESS_MPEG2_SEQUENCE: return "MPEG-2 Sequence Header"; + case V4L2_CID_STATELESS_MPEG2_PICTURE: return "MPEG-2 Picture Header"; + case V4L2_CID_STATELESS_MPEG2_QUANTISATION: return "MPEG-2 Quantisation Matrices"; /* Colorimetry controls */ /* Keep the order of the 'case's the same as in v4l2-controls.h! */ @@ -1500,13 +1500,13 @@ void v4l2_ctrl_fill(u32 id, const char **name, enum v4l2_ctrl_type *type, case V4L2_CID_RDS_TX_ALT_FREQS: *type = V4L2_CTRL_TYPE_U32; break; - case V4L2_CID_MPEG_VIDEO_MPEG2_SEQUENCE: + case V4L2_CID_STATELESS_MPEG2_SEQUENCE: *type = V4L2_CTRL_TYPE_MPEG2_SEQUENCE; break; - case V4L2_CID_MPEG_VIDEO_MPEG2_PICTURE: + case V4L2_CID_STATELESS_MPEG2_PICTURE: *type = V4L2_CTRL_TYPE_MPEG2_PICTURE; break; - case V4L2_CID_MPEG_VIDEO_MPEG2_QUANTISATION: + case V4L2_CID_STATELESS_MPEG2_QUANTISATION: *type = V4L2_CTRL_TYPE_MPEG2_QUANTISATION; break; case V4L2_CID_STATELESS_FWHT_PARAMS: diff --git a/drivers/staging/media/hantro/hantro_drv.c b/drivers/staging/media/hantro/hantro_drv.c index dc9478ac7141..2f6b01c7a6a0 100644 --- a/drivers/staging/media/hantro/hantro_drv.c +++ b/drivers/staging/media/hantro/hantro_drv.c @@ -298,17 +298,17 @@ static const struct hantro_ctrl controls[] = { }, { .codec = HANTRO_MPEG2_DECODER, .cfg = { - .id = V4L2_CID_MPEG_VIDEO_MPEG2_SEQUENCE, + .id = V4L2_CID_STATELESS_MPEG2_SEQUENCE, }, }, { .codec = HANTRO_MPEG2_DECODER, .cfg = { - .id = V4L2_CID_MPEG_VIDEO_MPEG2_PICTURE, + .id = V4L2_CID_STATELESS_MPEG2_PICTURE, }, }, { .codec = HANTRO_MPEG2_DECODER, .cfg = { - .id = V4L2_CID_MPEG_VIDEO_MPEG2_QUANTISATION, + .id = V4L2_CID_STATELESS_MPEG2_QUANTISATION, }, }, { .codec = HANTRO_VP8_DECODER, diff --git a/drivers/staging/media/hantro/hantro_g1_mpeg2_dec.c b/drivers/staging/media/hantro/hantro_g1_mpeg2_dec.c index 25d912cbe2ff..6ee1a19d189b 100644 --- a/drivers/staging/media/hantro/hantro_g1_mpeg2_dec.c +++ b/drivers/staging/media/hantro/hantro_g1_mpeg2_dec.c @@ -83,7 +83,7 @@ hantro_g1_mpeg2_dec_set_quantisation(struct hantro_dev *vpu, { struct v4l2_ctrl_mpeg2_quantisation *q; - q = hantro_get_ctrl(ctx, V4L2_CID_MPEG_VIDEO_MPEG2_QUANTISATION); + q = hantro_get_ctrl(ctx, V4L2_CID_STATELESS_MPEG2_QUANTISATION); hantro_mpeg2_dec_copy_qtable(ctx->mpeg2_dec.qtable.cpu, q); vdpu_write_relaxed(vpu, ctx->mpeg2_dec.qtable.dma, G1_REG_QTABLE_BASE); } @@ -160,9 +160,9 @@ void hantro_g1_mpeg2_dec_run(struct hantro_ctx *ctx) hantro_start_prepare_run(ctx); seq = hantro_get_ctrl(ctx, - V4L2_CID_MPEG_VIDEO_MPEG2_SEQUENCE); + V4L2_CID_STATELESS_MPEG2_SEQUENCE); pic = hantro_get_ctrl(ctx, - V4L2_CID_MPEG_VIDEO_MPEG2_PICTURE); + V4L2_CID_STATELESS_MPEG2_PICTURE); reg = G1_REG_DEC_AXI_RD_ID(0) | G1_REG_DEC_TIMEOUT_E(1) | diff --git a/drivers/staging/media/hantro/rk3399_vpu_hw_mpeg2_dec.c b/drivers/staging/media/hantro/rk3399_vpu_hw_mpeg2_dec.c index d16d76760278..2527dce7eb18 100644 --- a/drivers/staging/media/hantro/rk3399_vpu_hw_mpeg2_dec.c +++ b/drivers/staging/media/hantro/rk3399_vpu_hw_mpeg2_dec.c @@ -85,7 +85,7 @@ rk3399_vpu_mpeg2_dec_set_quantisation(struct hantro_dev *vpu, { struct v4l2_ctrl_mpeg2_quantisation *q; - q = hantro_get_ctrl(ctx, V4L2_CID_MPEG_VIDEO_MPEG2_QUANTISATION); + q = hantro_get_ctrl(ctx, V4L2_CID_STATELESS_MPEG2_QUANTISATION); hantro_mpeg2_dec_copy_qtable(ctx->mpeg2_dec.qtable.cpu, q); vdpu_write_relaxed(vpu, ctx->mpeg2_dec.qtable.dma, VDPU_REG_QTABLE_BASE); } @@ -162,9 +162,9 @@ void rk3399_vpu_mpeg2_dec_run(struct hantro_ctx *ctx) hantro_start_prepare_run(ctx); seq = hantro_get_ctrl(ctx, - V4L2_CID_MPEG_VIDEO_MPEG2_SEQUENCE); + V4L2_CID_STATELESS_MPEG2_SEQUENCE); pic = hantro_get_ctrl(ctx, - V4L2_CID_MPEG_VIDEO_MPEG2_PICTURE); + V4L2_CID_STATELESS_MPEG2_PICTURE); reg = VDPU_REG_DEC_ADV_PRE_DIS(0) | VDPU_REG_DEC_SCMD_DIS(0) | diff --git a/drivers/staging/media/sunxi/cedrus/cedrus.c b/drivers/staging/media/sunxi/cedrus/cedrus.c index 4430c8fa2cc7..fa348c09f844 100644 --- a/drivers/staging/media/sunxi/cedrus/cedrus.c +++ b/drivers/staging/media/sunxi/cedrus/cedrus.c @@ -31,19 +31,19 @@ static const struct cedrus_control cedrus_controls[] = { { .cfg = { - .id = V4L2_CID_MPEG_VIDEO_MPEG2_SEQUENCE, + .id = V4L2_CID_STATELESS_MPEG2_SEQUENCE, }, .codec = CEDRUS_CODEC_MPEG2, }, { .cfg = { - .id = V4L2_CID_MPEG_VIDEO_MPEG2_PICTURE, + .id = V4L2_CID_STATELESS_MPEG2_PICTURE, }, .codec = CEDRUS_CODEC_MPEG2, }, { .cfg = { - .id = V4L2_CID_MPEG_VIDEO_MPEG2_QUANTISATION, + .id = V4L2_CID_STATELESS_MPEG2_QUANTISATION, }, .codec = CEDRUS_CODEC_MPEG2, }, diff --git a/drivers/staging/media/sunxi/cedrus/cedrus_dec.c b/drivers/staging/media/sunxi/cedrus/cedrus_dec.c index e98185c1f5a7..97e410d92506 100644 --- a/drivers/staging/media/sunxi/cedrus/cedrus_dec.c +++ b/drivers/staging/media/sunxi/cedrus/cedrus_dec.c @@ -41,11 +41,11 @@ void cedrus_device_run(void *priv) switch (ctx->src_fmt.pixelformat) { case V4L2_PIX_FMT_MPEG2_SLICE: run.mpeg2.sequence = cedrus_find_control_data(ctx, - V4L2_CID_MPEG_VIDEO_MPEG2_SEQUENCE); + V4L2_CID_STATELESS_MPEG2_SEQUENCE); run.mpeg2.picture = cedrus_find_control_data(ctx, - V4L2_CID_MPEG_VIDEO_MPEG2_PICTURE); + V4L2_CID_STATELESS_MPEG2_PICTURE); run.mpeg2.quantisation = cedrus_find_control_data(ctx, - V4L2_CID_MPEG_VIDEO_MPEG2_QUANTISATION); + V4L2_CID_STATELESS_MPEG2_QUANTISATION); break; case V4L2_PIX_FMT_H264_SLICE: diff --git a/include/media/mpeg2-ctrls.h b/include/media/mpeg2-ctrls.h deleted file mode 100644 index a3d19de9e53a..000000000000 --- a/include/media/mpeg2-ctrls.h +++ /dev/null @@ -1,126 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * These are the MPEG2 state controls for use with stateless MPEG-2 - * codec drivers. - * - * It turns out that these structs are not stable yet and will undergo - * more changes. So keep them private until they are stable and ready to - * become part of the official public API. - */ - -#ifndef _MPEG2_CTRLS_H_ -#define _MPEG2_CTRLS_H_ - -#define V4L2_CID_MPEG_VIDEO_MPEG2_QUANTISATION (V4L2_CID_CODEC_BASE+251) -#define V4L2_CID_MPEG_VIDEO_MPEG2_SEQUENCE (V4L2_CID_CODEC_BASE+252) -#define V4L2_CID_MPEG_VIDEO_MPEG2_PICTURE (V4L2_CID_CODEC_BASE+253) - -/* enum v4l2_ctrl_type type values */ -#define V4L2_MPEG2_SEQ_FLAG_PROGRESSIVE 0x01 - -/** - * struct v4l2_ctrl_mpeg2_sequence - MPEG-2 sequence header - * - * All the members on this structure match the sequence header and sequence - * extension syntaxes as specified by the MPEG-2 specification. - * - * Fields horizontal_size, vertical_size and vbv_buffer_size are a - * combination of respective _value and extension syntax elements, - * as described in section 6.3.3 "Sequence header". - * - * @horizontal_size: combination of elements horizontal_size_value and - * horizontal_size_extension. - * @vertical_size: combination of elements vertical_size_value and - * vertical_size_extension. - * @vbv_buffer_size: combination of elements vbv_buffer_size_value and - * vbv_buffer_size_extension. - * @profile_and_level_indication: see MPEG-2 specification. - * @chroma_format: see MPEG-2 specification. - * @flags: see V4L2_MPEG2_SEQ_FLAG_{}. - */ -struct v4l2_ctrl_mpeg2_sequence { - __u16 horizontal_size; - __u16 vertical_size; - __u32 vbv_buffer_size; - __u16 profile_and_level_indication; - __u8 chroma_format; - __u8 flags; -}; - -#define V4L2_MPEG2_PIC_CODING_TYPE_I 1 -#define V4L2_MPEG2_PIC_CODING_TYPE_P 2 -#define V4L2_MPEG2_PIC_CODING_TYPE_B 3 -#define V4L2_MPEG2_PIC_CODING_TYPE_D 4 - -#define V4L2_MPEG2_PIC_TOP_FIELD 0x1 -#define V4L2_MPEG2_PIC_BOTTOM_FIELD 0x2 -#define V4L2_MPEG2_PIC_FRAME 0x3 - -#define V4L2_MPEG2_PIC_FLAG_TOP_FIELD_FIRST 0x0001 -#define V4L2_MPEG2_PIC_FLAG_FRAME_PRED_DCT 0x0002 -#define V4L2_MPEG2_PIC_FLAG_CONCEALMENT_MV 0x0004 -#define V4L2_MPEG2_PIC_FLAG_Q_SCALE_TYPE 0x0008 -#define V4L2_MPEG2_PIC_FLAG_INTRA_VLC 0x0010 -#define V4L2_MPEG2_PIC_FLAG_ALT_SCAN 0x0020 -#define V4L2_MPEG2_PIC_FLAG_REPEAT_FIRST 0x0040 -#define V4L2_MPEG2_PIC_FLAG_PROGRESSIVE 0x0080 - -/** - * struct v4l2_ctrl_mpeg2_picture - MPEG-2 picture header - * - * All the members on this structure match the picture header and picture - * coding extension syntaxes as specified by the MPEG-2 specification. - * - * @backward_ref_ts: timestamp of the V4L2 capture buffer to use as - * reference for backward prediction. - * @forward_ref_ts: timestamp of the V4L2 capture buffer to use as - * reference for forward prediction. These timestamp refers to the - * timestamp field in struct v4l2_buffer. Use v4l2_timeval_to_ns() - * to convert the struct timeval to a __u64. - * @flags: see V4L2_MPEG2_PIC_FLAG_{}. - * @f_code[2][2]: see MPEG-2 specification. - * @picture_coding_type: see MPEG-2 specification. - * @picture_structure: see V4L2_MPEG2_PIC_{}_FIELD. - * @intra_dc_precision: see MPEG-2 specification. - * @reserved: padding field. Should be zeroed by applications. - */ -struct v4l2_ctrl_mpeg2_picture { - __u64 backward_ref_ts; - __u64 forward_ref_ts; - __u32 flags; - __u8 f_code[2][2]; - __u8 picture_coding_type; - __u8 picture_structure; - __u8 intra_dc_precision; - __u8 reserved[5]; -}; - -/** - * struct v4l2_ctrl_mpeg2_quantisation - MPEG-2 quantisation - * - * Quantization matrices as specified by section 6.3.7 - * "Quant matrix extension". - * - * @intra_quantiser_matrix: The quantisation matrix coefficients - * for intra-coded frames, in zigzag scanning order. It is relevant - * for both luma and chroma components, although it can be superseded - * by the chroma-specific matrix for non-4:2:0 YUV formats. - * @non_intra_quantiser_matrix: The quantisation matrix coefficients - * for non-intra-coded frames, in zigzag scanning order. It is relevant - * for both luma and chroma components, although it can be superseded - * by the chroma-specific matrix for non-4:2:0 YUV formats. - * @chroma_intra_quantiser_matrix: The quantisation matrix coefficients - * for the chominance component of intra-coded frames, in zigzag scanning - * order. Only relevant for 4:2:2 and 4:4:4 YUV formats. - * @chroma_non_intra_quantiser_matrix: The quantisation matrix coefficients - * for the chrominance component of non-intra-coded frames, in zigzag scanning - * order. Only relevant for 4:2:2 and 4:4:4 YUV formats. - */ -struct v4l2_ctrl_mpeg2_quantisation { - __u8 intra_quantiser_matrix[64]; - __u8 non_intra_quantiser_matrix[64]; - __u8 chroma_intra_quantiser_matrix[64]; - __u8 chroma_non_intra_quantiser_matrix[64]; -}; - -#endif diff --git a/include/media/v4l2-ctrls.h b/include/media/v4l2-ctrls.h index 215e44172c66..575b59fbac77 100644 --- a/include/media/v4l2-ctrls.h +++ b/include/media/v4l2-ctrls.h @@ -17,7 +17,6 @@ * Include the stateless codec compound control definitions. * This will move to the public headers once this API is fully stable. */ -#include #include /* forward references */ diff --git a/include/uapi/linux/v4l2-controls.h b/include/uapi/linux/v4l2-controls.h index d43bec5f1afd..f96bea19c991 100644 --- a/include/uapi/linux/v4l2-controls.h +++ b/include/uapi/linux/v4l2-controls.h @@ -1862,6 +1862,118 @@ struct v4l2_ctrl_vp8_frame { __u64 flags; }; +/* Stateless MPEG-2 controls */ + +#define V4L2_MPEG2_SEQ_FLAG_PROGRESSIVE 0x01 + +#define V4L2_CID_STATELESS_MPEG2_SEQUENCE (V4L2_CID_CODEC_STATELESS_BASE+220) +/** + * struct v4l2_ctrl_mpeg2_sequence - MPEG-2 sequence header + * + * All the members on this structure match the sequence header and sequence + * extension syntaxes as specified by the MPEG-2 specification. + * + * Fields horizontal_size, vertical_size and vbv_buffer_size are a + * combination of respective _value and extension syntax elements, + * as described in section 6.3.3 "Sequence header". + * + * @horizontal_size: combination of elements horizontal_size_value and + * horizontal_size_extension. + * @vertical_size: combination of elements vertical_size_value and + * vertical_size_extension. + * @vbv_buffer_size: combination of elements vbv_buffer_size_value and + * vbv_buffer_size_extension. + * @profile_and_level_indication: see MPEG-2 specification. + * @chroma_format: see MPEG-2 specification. + * @flags: see V4L2_MPEG2_SEQ_FLAG_{}. + */ +struct v4l2_ctrl_mpeg2_sequence { + __u16 horizontal_size; + __u16 vertical_size; + __u32 vbv_buffer_size; + __u16 profile_and_level_indication; + __u8 chroma_format; + __u8 flags; +}; + +#define V4L2_MPEG2_PIC_CODING_TYPE_I 1 +#define V4L2_MPEG2_PIC_CODING_TYPE_P 2 +#define V4L2_MPEG2_PIC_CODING_TYPE_B 3 +#define V4L2_MPEG2_PIC_CODING_TYPE_D 4 + +#define V4L2_MPEG2_PIC_TOP_FIELD 0x1 +#define V4L2_MPEG2_PIC_BOTTOM_FIELD 0x2 +#define V4L2_MPEG2_PIC_FRAME 0x3 + +#define V4L2_MPEG2_PIC_FLAG_TOP_FIELD_FIRST 0x0001 +#define V4L2_MPEG2_PIC_FLAG_FRAME_PRED_DCT 0x0002 +#define V4L2_MPEG2_PIC_FLAG_CONCEALMENT_MV 0x0004 +#define V4L2_MPEG2_PIC_FLAG_Q_SCALE_TYPE 0x0008 +#define V4L2_MPEG2_PIC_FLAG_INTRA_VLC 0x0010 +#define V4L2_MPEG2_PIC_FLAG_ALT_SCAN 0x0020 +#define V4L2_MPEG2_PIC_FLAG_REPEAT_FIRST 0x0040 +#define V4L2_MPEG2_PIC_FLAG_PROGRESSIVE 0x0080 + +#define V4L2_CID_STATELESS_MPEG2_PICTURE (V4L2_CID_CODEC_STATELESS_BASE+221) +/** + * struct v4l2_ctrl_mpeg2_picture - MPEG-2 picture header + * + * All the members on this structure match the picture header and picture + * coding extension syntaxes as specified by the MPEG-2 specification. + * + * @backward_ref_ts: timestamp of the V4L2 capture buffer to use as + * reference for backward prediction. + * @forward_ref_ts: timestamp of the V4L2 capture buffer to use as + * reference for forward prediction. These timestamp refers to the + * timestamp field in struct v4l2_buffer. Use v4l2_timeval_to_ns() + * to convert the struct timeval to a __u64. + * @flags: see V4L2_MPEG2_PIC_FLAG_{}. + * @f_code: see MPEG-2 specification. + * @picture_coding_type: see MPEG-2 specification. + * @picture_structure: see V4L2_MPEG2_PIC_{}_FIELD. + * @intra_dc_precision: see MPEG-2 specification. + * @reserved: padding field. Should be zeroed by applications. + */ +struct v4l2_ctrl_mpeg2_picture { + __u64 backward_ref_ts; + __u64 forward_ref_ts; + __u32 flags; + __u8 f_code[2][2]; + __u8 picture_coding_type; + __u8 picture_structure; + __u8 intra_dc_precision; + __u8 reserved[5]; +}; + +#define V4L2_CID_STATELESS_MPEG2_QUANTISATION (V4L2_CID_CODEC_STATELESS_BASE+222) +/** + * struct v4l2_ctrl_mpeg2_quantisation - MPEG-2 quantisation + * + * Quantisation matrices as specified by section 6.3.7 + * "Quant matrix extension". + * + * @intra_quantiser_matrix: The quantisation matrix coefficients + * for intra-coded frames, in zigzag scanning order. It is relevant + * for both luma and chroma components, although it can be superseded + * by the chroma-specific matrix for non-4:2:0 YUV formats. + * @non_intra_quantiser_matrix: The quantisation matrix coefficients + * for non-intra-coded frames, in zigzag scanning order. It is relevant + * for both luma and chroma components, although it can be superseded + * by the chroma-specific matrix for non-4:2:0 YUV formats. + * @chroma_intra_quantiser_matrix: The quantisation matrix coefficients + * for the chominance component of intra-coded frames, in zigzag scanning + * order. Only relevant for 4:2:2 and 4:4:4 YUV formats. + * @chroma_non_intra_quantiser_matrix: The quantisation matrix coefficients + * for the chrominance component of non-intra-coded frames, in zigzag scanning + * order. Only relevant for 4:2:2 and 4:4:4 YUV formats. + */ +struct v4l2_ctrl_mpeg2_quantisation { + __u8 intra_quantiser_matrix[64]; + __u8 non_intra_quantiser_matrix[64]; + __u8 chroma_intra_quantiser_matrix[64]; + __u8 chroma_non_intra_quantiser_matrix[64]; +}; + #define V4L2_CID_COLORIMETRY_CLASS_BASE (V4L2_CTRL_CLASS_COLORIMETRY | 0x900) #define V4L2_CID_COLORIMETRY_CLASS (V4L2_CTRL_CLASS_COLORIMETRY | 1) diff --git a/include/uapi/linux/videodev2.h b/include/uapi/linux/videodev2.h index d3bb18a3a51b..9260791b8438 100644 --- a/include/uapi/linux/videodev2.h +++ b/include/uapi/linux/videodev2.h @@ -1747,6 +1747,9 @@ struct v4l2_ext_control { struct v4l2_ctrl_h264_decode_params __user *p_h264_decode_params; struct v4l2_ctrl_fwht_params __user *p_fwht_params; struct v4l2_ctrl_vp8_frame __user *p_vp8_frame; + struct v4l2_ctrl_mpeg2_sequence __user *p_mpeg2_sequence; + struct v4l2_ctrl_mpeg2_picture __user *p_mpeg2_picture; + struct v4l2_ctrl_mpeg2_quantisation __user *p_mpeg2_quantisation; void __user *ptr; }; } __attribute__ ((packed)); -- cgit v1.2.3 From 3e87f192b405960c0fe83e0925bd0dadf4f8cf43 Mon Sep 17 00:00:00 2001 From: Denis Salopek Date: Tue, 11 May 2021 23:00:04 +0200 Subject: bpf: Add lookup_and_delete_elem support to hashtab Extend the existing bpf_map_lookup_and_delete_elem() functionality to hashtab map types, in addition to stacks and queues. Create a new hashtab bpf_map_ops function that does lookup and deletion of the element under the same bucket lock and add the created map_ops to bpf.h. Signed-off-by: Denis Salopek Signed-off-by: Andrii Nakryiko Acked-by: Yonghong Song Link: https://lore.kernel.org/bpf/4d18480a3e990ffbf14751ddef0325eed3be2966.1620763117.git.denis.salopek@sartura.hr --- include/linux/bpf.h | 2 + include/uapi/linux/bpf.h | 13 ++++++ kernel/bpf/hashtab.c | 98 ++++++++++++++++++++++++++++++++++++++++++ kernel/bpf/syscall.c | 34 +++++++++++++-- tools/include/uapi/linux/bpf.h | 13 ++++++ 5 files changed, 156 insertions(+), 4 deletions(-) (limited to 'include/uapi') diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 9dc44ba97584..1e9a0ff3217b 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -70,6 +70,8 @@ struct bpf_map_ops { void *(*map_lookup_elem_sys_only)(struct bpf_map *map, void *key); int (*map_lookup_batch)(struct bpf_map *map, const union bpf_attr *attr, union bpf_attr __user *uattr); + int (*map_lookup_and_delete_elem)(struct bpf_map *map, void *key, + void *value, u64 flags); int (*map_lookup_and_delete_batch)(struct bpf_map *map, const union bpf_attr *attr, union bpf_attr __user *uattr); diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h index 418b9b813d65..562adeac1d67 100644 --- a/include/uapi/linux/bpf.h +++ b/include/uapi/linux/bpf.h @@ -527,6 +527,15 @@ union bpf_iter_link_info { * Look up an element with the given *key* in the map referred to * by the file descriptor *fd*, and if found, delete the element. * + * For **BPF_MAP_TYPE_QUEUE** and **BPF_MAP_TYPE_STACK** map + * types, the *flags* argument needs to be set to 0, but for other + * map types, it may be specified as: + * + * **BPF_F_LOCK** + * Look up and delete the value of a spin-locked map + * without returning the lock. This must be specified if + * the elements contain a spinlock. + * * The **BPF_MAP_TYPE_QUEUE** and **BPF_MAP_TYPE_STACK** map types * implement this command as a "pop" operation, deleting the top * element rather than one corresponding to *key*. @@ -536,6 +545,10 @@ union bpf_iter_link_info { * This command is only valid for the following map types: * * **BPF_MAP_TYPE_QUEUE** * * **BPF_MAP_TYPE_STACK** + * * **BPF_MAP_TYPE_HASH** + * * **BPF_MAP_TYPE_PERCPU_HASH** + * * **BPF_MAP_TYPE_LRU_HASH** + * * **BPF_MAP_TYPE_LRU_PERCPU_HASH** * * Return * Returns zero on success. On error, -1 is returned and *errno* diff --git a/kernel/bpf/hashtab.c b/kernel/bpf/hashtab.c index d7ebb12ffffc..9da0a0413a53 100644 --- a/kernel/bpf/hashtab.c +++ b/kernel/bpf/hashtab.c @@ -1401,6 +1401,100 @@ static void htab_map_seq_show_elem(struct bpf_map *map, void *key, rcu_read_unlock(); } +static int __htab_map_lookup_and_delete_elem(struct bpf_map *map, void *key, + void *value, bool is_lru_map, + bool is_percpu, u64 flags) +{ + struct bpf_htab *htab = container_of(map, struct bpf_htab, map); + struct hlist_nulls_head *head; + unsigned long bflags; + struct htab_elem *l; + u32 hash, key_size; + struct bucket *b; + int ret; + + key_size = map->key_size; + + hash = htab_map_hash(key, key_size, htab->hashrnd); + b = __select_bucket(htab, hash); + head = &b->head; + + ret = htab_lock_bucket(htab, b, hash, &bflags); + if (ret) + return ret; + + l = lookup_elem_raw(head, hash, key, key_size); + if (!l) { + ret = -ENOENT; + } else { + if (is_percpu) { + u32 roundup_value_size = round_up(map->value_size, 8); + void __percpu *pptr; + int off = 0, cpu; + + pptr = htab_elem_get_ptr(l, key_size); + for_each_possible_cpu(cpu) { + bpf_long_memcpy(value + off, + per_cpu_ptr(pptr, cpu), + roundup_value_size); + off += roundup_value_size; + } + } else { + u32 roundup_key_size = round_up(map->key_size, 8); + + if (flags & BPF_F_LOCK) + copy_map_value_locked(map, value, l->key + + roundup_key_size, + true); + else + copy_map_value(map, value, l->key + + roundup_key_size); + check_and_init_map_lock(map, value); + } + + hlist_nulls_del_rcu(&l->hash_node); + if (!is_lru_map) + free_htab_elem(htab, l); + } + + htab_unlock_bucket(htab, b, hash, bflags); + + if (is_lru_map && l) + bpf_lru_push_free(&htab->lru, &l->lru_node); + + return ret; +} + +static int htab_map_lookup_and_delete_elem(struct bpf_map *map, void *key, + void *value, u64 flags) +{ + return __htab_map_lookup_and_delete_elem(map, key, value, false, false, + flags); +} + +static int htab_percpu_map_lookup_and_delete_elem(struct bpf_map *map, + void *key, void *value, + u64 flags) +{ + return __htab_map_lookup_and_delete_elem(map, key, value, false, true, + flags); +} + +static int htab_lru_map_lookup_and_delete_elem(struct bpf_map *map, void *key, + void *value, u64 flags) +{ + return __htab_map_lookup_and_delete_elem(map, key, value, true, false, + flags); +} + +static int htab_lru_percpu_map_lookup_and_delete_elem(struct bpf_map *map, + void *key, void *value, + u64 flags) +{ + return __htab_map_lookup_and_delete_elem(map, key, value, true, true, + flags); +} + static int __htab_map_lookup_and_delete_batch(struct bpf_map *map, const union bpf_attr *attr, @@ -1934,6 +2028,7 @@ const struct bpf_map_ops htab_map_ops = { .map_free = htab_map_free, .map_get_next_key = htab_map_get_next_key, .map_lookup_elem = htab_map_lookup_elem, + .map_lookup_and_delete_elem = htab_map_lookup_and_delete_elem, .map_update_elem = htab_map_update_elem, .map_delete_elem = htab_map_delete_elem, .map_gen_lookup = htab_map_gen_lookup, @@ -1954,6 +2049,7 @@ const struct bpf_map_ops htab_lru_map_ops = { .map_free = htab_map_free, .map_get_next_key = htab_map_get_next_key, .map_lookup_elem = htab_lru_map_lookup_elem, + .map_lookup_and_delete_elem = htab_lru_map_lookup_and_delete_elem, .map_lookup_elem_sys_only = htab_lru_map_lookup_elem_sys, .map_update_elem = htab_lru_map_update_elem, .map_delete_elem = htab_lru_map_delete_elem, @@ -2077,6 +2173,7 @@ const struct bpf_map_ops htab_percpu_map_ops = { .map_free = htab_map_free, .map_get_next_key = htab_map_get_next_key, .map_lookup_elem = htab_percpu_map_lookup_elem, + .map_lookup_and_delete_elem = htab_percpu_map_lookup_and_delete_elem, .map_update_elem = htab_percpu_map_update_elem, .map_delete_elem = htab_map_delete_elem, .map_seq_show_elem = htab_percpu_map_seq_show_elem, @@ -2096,6 +2193,7 @@ const struct bpf_map_ops htab_lru_percpu_map_ops = { .map_free = htab_map_free, .map_get_next_key = htab_map_get_next_key, .map_lookup_elem = htab_lru_percpu_map_lookup_elem, + .map_lookup_and_delete_elem = htab_lru_percpu_map_lookup_and_delete_elem, .map_update_elem = htab_lru_percpu_map_update_elem, .map_delete_elem = htab_lru_map_delete_elem, .map_seq_show_elem = htab_percpu_map_seq_show_elem, diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index 1d1cd80a6e67..50457019da27 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -1483,7 +1483,7 @@ free_buf: return err; } -#define BPF_MAP_LOOKUP_AND_DELETE_ELEM_LAST_FIELD value +#define BPF_MAP_LOOKUP_AND_DELETE_ELEM_LAST_FIELD flags static int map_lookup_and_delete_elem(union bpf_attr *attr) { @@ -1499,6 +1499,9 @@ static int map_lookup_and_delete_elem(union bpf_attr *attr) if (CHECK_ATTR(BPF_MAP_LOOKUP_AND_DELETE_ELEM)) return -EINVAL; + if (attr->flags & ~BPF_F_LOCK) + return -EINVAL; + f = fdget(ufd); map = __bpf_map_get(f); if (IS_ERR(map)) @@ -1509,24 +1512,47 @@ static int map_lookup_and_delete_elem(union bpf_attr *attr) goto err_put; } + if (attr->flags && + (map->map_type == BPF_MAP_TYPE_QUEUE || + map->map_type == BPF_MAP_TYPE_STACK)) { + err = -EINVAL; + goto err_put; + } + + if ((attr->flags & BPF_F_LOCK) && + !map_value_has_spin_lock(map)) { + err = -EINVAL; + goto err_put; + } + key = __bpf_copy_key(ukey, map->key_size); if (IS_ERR(key)) { err = PTR_ERR(key); goto err_put; } - value_size = map->value_size; + value_size = bpf_map_value_size(map); err = -ENOMEM; value = kmalloc(value_size, GFP_USER | __GFP_NOWARN); if (!value) goto free_key; + err = -ENOTSUPP; if (map->map_type == BPF_MAP_TYPE_QUEUE || map->map_type == BPF_MAP_TYPE_STACK) { err = map->ops->map_pop_elem(map, value); - } else { - err = -ENOTSUPP; + } else if (map->map_type == BPF_MAP_TYPE_HASH || + map->map_type == BPF_MAP_TYPE_PERCPU_HASH || + map->map_type == BPF_MAP_TYPE_LRU_HASH || + map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH) { + if (!bpf_map_is_dev_bound(map)) { + bpf_disable_instrumentation(); + rcu_read_lock(); + err = map->ops->map_lookup_and_delete_elem(map, key, value, attr->flags); + rcu_read_unlock(); + bpf_enable_instrumentation(); + } } if (err) diff --git a/tools/include/uapi/linux/bpf.h b/tools/include/uapi/linux/bpf.h index 418b9b813d65..562adeac1d67 100644 --- a/tools/include/uapi/linux/bpf.h +++ b/tools/include/uapi/linux/bpf.h @@ -527,6 +527,15 @@ union bpf_iter_link_info { * Look up an element with the given *key* in the map referred to * by the file descriptor *fd*, and if found, delete the element. * + * For **BPF_MAP_TYPE_QUEUE** and **BPF_MAP_TYPE_STACK** map + * types, the *flags* argument needs to be set to 0, but for other + * map types, it may be specified as: + * + * **BPF_F_LOCK** + * Look up and delete the value of a spin-locked map + * without returning the lock. This must be specified if + * the elements contain a spinlock. + * * The **BPF_MAP_TYPE_QUEUE** and **BPF_MAP_TYPE_STACK** map types * implement this command as a "pop" operation, deleting the top * element rather than one corresponding to *key*. @@ -536,6 +545,10 @@ union bpf_iter_link_info { * This command is only valid for the following map types: * * **BPF_MAP_TYPE_QUEUE** * * **BPF_MAP_TYPE_STACK** + * * **BPF_MAP_TYPE_HASH** + * * **BPF_MAP_TYPE_PERCPU_HASH** + * * **BPF_MAP_TYPE_LRU_HASH** + * * **BPF_MAP_TYPE_LRU_PERCPU_HASH** * * Return * Returns zero on success. On error, -1 is returned and *errno* -- cgit v1.2.3 From e624d4ed4aa8cc3c69d1359b0aaea539203ed266 Mon Sep 17 00:00:00 2001 From: Hangbin Liu Date: Wed, 19 May 2021 17:07:45 +0800 Subject: xdp: Extend xdp_redirect_map with broadcast support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This patch adds two flags BPF_F_BROADCAST and BPF_F_EXCLUDE_INGRESS to extend xdp_redirect_map for broadcast support. With BPF_F_BROADCAST the packet will be broadcasted to all the interfaces in the map. with BPF_F_EXCLUDE_INGRESS the ingress interface will be excluded when do broadcasting. When getting the devices in dev hash map via dev_map_hash_get_next_key(), there is a possibility that we fall back to the first key when a device was removed. This will duplicate packets on some interfaces. So just walk the whole buckets to avoid this issue. For dev array map, we also walk the whole map to find valid interfaces. Function bpf_clear_redirect_map() was removed in commit ee75aef23afe ("bpf, xdp: Restructure redirect actions"). Add it back as we need to use ri->map again. With test topology: +-------------------+ +-------------------+ | Host A (i40e 10G) | ---------- | eno1(i40e 10G) | +-------------------+ | | | Host B | +-------------------+ | | | Host C (i40e 10G) | ---------- | eno2(i40e 10G) | +-------------------+ | | | +------+ | | veth0 -- | Peer | | | veth1 -- | | | | veth2 -- | NS | | | +------+ | +-------------------+ On Host A: # pktgen/pktgen_sample03_burst_single_flow.sh -i eno1 -d $dst_ip -m $dst_mac -s 64 On Host B(Intel(R) Xeon(R) CPU E5-2690 v3 @ 2.60GHz, 128G Memory): Use xdp_redirect_map and xdp_redirect_map_multi in samples/bpf for testing. All the veth peers in the NS have a XDP_DROP program loaded. The forward_map max_entries in xdp_redirect_map_multi is modify to 4. Testing the performance impact on the regular xdp_redirect path with and without patch (to check impact of additional check for broadcast mode): 5.12 rc4 | redirect_map i40e->i40e | 2.0M | 9.7M 5.12 rc4 | redirect_map i40e->veth | 1.7M | 11.8M 5.12 rc4 + patch | redirect_map i40e->i40e | 2.0M | 9.6M 5.12 rc4 + patch | redirect_map i40e->veth | 1.7M | 11.7M Testing the performance when cloning packets with the redirect_map_multi test, using a redirect map size of 4, filled with 1-3 devices: 5.12 rc4 + patch | redirect_map multi i40e->veth (x1) | 1.7M | 11.4M 5.12 rc4 + patch | redirect_map multi i40e->veth (x2) | 1.1M | 4.3M 5.12 rc4 + patch | redirect_map multi i40e->veth (x3) | 0.8M | 2.6M Signed-off-by: Hangbin Liu Signed-off-by: Daniel Borkmann Acked-by: Toke Høiland-Jørgensen Acked-by: Martin KaFai Lau Acked-by: John Fastabend Acked-by: Jesper Dangaard Brouer Link: https://lore.kernel.org/bpf/20210519090747.1655268-3-liuhangbin@gmail.com --- include/linux/bpf.h | 20 +++++ include/linux/filter.h | 19 ++++- include/net/xdp.h | 1 + include/trace/events/xdp.h | 6 +- include/uapi/linux/bpf.h | 14 +++- kernel/bpf/cpumap.c | 3 +- kernel/bpf/devmap.c | 183 ++++++++++++++++++++++++++++++++++++++++- net/core/filter.c | 37 ++++++++- net/core/xdp.c | 28 +++++++ net/xdp/xskmap.c | 3 +- tools/include/uapi/linux/bpf.h | 14 +++- 11 files changed, 313 insertions(+), 15 deletions(-) (limited to 'include/uapi') diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 1e9a0ff3217b..86dec5001ae2 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -1501,8 +1501,13 @@ int dev_xdp_enqueue(struct net_device *dev, struct xdp_buff *xdp, struct net_device *dev_rx); int dev_map_enqueue(struct bpf_dtab_netdev *dst, struct xdp_buff *xdp, struct net_device *dev_rx); +int dev_map_enqueue_multi(struct xdp_buff *xdp, struct net_device *dev_rx, + struct bpf_map *map, bool exclude_ingress); int dev_map_generic_redirect(struct bpf_dtab_netdev *dst, struct sk_buff *skb, struct bpf_prog *xdp_prog); +int dev_map_redirect_multi(struct net_device *dev, struct sk_buff *skb, + struct bpf_prog *xdp_prog, struct bpf_map *map, + bool exclude_ingress); bool dev_map_can_have_prog(struct bpf_map *map); void __cpu_map_flush(void); @@ -1670,6 +1675,13 @@ int dev_map_enqueue(struct bpf_dtab_netdev *dst, struct xdp_buff *xdp, return 0; } +static inline +int dev_map_enqueue_multi(struct xdp_buff *xdp, struct net_device *dev_rx, + struct bpf_map *map, bool exclude_ingress) +{ + return 0; +} + struct sk_buff; static inline int dev_map_generic_redirect(struct bpf_dtab_netdev *dst, @@ -1679,6 +1691,14 @@ static inline int dev_map_generic_redirect(struct bpf_dtab_netdev *dst, return 0; } +static inline +int dev_map_redirect_multi(struct net_device *dev, struct sk_buff *skb, + struct bpf_prog *xdp_prog, struct bpf_map *map, + bool exclude_ingress) +{ + return 0; +} + static inline void __cpu_map_flush(void) { } diff --git a/include/linux/filter.h b/include/linux/filter.h index 9a09547bc7ba..c5ad7df029ed 100644 --- a/include/linux/filter.h +++ b/include/linux/filter.h @@ -646,6 +646,7 @@ struct bpf_redirect_info { u32 flags; u32 tgt_index; void *tgt_value; + struct bpf_map *map; u32 map_id; enum bpf_map_type map_type; u32 kern_flags; @@ -1464,17 +1465,19 @@ static inline bool bpf_sk_lookup_run_v6(struct net *net, int protocol, } #endif /* IS_ENABLED(CONFIG_IPV6) */ -static __always_inline int __bpf_xdp_redirect_map(struct bpf_map *map, u32 ifindex, u64 flags, +static __always_inline int __bpf_xdp_redirect_map(struct bpf_map *map, u32 ifindex, + u64 flags, const u64 flag_mask, void *lookup_elem(struct bpf_map *map, u32 key)) { struct bpf_redirect_info *ri = this_cpu_ptr(&bpf_redirect_info); + const u64 action_mask = XDP_ABORTED | XDP_DROP | XDP_PASS | XDP_TX; /* Lower bits of the flags are used as return code on lookup failure */ - if (unlikely(flags > XDP_TX)) + if (unlikely(flags & ~(action_mask | flag_mask))) return XDP_ABORTED; ri->tgt_value = lookup_elem(map, ifindex); - if (unlikely(!ri->tgt_value)) { + if (unlikely(!ri->tgt_value) && !(flags & BPF_F_BROADCAST)) { /* If the lookup fails we want to clear out the state in the * redirect_info struct completely, so that if an eBPF program * performs multiple lookups, the last one always takes @@ -1482,13 +1485,21 @@ static __always_inline int __bpf_xdp_redirect_map(struct bpf_map *map, u32 ifind */ ri->map_id = INT_MAX; /* Valid map id idr range: [1,INT_MAX[ */ ri->map_type = BPF_MAP_TYPE_UNSPEC; - return flags; + return flags & action_mask; } ri->tgt_index = ifindex; ri->map_id = map->id; ri->map_type = map->map_type; + if (flags & BPF_F_BROADCAST) { + WRITE_ONCE(ri->map, map); + ri->flags = flags; + } else { + WRITE_ONCE(ri->map, NULL); + ri->flags = 0; + } + return XDP_REDIRECT; } diff --git a/include/net/xdp.h b/include/net/xdp.h index a5bc214a49d9..5533f0ab2afc 100644 --- a/include/net/xdp.h +++ b/include/net/xdp.h @@ -170,6 +170,7 @@ struct sk_buff *__xdp_build_skb_from_frame(struct xdp_frame *xdpf, struct sk_buff *xdp_build_skb_from_frame(struct xdp_frame *xdpf, struct net_device *dev); int xdp_alloc_skb_bulk(void **skbs, int n_skb, gfp_t gfp); +struct xdp_frame *xdpf_clone(struct xdp_frame *xdpf); static inline void xdp_convert_frame_to_buff(struct xdp_frame *frame, struct xdp_buff *xdp) diff --git a/include/trace/events/xdp.h b/include/trace/events/xdp.h index fcad3645a70b..c40fc97f9417 100644 --- a/include/trace/events/xdp.h +++ b/include/trace/events/xdp.h @@ -110,7 +110,11 @@ DECLARE_EVENT_CLASS(xdp_redirect_template, u32 ifindex = 0, map_index = index; if (map_type == BPF_MAP_TYPE_DEVMAP || map_type == BPF_MAP_TYPE_DEVMAP_HASH) { - ifindex = ((struct _bpf_dtab_netdev *)tgt)->dev->ifindex; + /* Just leave to_ifindex to 0 if do broadcast redirect, + * as tgt will be NULL. + */ + if (tgt) + ifindex = ((struct _bpf_dtab_netdev *)tgt)->dev->ifindex; } else if (map_type == BPF_MAP_TYPE_UNSPEC && map_id == INT_MAX) { ifindex = index; map_index = 0; diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h index 562adeac1d67..2c1ba70abbf1 100644 --- a/include/uapi/linux/bpf.h +++ b/include/uapi/linux/bpf.h @@ -2555,8 +2555,12 @@ union bpf_attr { * The lower two bits of *flags* are used as the return code if * the map lookup fails. This is so that the return value can be * one of the XDP program return codes up to **XDP_TX**, as chosen - * by the caller. Any higher bits in the *flags* argument must be - * unset. + * by the caller. The higher bits of *flags* can be set to + * BPF_F_BROADCAST or BPF_F_EXCLUDE_INGRESS as defined below. + * + * With BPF_F_BROADCAST the packet will be broadcasted to all the + * interfaces in the map, with BPF_F_EXCLUDE_INGRESS the ingress + * interface will be excluded when do broadcasting. * * See also **bpf_redirect**\ (), which only supports redirecting * to an ifindex, but doesn't require a map to do so. @@ -5122,6 +5126,12 @@ enum { BPF_F_BPRM_SECUREEXEC = (1ULL << 0), }; +/* Flags for bpf_redirect_map helper */ +enum { + BPF_F_BROADCAST = (1ULL << 3), + BPF_F_EXCLUDE_INGRESS = (1ULL << 4), +}; + #define __bpf_md_ptr(type, name) \ union { \ type name; \ diff --git a/kernel/bpf/cpumap.c b/kernel/bpf/cpumap.c index 5dd3e866599a..a1a0c4e791c6 100644 --- a/kernel/bpf/cpumap.c +++ b/kernel/bpf/cpumap.c @@ -601,7 +601,8 @@ static int cpu_map_get_next_key(struct bpf_map *map, void *key, void *next_key) static int cpu_map_redirect(struct bpf_map *map, u32 ifindex, u64 flags) { - return __bpf_xdp_redirect_map(map, ifindex, flags, __cpu_map_lookup_elem); + return __bpf_xdp_redirect_map(map, ifindex, flags, 0, + __cpu_map_lookup_elem); } static int cpu_map_btf_id; diff --git a/kernel/bpf/devmap.c b/kernel/bpf/devmap.c index 642264e32abd..f9148daab0e3 100644 --- a/kernel/bpf/devmap.c +++ b/kernel/bpf/devmap.c @@ -198,6 +198,7 @@ static void dev_map_free(struct bpf_map *map) list_del_rcu(&dtab->list); spin_unlock(&dev_map_lock); + bpf_clear_redirect_map(map); synchronize_rcu(); /* Make sure prior __dev_map_entry_free() have completed. */ @@ -515,6 +516,99 @@ int dev_map_enqueue(struct bpf_dtab_netdev *dst, struct xdp_buff *xdp, return __xdp_enqueue(dev, xdp, dev_rx, dst->xdp_prog); } +static bool is_valid_dst(struct bpf_dtab_netdev *obj, struct xdp_buff *xdp, + int exclude_ifindex) +{ + if (!obj || obj->dev->ifindex == exclude_ifindex || + !obj->dev->netdev_ops->ndo_xdp_xmit) + return false; + + if (xdp_ok_fwd_dev(obj->dev, xdp->data_end - xdp->data)) + return false; + + return true; +} + +static int dev_map_enqueue_clone(struct bpf_dtab_netdev *obj, + struct net_device *dev_rx, + struct xdp_frame *xdpf) +{ + struct xdp_frame *nxdpf; + + nxdpf = xdpf_clone(xdpf); + if (!nxdpf) + return -ENOMEM; + + bq_enqueue(obj->dev, nxdpf, dev_rx, obj->xdp_prog); + + return 0; +} + +int dev_map_enqueue_multi(struct xdp_buff *xdp, struct net_device *dev_rx, + struct bpf_map *map, bool exclude_ingress) +{ + struct bpf_dtab *dtab = container_of(map, struct bpf_dtab, map); + int exclude_ifindex = exclude_ingress ? dev_rx->ifindex : 0; + struct bpf_dtab_netdev *dst, *last_dst = NULL; + struct hlist_head *head; + struct xdp_frame *xdpf; + unsigned int i; + int err; + + xdpf = xdp_convert_buff_to_frame(xdp); + if (unlikely(!xdpf)) + return -EOVERFLOW; + + if (map->map_type == BPF_MAP_TYPE_DEVMAP) { + for (i = 0; i < map->max_entries; i++) { + dst = READ_ONCE(dtab->netdev_map[i]); + if (!is_valid_dst(dst, xdp, exclude_ifindex)) + continue; + + /* we only need n-1 clones; last_dst enqueued below */ + if (!last_dst) { + last_dst = dst; + continue; + } + + err = dev_map_enqueue_clone(last_dst, dev_rx, xdpf); + if (err) + return err; + + last_dst = dst; + } + } else { /* BPF_MAP_TYPE_DEVMAP_HASH */ + for (i = 0; i < dtab->n_buckets; i++) { + head = dev_map_index_hash(dtab, i); + hlist_for_each_entry_rcu(dst, head, index_hlist, + lockdep_is_held(&dtab->index_lock)) { + if (!is_valid_dst(dst, xdp, exclude_ifindex)) + continue; + + /* we only need n-1 clones; last_dst enqueued below */ + if (!last_dst) { + last_dst = dst; + continue; + } + + err = dev_map_enqueue_clone(last_dst, dev_rx, xdpf); + if (err) + return err; + + last_dst = dst; + } + } + } + + /* consume the last copy of the frame */ + if (last_dst) + bq_enqueue(last_dst->dev, xdpf, dev_rx, last_dst->xdp_prog); + else + xdp_return_frame_rx_napi(xdpf); /* dtab is empty */ + + return 0; +} + int dev_map_generic_redirect(struct bpf_dtab_netdev *dst, struct sk_buff *skb, struct bpf_prog *xdp_prog) { @@ -529,6 +623,87 @@ int dev_map_generic_redirect(struct bpf_dtab_netdev *dst, struct sk_buff *skb, return 0; } +static int dev_map_redirect_clone(struct bpf_dtab_netdev *dst, + struct sk_buff *skb, + struct bpf_prog *xdp_prog) +{ + struct sk_buff *nskb; + int err; + + nskb = skb_clone(skb, GFP_ATOMIC); + if (!nskb) + return -ENOMEM; + + err = dev_map_generic_redirect(dst, nskb, xdp_prog); + if (unlikely(err)) { + consume_skb(nskb); + return err; + } + + return 0; +} + +int dev_map_redirect_multi(struct net_device *dev, struct sk_buff *skb, + struct bpf_prog *xdp_prog, struct bpf_map *map, + bool exclude_ingress) +{ + struct bpf_dtab *dtab = container_of(map, struct bpf_dtab, map); + int exclude_ifindex = exclude_ingress ? dev->ifindex : 0; + struct bpf_dtab_netdev *dst, *last_dst = NULL; + struct hlist_head *head; + struct hlist_node *next; + unsigned int i; + int err; + + if (map->map_type == BPF_MAP_TYPE_DEVMAP) { + for (i = 0; i < map->max_entries; i++) { + dst = READ_ONCE(dtab->netdev_map[i]); + if (!dst || dst->dev->ifindex == exclude_ifindex) + continue; + + /* we only need n-1 clones; last_dst enqueued below */ + if (!last_dst) { + last_dst = dst; + continue; + } + + err = dev_map_redirect_clone(last_dst, skb, xdp_prog); + if (err) + return err; + + last_dst = dst; + } + } else { /* BPF_MAP_TYPE_DEVMAP_HASH */ + for (i = 0; i < dtab->n_buckets; i++) { + head = dev_map_index_hash(dtab, i); + hlist_for_each_entry_safe(dst, next, head, index_hlist) { + if (!dst || dst->dev->ifindex == exclude_ifindex) + continue; + + /* we only need n-1 clones; last_dst enqueued below */ + if (!last_dst) { + last_dst = dst; + continue; + } + + err = dev_map_redirect_clone(last_dst, skb, xdp_prog); + if (err) + return err; + + last_dst = dst; + } + } + } + + /* consume the first skb and return */ + if (last_dst) + return dev_map_generic_redirect(last_dst, skb, xdp_prog); + + /* dtab is empty */ + consume_skb(skb); + return 0; +} + static void *dev_map_lookup_elem(struct bpf_map *map, void *key) { struct bpf_dtab_netdev *obj = __dev_map_lookup_elem(map, *(u32 *)key); @@ -755,12 +930,16 @@ static int dev_map_hash_update_elem(struct bpf_map *map, void *key, void *value, static int dev_map_redirect(struct bpf_map *map, u32 ifindex, u64 flags) { - return __bpf_xdp_redirect_map(map, ifindex, flags, __dev_map_lookup_elem); + return __bpf_xdp_redirect_map(map, ifindex, flags, + BPF_F_BROADCAST | BPF_F_EXCLUDE_INGRESS, + __dev_map_lookup_elem); } static int dev_hash_map_redirect(struct bpf_map *map, u32 ifindex, u64 flags) { - return __bpf_xdp_redirect_map(map, ifindex, flags, __dev_map_hash_lookup_elem); + return __bpf_xdp_redirect_map(map, ifindex, flags, + BPF_F_BROADCAST | BPF_F_EXCLUDE_INGRESS, + __dev_map_hash_lookup_elem); } static int dev_map_btf_id; diff --git a/net/core/filter.c b/net/core/filter.c index 582ac196fd94..caa88955562e 100644 --- a/net/core/filter.c +++ b/net/core/filter.c @@ -3930,6 +3930,23 @@ void xdp_do_flush(void) } EXPORT_SYMBOL_GPL(xdp_do_flush); +void bpf_clear_redirect_map(struct bpf_map *map) +{ + struct bpf_redirect_info *ri; + int cpu; + + for_each_possible_cpu(cpu) { + ri = per_cpu_ptr(&bpf_redirect_info, cpu); + /* Avoid polluting remote cacheline due to writes if + * not needed. Once we pass this test, we need the + * cmpxchg() to make sure it hasn't been changed in + * the meantime by remote CPU. + */ + if (unlikely(READ_ONCE(ri->map) == map)) + cmpxchg(&ri->map, map, NULL); + } +} + int xdp_do_redirect(struct net_device *dev, struct xdp_buff *xdp, struct bpf_prog *xdp_prog) { @@ -3937,6 +3954,7 @@ int xdp_do_redirect(struct net_device *dev, struct xdp_buff *xdp, enum bpf_map_type map_type = ri->map_type; void *fwd = ri->tgt_value; u32 map_id = ri->map_id; + struct bpf_map *map; int err; ri->map_id = 0; /* Valid map id idr range: [1,INT_MAX[ */ @@ -3946,7 +3964,14 @@ int xdp_do_redirect(struct net_device *dev, struct xdp_buff *xdp, case BPF_MAP_TYPE_DEVMAP: fallthrough; case BPF_MAP_TYPE_DEVMAP_HASH: - err = dev_map_enqueue(fwd, xdp, dev); + map = READ_ONCE(ri->map); + if (unlikely(map)) { + WRITE_ONCE(ri->map, NULL); + err = dev_map_enqueue_multi(xdp, dev, map, + ri->flags & BPF_F_EXCLUDE_INGRESS); + } else { + err = dev_map_enqueue(fwd, xdp, dev); + } break; case BPF_MAP_TYPE_CPUMAP: err = cpu_map_enqueue(fwd, xdp, dev); @@ -3988,13 +4013,21 @@ static int xdp_do_generic_redirect_map(struct net_device *dev, enum bpf_map_type map_type, u32 map_id) { struct bpf_redirect_info *ri = this_cpu_ptr(&bpf_redirect_info); + struct bpf_map *map; int err; switch (map_type) { case BPF_MAP_TYPE_DEVMAP: fallthrough; case BPF_MAP_TYPE_DEVMAP_HASH: - err = dev_map_generic_redirect(fwd, skb, xdp_prog); + map = READ_ONCE(ri->map); + if (unlikely(map)) { + WRITE_ONCE(ri->map, NULL); + err = dev_map_redirect_multi(dev, skb, xdp_prog, map, + ri->flags & BPF_F_EXCLUDE_INGRESS); + } else { + err = dev_map_generic_redirect(fwd, skb, xdp_prog); + } if (unlikely(err)) goto err; break; diff --git a/net/core/xdp.c b/net/core/xdp.c index 858276e72c68..725d20f1b100 100644 --- a/net/core/xdp.c +++ b/net/core/xdp.c @@ -584,3 +584,31 @@ struct sk_buff *xdp_build_skb_from_frame(struct xdp_frame *xdpf, return __xdp_build_skb_from_frame(xdpf, skb, dev); } EXPORT_SYMBOL_GPL(xdp_build_skb_from_frame); + +struct xdp_frame *xdpf_clone(struct xdp_frame *xdpf) +{ + unsigned int headroom, totalsize; + struct xdp_frame *nxdpf; + struct page *page; + void *addr; + + headroom = xdpf->headroom + sizeof(*xdpf); + totalsize = headroom + xdpf->len; + + if (unlikely(totalsize > PAGE_SIZE)) + return NULL; + page = dev_alloc_page(); + if (!page) + return NULL; + addr = page_to_virt(page); + + memcpy(addr, xdpf, totalsize); + + nxdpf = addr; + nxdpf->data = addr + headroom; + nxdpf->frame_sz = PAGE_SIZE; + nxdpf->mem.type = MEM_TYPE_PAGE_ORDER0; + nxdpf->mem.id = 0; + + return nxdpf; +} diff --git a/net/xdp/xskmap.c b/net/xdp/xskmap.c index 67b4ce504852..9df75ea4a567 100644 --- a/net/xdp/xskmap.c +++ b/net/xdp/xskmap.c @@ -226,7 +226,8 @@ static int xsk_map_delete_elem(struct bpf_map *map, void *key) static int xsk_map_redirect(struct bpf_map *map, u32 ifindex, u64 flags) { - return __bpf_xdp_redirect_map(map, ifindex, flags, __xsk_map_lookup_elem); + return __bpf_xdp_redirect_map(map, ifindex, flags, 0, + __xsk_map_lookup_elem); } void xsk_map_try_sock_delete(struct xsk_map *map, struct xdp_sock *xs, diff --git a/tools/include/uapi/linux/bpf.h b/tools/include/uapi/linux/bpf.h index 562adeac1d67..2c1ba70abbf1 100644 --- a/tools/include/uapi/linux/bpf.h +++ b/tools/include/uapi/linux/bpf.h @@ -2555,8 +2555,12 @@ union bpf_attr { * The lower two bits of *flags* are used as the return code if * the map lookup fails. This is so that the return value can be * one of the XDP program return codes up to **XDP_TX**, as chosen - * by the caller. Any higher bits in the *flags* argument must be - * unset. + * by the caller. The higher bits of *flags* can be set to + * BPF_F_BROADCAST or BPF_F_EXCLUDE_INGRESS as defined below. + * + * With BPF_F_BROADCAST the packet will be broadcasted to all the + * interfaces in the map, with BPF_F_EXCLUDE_INGRESS the ingress + * interface will be excluded when do broadcasting. * * See also **bpf_redirect**\ (), which only supports redirecting * to an ifindex, but doesn't require a map to do so. @@ -5122,6 +5126,12 @@ enum { BPF_F_BPRM_SECUREEXEC = (1ULL << 0), }; +/* Flags for bpf_redirect_map helper */ +enum { + BPF_F_BROADCAST = (1ULL << 3), + BPF_F_EXCLUDE_INGRESS = (1ULL << 4), +}; + #define __bpf_md_ptr(type, name) \ union { \ type name; \ -- cgit v1.2.3 From 7e97d274db920df479e222fed10e7b242f90ffb0 Mon Sep 17 00:00:00 2001 From: Marc Kleine-Budde Date: Mon, 17 May 2021 13:24:25 +0200 Subject: can: uapi: update CAN-FD frame description Since an early version of the CAN-FD specification the bit that defines a CAN-FD frame on the wire, has been renamed from Extended Data Length (EDL) to FD Frame (FDF). To avoid confusion, update the struct canfd_frame description in the UAPI headers accordingly. Link: https://lore.kernel.org/r/20210517113727.77597-1-mkl@pengutronix.de Suggested-by: Ayoub Kaanich Acked-by: Oliver Hartkopp Signed-off-by: Marc Kleine-Budde --- include/uapi/linux/can.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'include/uapi') diff --git a/include/uapi/linux/can.h b/include/uapi/linux/can.h index c7535352fef6..ac5d7a31671f 100644 --- a/include/uapi/linux/can.h +++ b/include/uapi/linux/can.h @@ -123,8 +123,8 @@ struct can_frame { /* * defined bits for canfd_frame.flags * - * The use of struct canfd_frame implies the Extended Data Length (EDL) bit to - * be set in the CAN frame bitstream on the wire. The EDL bit switch turns + * The use of struct canfd_frame implies the FD Frame (FDF) bit to + * be set in the CAN frame bitstream on the wire. The FDF bit switch turns * the CAN controllers bitstream processor into the CAN FD mode which creates * two new options within the CAN FD frame specification: * -- cgit v1.2.3 From 02546884221279da2725e87e35348290470363d7 Mon Sep 17 00:00:00 2001 From: Oliver Hartkopp Date: Tue, 11 Apr 2017 15:43:43 +0200 Subject: can: uapi: introduce CANFD_FDF flag for mixed content in struct canfd_frame The struct can_frame and struct canfd_frame intentionally share the same layout to be able to write CAN frame content into a CAN FD frame structure. When this is done the former differentiation via CAN_MTU / CANFD_MTU is lost. CANFD_FDF allows programmers to mark CAN FD frames in the case of using struct canfd_frame for mixed CAN/CAN FD content (dual use). N.B. the Kernel APIs do NOT provide mixed CAN / CAN FD content inside of struct canfd_frame therefore the CANFD_FDF flag is disregarded by Linux. Link: https://lore.kernel.org/r/20170411134343.3089-1-socketcan@hartkopp.net Signed-off-by: Oliver Hartkopp Signed-off-by: Marc Kleine-Budde --- include/uapi/linux/can.h | 9 +++++++++ 1 file changed, 9 insertions(+) (limited to 'include/uapi') diff --git a/include/uapi/linux/can.h b/include/uapi/linux/can.h index ac5d7a31671f..90801ada2bbe 100644 --- a/include/uapi/linux/can.h +++ b/include/uapi/linux/can.h @@ -135,9 +135,18 @@ struct can_frame { * controller only the CANFD_BRS bit is relevant for real CAN controllers when * building a CAN FD frame for transmission. Setting the CANFD_ESI bit can make * sense for virtual CAN interfaces to test applications with echoed frames. + * + * The struct can_frame and struct canfd_frame intentionally share the same + * layout to be able to write CAN frame content into a CAN FD frame structure. + * When this is done the former differentiation via CAN_MTU / CANFD_MTU gets + * lost. CANFD_FDF allows programmers to mark CAN FD frames in the case of + * using struct canfd_frame for mixed CAN / CAN FD content (dual use). + * N.B. the Kernel APIs do NOT provide mixed CAN / CAN FD content inside of + * struct canfd_frame therefore the CANFD_FDF flag is disregarded by Linux. */ #define CANFD_BRS 0x01 /* bit rate switch (second bitrate for payload data) */ #define CANFD_ESI 0x02 /* error state indicator of the transmitting node */ +#define CANFD_FDF 0x04 /* mark CAN FD for dual use of struct canfd_frame */ /** * struct canfd_frame - CAN flexible data rate frame structure -- cgit v1.2.3 From fb1070d18edb37daf3979662975bc54625a19953 Mon Sep 17 00:00:00 2001 From: Joe Richey Date: Fri, 21 May 2021 01:58:43 -0700 Subject: KVM: X86: Use _BITUL() macro in UAPI headers Replace BIT() in KVM's UPAI header with _BITUL(). BIT() is not defined in the UAPI headers and its usage may cause userspace build errors. Fixes: fb04a1eddb1a ("KVM: X86: Implement ring-based dirty memory tracking") Signed-off-by: Joe Richey Message-Id: <20210521085849.37676-3-joerichey94@gmail.com> Signed-off-by: Paolo Bonzini --- include/uapi/linux/kvm.h | 5 +++-- tools/include/uapi/linux/kvm.h | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) (limited to 'include/uapi') diff --git a/include/uapi/linux/kvm.h b/include/uapi/linux/kvm.h index 3fd9a7e9d90c..79d9c44d1ad7 100644 --- a/include/uapi/linux/kvm.h +++ b/include/uapi/linux/kvm.h @@ -8,6 +8,7 @@ * Note: you must update KVM_API_VERSION if you change this interface. */ +#include #include #include #include @@ -1879,8 +1880,8 @@ struct kvm_hyperv_eventfd { * conversion after harvesting an entry. Also, it must not skip any * dirty bits, so that dirty bits are always harvested in sequence. */ -#define KVM_DIRTY_GFN_F_DIRTY BIT(0) -#define KVM_DIRTY_GFN_F_RESET BIT(1) +#define KVM_DIRTY_GFN_F_DIRTY _BITUL(0) +#define KVM_DIRTY_GFN_F_RESET _BITUL(1) #define KVM_DIRTY_GFN_F_MASK 0x3 /* diff --git a/tools/include/uapi/linux/kvm.h b/tools/include/uapi/linux/kvm.h index f6afee209620..26e6d94d64ed 100644 --- a/tools/include/uapi/linux/kvm.h +++ b/tools/include/uapi/linux/kvm.h @@ -8,6 +8,7 @@ * Note: you must update KVM_API_VERSION if you change this interface. */ +#include #include #include #include @@ -1834,8 +1835,8 @@ struct kvm_hyperv_eventfd { * conversion after harvesting an entry. Also, it must not skip any * dirty bits, so that dirty bits are always harvested in sequence. */ -#define KVM_DIRTY_GFN_F_DIRTY BIT(0) -#define KVM_DIRTY_GFN_F_RESET BIT(1) +#define KVM_DIRTY_GFN_F_DIRTY _BITUL(0) +#define KVM_DIRTY_GFN_F_RESET _BITUL(1) #define KVM_DIRTY_GFN_F_MASK 0x3 /* -- cgit v1.2.3 From 133dc203d77dff617d9c4673973ef3859be2c476 Mon Sep 17 00:00:00 2001 From: Phil Sutter Date: Tue, 4 May 2021 17:54:06 +0200 Subject: netfilter: nft_exthdr: Support SCTP chunks Chunks are SCTP header extensions similar in implementation to IPv6 extension headers or TCP options. Reusing exthdr expression to find and extract field values from them is therefore pretty straightforward. For now, this supports extracting data from chunks at a fixed offset (and length) only - chunks themselves are an extensible data structure; in order to make all fields available, a nested extension search is needed. Signed-off-by: Phil Sutter Signed-off-by: Pablo Neira Ayuso --- include/uapi/linux/netfilter/nf_tables.h | 2 ++ net/netfilter/nft_exthdr.c | 51 ++++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+) (limited to 'include/uapi') diff --git a/include/uapi/linux/netfilter/nf_tables.h b/include/uapi/linux/netfilter/nf_tables.h index 1fb4ca18ffbb..19715e2679d1 100644 --- a/include/uapi/linux/netfilter/nf_tables.h +++ b/include/uapi/linux/netfilter/nf_tables.h @@ -813,11 +813,13 @@ enum nft_exthdr_flags { * @NFT_EXTHDR_OP_IPV6: match against ipv6 extension headers * @NFT_EXTHDR_OP_TCP: match against tcp options * @NFT_EXTHDR_OP_IPV4: match against ipv4 options + * @NFT_EXTHDR_OP_SCTP: match against sctp chunks */ enum nft_exthdr_op { NFT_EXTHDR_OP_IPV6, NFT_EXTHDR_OP_TCPOPT, NFT_EXTHDR_OP_IPV4, + NFT_EXTHDR_OP_SCTP, __NFT_EXTHDR_OP_MAX }; #define NFT_EXTHDR_OP_MAX (__NFT_EXTHDR_OP_MAX - 1) diff --git a/net/netfilter/nft_exthdr.c b/net/netfilter/nft_exthdr.c index f64f0017e9a5..4d0b8e1c40c0 100644 --- a/net/netfilter/nft_exthdr.c +++ b/net/netfilter/nft_exthdr.c @@ -10,8 +10,10 @@ #include #include #include +#include #include #include +#include #include struct nft_exthdr { @@ -300,6 +302,43 @@ static void nft_exthdr_tcp_set_eval(const struct nft_expr *expr, } } +static void nft_exthdr_sctp_eval(const struct nft_expr *expr, + struct nft_regs *regs, + const struct nft_pktinfo *pkt) +{ + unsigned int offset = pkt->xt.thoff + sizeof(struct sctphdr); + struct nft_exthdr *priv = nft_expr_priv(expr); + u32 *dest = ®s->data[priv->dreg]; + const struct sctp_chunkhdr *sch; + struct sctp_chunkhdr _sch; + + do { + sch = skb_header_pointer(pkt->skb, offset, sizeof(_sch), &_sch); + if (!sch || !sch->length) + break; + + if (sch->type == priv->type) { + if (priv->flags & NFT_EXTHDR_F_PRESENT) { + nft_reg_store8(dest, true); + return; + } + if (priv->offset + priv->len > ntohs(sch->length) || + offset + ntohs(sch->length) > pkt->skb->len) + break; + + dest[priv->len / NFT_REG32_SIZE] = 0; + memcpy(dest, (char *)sch + priv->offset, priv->len); + return; + } + offset += SCTP_PAD4(ntohs(sch->length)); + } while (offset < pkt->skb->len); + + if (priv->flags & NFT_EXTHDR_F_PRESENT) + nft_reg_store8(dest, false); + else + regs->verdict.code = NFT_BREAK; +} + static const struct nla_policy nft_exthdr_policy[NFTA_EXTHDR_MAX + 1] = { [NFTA_EXTHDR_DREG] = { .type = NLA_U32 }, [NFTA_EXTHDR_TYPE] = { .type = NLA_U8 }, @@ -499,6 +538,14 @@ static const struct nft_expr_ops nft_exthdr_tcp_set_ops = { .dump = nft_exthdr_dump_set, }; +static const struct nft_expr_ops nft_exthdr_sctp_ops = { + .type = &nft_exthdr_type, + .size = NFT_EXPR_SIZE(sizeof(struct nft_exthdr)), + .eval = nft_exthdr_sctp_eval, + .init = nft_exthdr_init, + .dump = nft_exthdr_dump, +}; + static const struct nft_expr_ops * nft_exthdr_select_ops(const struct nft_ctx *ctx, const struct nlattr * const tb[]) @@ -529,6 +576,10 @@ nft_exthdr_select_ops(const struct nft_ctx *ctx, return &nft_exthdr_ipv4_ops; } break; + case NFT_EXTHDR_OP_SCTP: + if (tb[NFTA_EXTHDR_DREG]) + return &nft_exthdr_sctp_ops; + break; } return ERR_PTR(-EOPNOTSUPP); -- cgit v1.2.3 From dd8b477f9a3d8edb136207acb3652e1a34a661b7 Mon Sep 17 00:00:00 2001 From: Christian Brauner Date: Tue, 1 Jun 2021 11:33:59 +0200 Subject: mount: Support "nosymfollow" in new mount api Commit dab741e0e02b ("Add a "nosymfollow" mount option.") added support for the "nosymfollow" mount option allowing to block following symlinks when resolving paths. The mount option so far was only available in the old mount api. Make it available in the new mount api as well. Bonus is that it can be applied to a whole subtree not just a single mount. Cc: Christoph Hellwig Cc: Mattias Nissler Cc: Aleksa Sarai Cc: Al Viro Cc: Ross Zwisler Signed-off-by: Christian Brauner --- fs/namespace.c | 9 ++++++--- include/uapi/linux/mount.h | 1 + 2 files changed, 7 insertions(+), 3 deletions(-) (limited to 'include/uapi') diff --git a/fs/namespace.c b/fs/namespace.c index c3f1a78ba369..ab4174a3c802 100644 --- a/fs/namespace.c +++ b/fs/namespace.c @@ -3464,9 +3464,10 @@ out_type: return ret; } -#define FSMOUNT_VALID_FLAGS \ - (MOUNT_ATTR_RDONLY | MOUNT_ATTR_NOSUID | MOUNT_ATTR_NODEV | \ - MOUNT_ATTR_NOEXEC | MOUNT_ATTR__ATIME | MOUNT_ATTR_NODIRATIME) +#define FSMOUNT_VALID_FLAGS \ + (MOUNT_ATTR_RDONLY | MOUNT_ATTR_NOSUID | MOUNT_ATTR_NODEV | \ + MOUNT_ATTR_NOEXEC | MOUNT_ATTR__ATIME | MOUNT_ATTR_NODIRATIME | \ + MOUNT_ATTR_NOSYMFOLLOW) #define MOUNT_SETATTR_VALID_FLAGS (FSMOUNT_VALID_FLAGS | MOUNT_ATTR_IDMAP) @@ -3487,6 +3488,8 @@ static unsigned int attr_flags_to_mnt_flags(u64 attr_flags) mnt_flags |= MNT_NOEXEC; if (attr_flags & MOUNT_ATTR_NODIRATIME) mnt_flags |= MNT_NODIRATIME; + if (attr_flags & MOUNT_ATTR_NOSYMFOLLOW) + mnt_flags |= MNT_NOSYMFOLLOW; return mnt_flags; } diff --git a/include/uapi/linux/mount.h b/include/uapi/linux/mount.h index e6524ead2b7b..dd7a166fdf9c 100644 --- a/include/uapi/linux/mount.h +++ b/include/uapi/linux/mount.h @@ -120,6 +120,7 @@ enum fsconfig_command { #define MOUNT_ATTR_STRICTATIME 0x00000020 /* - Always perform atime updates */ #define MOUNT_ATTR_NODIRATIME 0x00000080 /* Do not update directory access times */ #define MOUNT_ATTR_IDMAP 0x00100000 /* Idmap mount to @userns_fd in struct mount_attr. */ +#define MOUNT_ATTR_NOSYMFOLLOW 0x00200000 /* Do not follow symlinks */ /* * mount_setattr() -- cgit v1.2.3 From e1d9a90a9bfdb0735062d3adb16b07314b4b7b01 Mon Sep 17 00:00:00 2001 From: Sharath Chandra Vurukala Date: Wed, 2 Jun 2021 00:58:35 +0530 Subject: net: ethernet: rmnet: Support for ingress MAPv5 checksum offload Adding support for processing of MAPv5 downlink packets. It involves parsing the Mapv5 packet and checking the csum header to know whether the hardware has validated the checksum and is valid or not. Based on the checksum valid bit the corresponding stats are incremented and skb->ip_summed is marked either CHECKSUM_UNNECESSARY or left as CHEKSUM_NONE to let network stack revalidate the checksum and update the respective snmp stats. Current MAPV1 header has been modified, the reserved field in the Mapv1 header is now used for next header indication. Signed-off-by: Sharath Chandra Vurukala Signed-off-by: David S. Miller --- .../net/ethernet/qualcomm/rmnet/rmnet_handlers.c | 17 ++++--- drivers/net/ethernet/qualcomm/rmnet/rmnet_map.h | 3 +- .../net/ethernet/qualcomm/rmnet/rmnet_map_data.c | 57 +++++++++++++++++++++- include/linux/if_rmnet.h | 30 ++++++++++-- include/uapi/linux/if_link.h | 1 + 5 files changed, 96 insertions(+), 12 deletions(-) (limited to 'include/uapi') diff --git a/drivers/net/ethernet/qualcomm/rmnet/rmnet_handlers.c b/drivers/net/ethernet/qualcomm/rmnet/rmnet_handlers.c index 0be5ac7ab261..706a225075a3 100644 --- a/drivers/net/ethernet/qualcomm/rmnet/rmnet_handlers.c +++ b/drivers/net/ethernet/qualcomm/rmnet/rmnet_handlers.c @@ -1,5 +1,5 @@ // SPDX-License-Identifier: GPL-2.0-only -/* Copyright (c) 2013-2018, The Linux Foundation. All rights reserved. +/* Copyright (c) 2013-2018, 2021, The Linux Foundation. All rights reserved. * * RMNET Data ingress/egress handler */ @@ -82,11 +82,16 @@ __rmnet_map_ingress_handler(struct sk_buff *skb, skb->dev = ep->egress_dev; - /* Subtract MAP header */ - skb_pull(skb, sizeof(struct rmnet_map_header)); - rmnet_set_skb_proto(skb); - - if (port->data_format & RMNET_FLAGS_INGRESS_MAP_CKSUMV4) { + if ((port->data_format & RMNET_FLAGS_INGRESS_MAP_CKSUMV5) && + (map_header->flags & MAP_NEXT_HEADER_FLAG)) { + if (rmnet_map_process_next_hdr_packet(skb, len)) + goto free_skb; + skb_pull(skb, sizeof(*map_header)); + rmnet_set_skb_proto(skb); + } else if (port->data_format & RMNET_FLAGS_INGRESS_MAP_CKSUMV4) { + /* Subtract MAP header */ + skb_pull(skb, sizeof(*map_header)); + rmnet_set_skb_proto(skb); if (!rmnet_map_checksum_downlink_packet(skb, len + pad)) skb->ip_summed = CHECKSUM_UNNECESSARY; } diff --git a/drivers/net/ethernet/qualcomm/rmnet/rmnet_map.h b/drivers/net/ethernet/qualcomm/rmnet/rmnet_map.h index 2aea153f4247..1a399bfa07d2 100644 --- a/drivers/net/ethernet/qualcomm/rmnet/rmnet_map.h +++ b/drivers/net/ethernet/qualcomm/rmnet/rmnet_map.h @@ -1,5 +1,5 @@ /* SPDX-License-Identifier: GPL-2.0-only */ -/* Copyright (c) 2013-2018, The Linux Foundation. All rights reserved. +/* Copyright (c) 2013-2018, 2021, The Linux Foundation. All rights reserved. */ #ifndef _RMNET_MAP_H_ @@ -48,5 +48,6 @@ void rmnet_map_command(struct sk_buff *skb, struct rmnet_port *port); int rmnet_map_checksum_downlink_packet(struct sk_buff *skb, u16 len); void rmnet_map_checksum_uplink_packet(struct sk_buff *skb, struct net_device *orig_dev); +int rmnet_map_process_next_hdr_packet(struct sk_buff *skb, u16 len); #endif /* _RMNET_MAP_H_ */ diff --git a/drivers/net/ethernet/qualcomm/rmnet/rmnet_map_data.c b/drivers/net/ethernet/qualcomm/rmnet/rmnet_map_data.c index 0ac2ff828320..5c018bd64689 100644 --- a/drivers/net/ethernet/qualcomm/rmnet/rmnet_map_data.c +++ b/drivers/net/ethernet/qualcomm/rmnet/rmnet_map_data.c @@ -1,5 +1,5 @@ // SPDX-License-Identifier: GPL-2.0-only -/* Copyright (c) 2013-2018, The Linux Foundation. All rights reserved. +/* Copyright (c) 2013-2018, 2021, The Linux Foundation. All rights reserved. * * RMNET Data MAP protocol */ @@ -8,6 +8,7 @@ #include #include #include +#include #include "rmnet_config.h" #include "rmnet_map.h" #include "rmnet_private.h" @@ -300,8 +301,11 @@ done: struct sk_buff *rmnet_map_deaggregate(struct sk_buff *skb, struct rmnet_port *port) { + struct rmnet_map_v5_csum_header *next_hdr = NULL; struct rmnet_map_header *maph; + void *data = skb->data; struct sk_buff *skbn; + u8 nexthdr_type; u32 packet_len; if (skb->len == 0) @@ -310,8 +314,18 @@ struct sk_buff *rmnet_map_deaggregate(struct sk_buff *skb, maph = (struct rmnet_map_header *)skb->data; packet_len = ntohs(maph->pkt_len) + sizeof(*maph); - if (port->data_format & RMNET_FLAGS_INGRESS_MAP_CKSUMV4) + if (port->data_format & RMNET_FLAGS_INGRESS_MAP_CKSUMV4) { packet_len += sizeof(struct rmnet_map_dl_csum_trailer); + } else if (port->data_format & RMNET_FLAGS_INGRESS_MAP_CKSUMV5) { + if (!(maph->flags & MAP_CMD_FLAG)) { + packet_len += sizeof(*next_hdr); + if (maph->flags & MAP_NEXT_HEADER_FLAG) + next_hdr = data + sizeof(*maph); + else + /* Mapv5 data pkt without csum hdr is invalid */ + return NULL; + } + } if (((int)skb->len - (int)packet_len) < 0) return NULL; @@ -320,6 +334,13 @@ struct sk_buff *rmnet_map_deaggregate(struct sk_buff *skb, if (!maph->pkt_len) return NULL; + if (next_hdr) { + nexthdr_type = u8_get_bits(next_hdr->header_info, + MAPV5_HDRINFO_HDR_TYPE_FMASK); + if (nexthdr_type != RMNET_MAP_HEADER_TYPE_CSUM_OFFLOAD) + return NULL; + } + skbn = alloc_skb(packet_len + RMNET_MAP_DEAGGR_SPACING, GFP_ATOMIC); if (!skbn) return NULL; @@ -414,3 +435,35 @@ sw_csum: priv->stats.csum_sw++; } + +/* Process a MAPv5 packet header */ +int rmnet_map_process_next_hdr_packet(struct sk_buff *skb, + u16 len) +{ + struct rmnet_priv *priv = netdev_priv(skb->dev); + struct rmnet_map_v5_csum_header *next_hdr; + u8 nexthdr_type; + + next_hdr = (struct rmnet_map_v5_csum_header *)(skb->data + + sizeof(struct rmnet_map_header)); + + nexthdr_type = u8_get_bits(next_hdr->header_info, + MAPV5_HDRINFO_HDR_TYPE_FMASK); + + if (nexthdr_type != RMNET_MAP_HEADER_TYPE_CSUM_OFFLOAD) + return -EINVAL; + + if (unlikely(!(skb->dev->features & NETIF_F_RXCSUM))) { + priv->stats.csum_sw++; + } else if (next_hdr->csum_info & MAPV5_CSUMINFO_VALID_FLAG) { + priv->stats.csum_ok++; + skb->ip_summed = CHECKSUM_UNNECESSARY; + } else { + priv->stats.csum_valid_unset++; + } + + /* Pull csum v5 header */ + skb_pull(skb, sizeof(*next_hdr)); + + return 0; +} diff --git a/include/linux/if_rmnet.h b/include/linux/if_rmnet.h index 4efb537f57f3..be17610a981e 100644 --- a/include/linux/if_rmnet.h +++ b/include/linux/if_rmnet.h @@ -1,5 +1,5 @@ /* SPDX-License-Identifier: GPL-2.0-only - * Copyright (c) 2013-2019, The Linux Foundation. All rights reserved. + * Copyright (c) 2013-2019, 2021 The Linux Foundation. All rights reserved. */ #ifndef _LINUX_IF_RMNET_H_ @@ -12,10 +12,12 @@ struct rmnet_map_header { } __aligned(1); /* rmnet_map_header flags field: - * PAD_LEN: number of pad bytes following packet data - * CMD: 1 = packet contains a MAP command; 0 = packet contains data + * PAD_LEN: number of pad bytes following packet data + * CMD: 1 = packet contains a MAP command; 0 = packet contains data + * NEXT_HEADER: 1 = packet contains V5 CSUM header 0 = no V5 CSUM header */ #define MAP_PAD_LEN_MASK GENMASK(5, 0) +#define MAP_NEXT_HEADER_FLAG BIT(6) #define MAP_CMD_FLAG BIT(7) struct rmnet_map_dl_csum_trailer { @@ -45,4 +47,26 @@ struct rmnet_map_ul_csum_header { #define MAP_CSUM_UL_UDP_FLAG BIT(14) #define MAP_CSUM_UL_ENABLED_FLAG BIT(15) +/* MAP CSUM headers */ +struct rmnet_map_v5_csum_header { + u8 header_info; + u8 csum_info; + __be16 reserved; +} __aligned(1); + +/* v5 header_info field + * NEXT_HEADER: represents whether there is any next header + * HEADER_TYPE: represents the type of this header + * + * csum_info field + * CSUM_VALID_OR_REQ: + * 1 = for UL, checksum computation is requested. + * 1 = for DL, validated the checksum and has found it valid + */ + +#define MAPV5_HDRINFO_NXT_HDR_FLAG BIT(0) +#define MAPV5_HDRINFO_HDR_TYPE_FMASK GENMASK(7, 1) +#define MAPV5_CSUMINFO_VALID_FLAG BIT(7) + +#define RMNET_MAP_HEADER_TYPE_CSUM_OFFLOAD 2 #endif /* !(_LINUX_IF_RMNET_H_) */ diff --git a/include/uapi/linux/if_link.h b/include/uapi/linux/if_link.h index cd5b382a4138..1f753dcd85e1 100644 --- a/include/uapi/linux/if_link.h +++ b/include/uapi/linux/if_link.h @@ -1236,6 +1236,7 @@ enum { #define RMNET_FLAGS_INGRESS_MAP_COMMANDS (1U << 1) #define RMNET_FLAGS_INGRESS_MAP_CKSUMV4 (1U << 2) #define RMNET_FLAGS_EGRESS_MAP_CKSUMV4 (1U << 3) +#define RMNET_FLAGS_INGRESS_MAP_CKSUMV5 (1U << 4) enum { IFLA_RMNET_UNSPEC, -- cgit v1.2.3 From b6e5d27e32ef6089d316ce7e1ecaf595584d4b84 Mon Sep 17 00:00:00 2001 From: Sharath Chandra Vurukala Date: Wed, 2 Jun 2021 00:58:36 +0530 Subject: net: ethernet: rmnet: Add support for MAPv5 egress packets Adding support for MAPv5 egress packets. This involves adding the MAPv5 header and setting the csum_valid_required in the checksum header to request HW compute the checksum. Corresponding stats are incremented based on whether the checksum is computed in software or HW. New stat has been added which represents the count of packets whose checksum is calculated by the HW. Signed-off-by: Sharath Chandra Vurukala Signed-off-by: David S. Miller --- drivers/net/ethernet/qualcomm/rmnet/rmnet_config.h | 4 +- .../net/ethernet/qualcomm/rmnet/rmnet_handlers.c | 23 +++--- drivers/net/ethernet/qualcomm/rmnet/rmnet_map.h | 8 +- .../net/ethernet/qualcomm/rmnet/rmnet_map_data.c | 92 ++++++++++++++++++++-- drivers/net/ethernet/qualcomm/rmnet/rmnet_vnd.c | 1 + include/uapi/linux/if_link.h | 1 + 6 files changed, 111 insertions(+), 18 deletions(-) (limited to 'include/uapi') diff --git a/drivers/net/ethernet/qualcomm/rmnet/rmnet_config.h b/drivers/net/ethernet/qualcomm/rmnet/rmnet_config.h index 8d8d4690a074..8e64ca98068d 100644 --- a/drivers/net/ethernet/qualcomm/rmnet/rmnet_config.h +++ b/drivers/net/ethernet/qualcomm/rmnet/rmnet_config.h @@ -1,5 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0-only */ -/* Copyright (c) 2013-2014, 2016-2018 The Linux Foundation. All rights reserved. +/* Copyright (c) 2013-2014, 2016-2018, 2021 The Linux Foundation. + * All rights reserved. * * RMNET Data configuration engine */ @@ -56,6 +57,7 @@ struct rmnet_priv_stats { u64 csum_fragmented_pkt; u64 csum_skipped; u64 csum_sw; + u64 csum_hw; }; struct rmnet_priv { diff --git a/drivers/net/ethernet/qualcomm/rmnet/rmnet_handlers.c b/drivers/net/ethernet/qualcomm/rmnet/rmnet_handlers.c index 706a225075a3..2504d0363b6b 100644 --- a/drivers/net/ethernet/qualcomm/rmnet/rmnet_handlers.c +++ b/drivers/net/ethernet/qualcomm/rmnet/rmnet_handlers.c @@ -133,7 +133,7 @@ static int rmnet_map_egress_handler(struct sk_buff *skb, struct rmnet_port *port, u8 mux_id, struct net_device *orig_dev) { - int required_headroom, additional_header_len; + int required_headroom, additional_header_len, csum_type = 0; struct rmnet_map_header *map_header; additional_header_len = 0; @@ -141,18 +141,23 @@ static int rmnet_map_egress_handler(struct sk_buff *skb, if (port->data_format & RMNET_FLAGS_EGRESS_MAP_CKSUMV4) { additional_header_len = sizeof(struct rmnet_map_ul_csum_header); - required_headroom += additional_header_len; + csum_type = RMNET_FLAGS_EGRESS_MAP_CKSUMV4; + } else if (port->data_format & RMNET_FLAGS_EGRESS_MAP_CKSUMV5) { + additional_header_len = sizeof(struct rmnet_map_v5_csum_header); + csum_type = RMNET_FLAGS_EGRESS_MAP_CKSUMV5; } - if (skb_headroom(skb) < required_headroom) { - if (pskb_expand_head(skb, required_headroom, 0, GFP_ATOMIC)) - return -ENOMEM; - } + required_headroom += additional_header_len; + + if (skb_cow_head(skb, required_headroom) < 0) + return -ENOMEM; - if (port->data_format & RMNET_FLAGS_EGRESS_MAP_CKSUMV4) - rmnet_map_checksum_uplink_packet(skb, orig_dev); + if (csum_type) + rmnet_map_checksum_uplink_packet(skb, port, orig_dev, + csum_type); - map_header = rmnet_map_add_map_header(skb, additional_header_len, 0); + map_header = rmnet_map_add_map_header(skb, additional_header_len, + port, 0); if (!map_header) return -ENOMEM; diff --git a/drivers/net/ethernet/qualcomm/rmnet/rmnet_map.h b/drivers/net/ethernet/qualcomm/rmnet/rmnet_map.h index 1a399bfa07d2..e5a0b38f7dbe 100644 --- a/drivers/net/ethernet/qualcomm/rmnet/rmnet_map.h +++ b/drivers/net/ethernet/qualcomm/rmnet/rmnet_map.h @@ -43,11 +43,15 @@ enum rmnet_map_commands { struct sk_buff *rmnet_map_deaggregate(struct sk_buff *skb, struct rmnet_port *port); struct rmnet_map_header *rmnet_map_add_map_header(struct sk_buff *skb, - int hdrlen, int pad); + int hdrlen, + struct rmnet_port *port, + int pad); void rmnet_map_command(struct sk_buff *skb, struct rmnet_port *port); int rmnet_map_checksum_downlink_packet(struct sk_buff *skb, u16 len); void rmnet_map_checksum_uplink_packet(struct sk_buff *skb, - struct net_device *orig_dev); + struct rmnet_port *port, + struct net_device *orig_dev, + int csum_type); int rmnet_map_process_next_hdr_packet(struct sk_buff *skb, u16 len); #endif /* _RMNET_MAP_H_ */ diff --git a/drivers/net/ethernet/qualcomm/rmnet/rmnet_map_data.c b/drivers/net/ethernet/qualcomm/rmnet/rmnet_map_data.c index 5c018bd64689..6492ec5bdec4 100644 --- a/drivers/net/ethernet/qualcomm/rmnet/rmnet_map_data.c +++ b/drivers/net/ethernet/qualcomm/rmnet/rmnet_map_data.c @@ -251,12 +251,69 @@ rmnet_map_ipv6_ul_csum_header(void *ip6hdr, } #endif +static void rmnet_map_v5_checksum_uplink_packet(struct sk_buff *skb, + struct rmnet_port *port, + struct net_device *orig_dev) +{ + struct rmnet_priv *priv = netdev_priv(orig_dev); + struct rmnet_map_v5_csum_header *ul_header; + + ul_header = skb_push(skb, sizeof(*ul_header)); + memset(ul_header, 0, sizeof(*ul_header)); + ul_header->header_info = u8_encode_bits(RMNET_MAP_HEADER_TYPE_CSUM_OFFLOAD, + MAPV5_HDRINFO_HDR_TYPE_FMASK); + + if (skb->ip_summed == CHECKSUM_PARTIAL) { + void *iph = ip_hdr(skb); + __sum16 *check; + void *trans; + u8 proto; + + if (skb->protocol != htons(ETH_P_IP) && + skb->protocol != htons(ETH_P_IPV6)) { + priv->stats.csum_err_invalid_ip_version++; + goto sw_csum; + } + + if (skb->protocol == htons(ETH_P_IP)) { + u16 ip_len = ((struct iphdr *)iph)->ihl * 4; + + proto = ((struct iphdr *)iph)->protocol; + trans = iph + ip_len; + } else if (skb->protocol == htons(ETH_P_IPV6)) { +#if IS_ENABLED(CONFIG_IPV6) + u16 ip_len = sizeof(struct ipv6hdr); + + proto = ((struct ipv6hdr *)iph)->nexthdr; + trans = iph + ip_len; +#else + priv->stats.csum_err_invalid_ip_version++; + goto sw_csum; +#endif /* CONFIG_IPV6 */ + } + + check = rmnet_map_get_csum_field(proto, trans); + if (check) { + skb->ip_summed = CHECKSUM_NONE; + /* Ask for checksum offloading */ + ul_header->csum_info |= MAPV5_CSUMINFO_VALID_FLAG; + priv->stats.csum_hw++; + return; + } + } + +sw_csum: + priv->stats.csum_sw++; +} + /* Adds MAP header to front of skb->data * Padding is calculated and set appropriately in MAP header. Mux ID is * initialized to 0. */ struct rmnet_map_header *rmnet_map_add_map_header(struct sk_buff *skb, - int hdrlen, int pad) + int hdrlen, + struct rmnet_port *port, + int pad) { struct rmnet_map_header *map_header; u32 padding, map_datalen; @@ -267,6 +324,10 @@ struct rmnet_map_header *rmnet_map_add_map_header(struct sk_buff *skb, skb_push(skb, sizeof(struct rmnet_map_header)); memset(map_header, 0, sizeof(struct rmnet_map_header)); + /* Set next_hdr bit for csum offload packets */ + if (port->data_format & RMNET_FLAGS_EGRESS_MAP_CKSUMV5) + map_header->flags |= MAP_NEXT_HEADER_FLAG; + if (pad == RMNET_MAP_NO_PAD_BYTES) { map_header->pkt_len = htons(map_datalen); return map_header; @@ -393,11 +454,8 @@ int rmnet_map_checksum_downlink_packet(struct sk_buff *skb, u16 len) return 0; } -/* Generates UL checksum meta info header for IPv4 and IPv6 over TCP and UDP - * packets that are supported for UL checksum offload. - */ -void rmnet_map_checksum_uplink_packet(struct sk_buff *skb, - struct net_device *orig_dev) +static void rmnet_map_v4_checksum_uplink_packet(struct sk_buff *skb, + struct net_device *orig_dev) { struct rmnet_priv *priv = netdev_priv(orig_dev); struct rmnet_map_ul_csum_header *ul_header; @@ -416,10 +474,12 @@ void rmnet_map_checksum_uplink_packet(struct sk_buff *skb, if (skb->protocol == htons(ETH_P_IP)) { rmnet_map_ipv4_ul_csum_header(iphdr, ul_header, skb); + priv->stats.csum_hw++; return; } else if (skb->protocol == htons(ETH_P_IPV6)) { #if IS_ENABLED(CONFIG_IPV6) rmnet_map_ipv6_ul_csum_header(iphdr, ul_header, skb); + priv->stats.csum_hw++; return; #else priv->stats.csum_err_invalid_ip_version++; @@ -436,6 +496,26 @@ sw_csum: priv->stats.csum_sw++; } +/* Generates UL checksum meta info header for IPv4 and IPv6 over TCP and UDP + * packets that are supported for UL checksum offload. + */ +void rmnet_map_checksum_uplink_packet(struct sk_buff *skb, + struct rmnet_port *port, + struct net_device *orig_dev, + int csum_type) +{ + switch (csum_type) { + case RMNET_FLAGS_EGRESS_MAP_CKSUMV4: + rmnet_map_v4_checksum_uplink_packet(skb, orig_dev); + break; + case RMNET_FLAGS_EGRESS_MAP_CKSUMV5: + rmnet_map_v5_checksum_uplink_packet(skb, port, orig_dev); + break; + default: + break; + } +} + /* Process a MAPv5 packet header */ int rmnet_map_process_next_hdr_packet(struct sk_buff *skb, u16 len) diff --git a/drivers/net/ethernet/qualcomm/rmnet/rmnet_vnd.c b/drivers/net/ethernet/qualcomm/rmnet/rmnet_vnd.c index 41fbd2ceeede..fe13017e9a41 100644 --- a/drivers/net/ethernet/qualcomm/rmnet/rmnet_vnd.c +++ b/drivers/net/ethernet/qualcomm/rmnet/rmnet_vnd.c @@ -174,6 +174,7 @@ static const char rmnet_gstrings_stats[][ETH_GSTRING_LEN] = { "Checksum skipped on ip fragment", "Checksum skipped", "Checksum computed in software", + "Checksum computed in hardware", }; static void rmnet_get_strings(struct net_device *dev, u32 stringset, u8 *buf) diff --git a/include/uapi/linux/if_link.h b/include/uapi/linux/if_link.h index 1f753dcd85e1..a5a7f0e64865 100644 --- a/include/uapi/linux/if_link.h +++ b/include/uapi/linux/if_link.h @@ -1237,6 +1237,7 @@ enum { #define RMNET_FLAGS_INGRESS_MAP_CKSUMV4 (1U << 2) #define RMNET_FLAGS_EGRESS_MAP_CKSUMV4 (1U << 3) #define RMNET_FLAGS_INGRESS_MAP_CKSUMV5 (1U << 4) +#define RMNET_FLAGS_EGRESS_MAP_CKSUMV5 (1U << 5) enum { IFLA_RMNET_UNSPEC, -- cgit v1.2.3 From d170ebb00472268410dce80ae4834c98e79315da Mon Sep 17 00:00:00 2001 From: Hans Verkuil Date: Fri, 21 May 2021 10:45:44 +0200 Subject: media: uapi/linux/cec-funcs.h: set delay to 1 if unnused If the audio_out_delay value is unused, then set it to 1, not 0. The value 0 is reserved, and 1 is a much safer value since it translates to a delay of (1 - 1) * 2 = 0 ms. Signed-off-by: Hans Verkuil Signed-off-by: Mauro Carvalho Chehab --- include/uapi/linux/cec-funcs.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/uapi') diff --git a/include/uapi/linux/cec-funcs.h b/include/uapi/linux/cec-funcs.h index 37590027b604..c3baaea0b8ef 100644 --- a/include/uapi/linux/cec-funcs.h +++ b/include/uapi/linux/cec-funcs.h @@ -1665,7 +1665,7 @@ static inline void cec_ops_report_current_latency(const struct cec_msg *msg, if (*audio_out_compensated == 3 && msg->len >= 7) *audio_out_delay = msg->msg[6]; else - *audio_out_delay = 0; + *audio_out_delay = 1; } static inline void cec_msg_request_current_latency(struct cec_msg *msg, -- cgit v1.2.3 From ce67eaca95f8ab5c6aae41a10adfe9a6e8efa58c Mon Sep 17 00:00:00 2001 From: Joe Richey Date: Fri, 21 May 2021 10:58:46 +0200 Subject: media: vicodec: Use _BITUL() macro in UAPI headers Replace BIT() in v4l2's UPAI header with _BITUL(). BIT() is not defined in the UAPI headers and its usage may cause userspace build errors. Fixes: 206bc0f6fb94 ("media: vicodec: mark the stateless FWHT API as stable") Signed-off-by: Joe Richey Signed-off-by: Hans Verkuil Signed-off-by: Mauro Carvalho Chehab --- include/uapi/linux/v4l2-controls.h | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) (limited to 'include/uapi') diff --git a/include/uapi/linux/v4l2-controls.h b/include/uapi/linux/v4l2-controls.h index f96bea19c991..fdf97a6d7d18 100644 --- a/include/uapi/linux/v4l2-controls.h +++ b/include/uapi/linux/v4l2-controls.h @@ -50,6 +50,7 @@ #ifndef __LINUX_V4L2_CONTROLS_H #define __LINUX_V4L2_CONTROLS_H +#include #include /* Control classes */ @@ -1602,30 +1603,30 @@ struct v4l2_ctrl_h264_decode_params { #define V4L2_FWHT_VERSION 3 /* Set if this is an interlaced format */ -#define V4L2_FWHT_FL_IS_INTERLACED BIT(0) +#define V4L2_FWHT_FL_IS_INTERLACED _BITUL(0) /* Set if this is a bottom-first (NTSC) interlaced format */ -#define V4L2_FWHT_FL_IS_BOTTOM_FIRST BIT(1) +#define V4L2_FWHT_FL_IS_BOTTOM_FIRST _BITUL(1) /* Set if each 'frame' contains just one field */ -#define V4L2_FWHT_FL_IS_ALTERNATE BIT(2) +#define V4L2_FWHT_FL_IS_ALTERNATE _BITUL(2) /* * If V4L2_FWHT_FL_IS_ALTERNATE was set, then this is set if this * 'frame' is the bottom field, else it is the top field. */ -#define V4L2_FWHT_FL_IS_BOTTOM_FIELD BIT(3) +#define V4L2_FWHT_FL_IS_BOTTOM_FIELD _BITUL(3) /* Set if the Y' plane is uncompressed */ -#define V4L2_FWHT_FL_LUMA_IS_UNCOMPRESSED BIT(4) +#define V4L2_FWHT_FL_LUMA_IS_UNCOMPRESSED _BITUL(4) /* Set if the Cb plane is uncompressed */ -#define V4L2_FWHT_FL_CB_IS_UNCOMPRESSED BIT(5) +#define V4L2_FWHT_FL_CB_IS_UNCOMPRESSED _BITUL(5) /* Set if the Cr plane is uncompressed */ -#define V4L2_FWHT_FL_CR_IS_UNCOMPRESSED BIT(6) +#define V4L2_FWHT_FL_CR_IS_UNCOMPRESSED _BITUL(6) /* Set if the chroma plane is full height, if cleared it is half height */ -#define V4L2_FWHT_FL_CHROMA_FULL_HEIGHT BIT(7) +#define V4L2_FWHT_FL_CHROMA_FULL_HEIGHT _BITUL(7) /* Set if the chroma plane is full width, if cleared it is half width */ -#define V4L2_FWHT_FL_CHROMA_FULL_WIDTH BIT(8) +#define V4L2_FWHT_FL_CHROMA_FULL_WIDTH _BITUL(8) /* Set if the alpha plane is uncompressed */ -#define V4L2_FWHT_FL_ALPHA_IS_UNCOMPRESSED BIT(9) +#define V4L2_FWHT_FL_ALPHA_IS_UNCOMPRESSED _BITUL(9) /* Set if this is an I Frame */ -#define V4L2_FWHT_FL_I_FRAME BIT(10) +#define V4L2_FWHT_FL_I_FRAME _BITUL(10) /* A 4-values flag - the number of components - 1 */ #define V4L2_FWHT_FL_COMPONENTS_NUM_MSK GENMASK(18, 16) -- cgit v1.2.3 From 4677efc486e1872f62d4632c50f7183f82296fa6 Mon Sep 17 00:00:00 2001 From: Dmytro Linkin Date: Wed, 2 Jun 2021 15:17:19 +0300 Subject: devlink: Introduce rate object Allow registering rate object for devlink ports with dedicated devlink_rate_leaf_{create|destroy}() API. Implement new netlink DEVLINK_CMD_RATE_GET command that is used to retrieve rate object info. Add new DEVLINK_CMD_RATE_{NEW|DEL} commands that are used for notifications when creating/deleting leaf rate object. Rate API is intended to be used for rate limiting of individual devlink ports (leafs) and their aggregates (nodes). Example: $ devlink port show pci/0000:03:00.0/0 pci/0000:03:00.0/1 $ devlink port function rate show pci/0000:03:00.0/0: type leaf pci/0000:03:00.0/1: type leaf Co-developed-by: Vlad Buslov Signed-off-by: Vlad Buslov Signed-off-by: Dmytro Linkin Reviewed-by: Jiri Pirko Signed-off-by: David S. Miller --- include/net/devlink.h | 14 +++ include/uapi/linux/devlink.h | 11 +++ net/core/devlink.c | 229 ++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 253 insertions(+), 1 deletion(-) (limited to 'include/uapi') diff --git a/include/net/devlink.h b/include/net/devlink.h index 7c984cadfec4..2f5954d96c3e 100644 --- a/include/net/devlink.h +++ b/include/net/devlink.h @@ -34,6 +34,7 @@ struct devlink_ops; struct devlink { struct list_head list; struct list_head port_list; + struct list_head rate_list; struct list_head sb_list; struct list_head dpipe_table_list; struct list_head resource_list; @@ -133,6 +134,15 @@ struct devlink_port_attrs { }; }; +struct devlink_rate { + struct list_head list; + enum devlink_rate_type type; + struct devlink *devlink; + void *priv; + + struct devlink_port *devlink_port; +}; + struct devlink_port { struct list_head list; struct list_head param_list; @@ -152,6 +162,8 @@ struct devlink_port { struct delayed_work type_warn_dw; struct list_head reporter_list; struct mutex reporters_lock; /* Protects reporter_list */ + + struct devlink_rate *devlink_rate; }; struct devlink_port_new_attrs { @@ -1512,6 +1524,8 @@ void devlink_port_attrs_pci_vf_set(struct devlink_port *devlink_port, u32 contro void devlink_port_attrs_pci_sf_set(struct devlink_port *devlink_port, u32 controller, u16 pf, u32 sf, bool external); +int devlink_rate_leaf_create(struct devlink_port *port, void *priv); +void devlink_rate_leaf_destroy(struct devlink_port *devlink_port); int devlink_sb_register(struct devlink *devlink, unsigned int sb_index, u32 size, u16 ingress_pools_count, u16 egress_pools_count, u16 ingress_tc_count, diff --git a/include/uapi/linux/devlink.h b/include/uapi/linux/devlink.h index f6008b2fa60f..0c27b45c47db 100644 --- a/include/uapi/linux/devlink.h +++ b/include/uapi/linux/devlink.h @@ -126,6 +126,11 @@ enum devlink_command { DEVLINK_CMD_HEALTH_REPORTER_TEST, + DEVLINK_CMD_RATE_GET, /* can dump */ + DEVLINK_CMD_RATE_SET, + DEVLINK_CMD_RATE_NEW, + DEVLINK_CMD_RATE_DEL, + /* add new commands above here */ __DEVLINK_CMD_MAX, DEVLINK_CMD_MAX = __DEVLINK_CMD_MAX - 1 @@ -206,6 +211,10 @@ enum devlink_port_flavour { */ }; +enum devlink_rate_type { + DEVLINK_RATE_TYPE_LEAF, +}; + enum devlink_param_cmode { DEVLINK_PARAM_CMODE_RUNTIME, DEVLINK_PARAM_CMODE_DRIVERINIT, @@ -534,6 +543,8 @@ enum devlink_attr { DEVLINK_ATTR_RELOAD_ACTION_STATS, /* nested */ DEVLINK_ATTR_PORT_PCI_SF_NUMBER, /* u32 */ + + DEVLINK_ATTR_RATE_TYPE, /* u16 */ /* add new attributes above here, update the policy in devlink.c */ __DEVLINK_ATTR_MAX, diff --git a/net/core/devlink.c b/net/core/devlink.c index 69681f19388e..3b785f51156f 100644 --- a/net/core/devlink.c +++ b/net/core/devlink.c @@ -190,6 +190,25 @@ static struct devlink_port *devlink_port_get_from_info(struct devlink *devlink, return devlink_port_get_from_attrs(devlink, info->attrs); } +static inline bool +devlink_rate_is_leaf(struct devlink_rate *devlink_rate) +{ + return devlink_rate->type == DEVLINK_RATE_TYPE_LEAF; +} + +static struct devlink_rate * +devlink_rate_leaf_get_from_info(struct devlink *devlink, struct genl_info *info) +{ + struct devlink_rate *devlink_rate; + struct devlink_port *devlink_port; + + devlink_port = devlink_port_get_from_attrs(devlink, info->attrs); + if (IS_ERR(devlink_port)) + return ERR_CAST(devlink_port); + devlink_rate = devlink_port->devlink_rate; + return devlink_rate ?: ERR_PTR(-ENODEV); +} + struct devlink_sb { struct list_head list; unsigned int index; @@ -408,12 +427,13 @@ devlink_region_snapshot_get_by_id(struct devlink_region *region, u32 id) #define DEVLINK_NL_FLAG_NEED_PORT BIT(0) #define DEVLINK_NL_FLAG_NEED_DEVLINK_OR_PORT BIT(1) +#define DEVLINK_NL_FLAG_NEED_RATE BIT(2) /* The per devlink instance lock is taken by default in the pre-doit * operation, yet several commands do not require this. The global * devlink lock is taken and protects from disruption by user-calls. */ -#define DEVLINK_NL_FLAG_NO_LOCK BIT(2) +#define DEVLINK_NL_FLAG_NO_LOCK BIT(3) static int devlink_nl_pre_doit(const struct genl_ops *ops, struct sk_buff *skb, struct genl_info *info) @@ -442,6 +462,15 @@ static int devlink_nl_pre_doit(const struct genl_ops *ops, devlink_port = devlink_port_get_from_info(devlink, info); if (!IS_ERR(devlink_port)) info->user_ptr[1] = devlink_port; + } else if (ops->internal_flags & DEVLINK_NL_FLAG_NEED_RATE) { + struct devlink_rate *devlink_rate; + + devlink_rate = devlink_rate_leaf_get_from_info(devlink, info); + if (IS_ERR(devlink_rate)) { + err = PTR_ERR(devlink_rate); + goto unlock; + } + info->user_ptr[1] = devlink_rate; } return 0; @@ -749,6 +778,39 @@ devlink_port_fn_hw_addr_fill(struct devlink *devlink, const struct devlink_ops * return 0; } +static int devlink_nl_rate_fill(struct sk_buff *msg, + struct devlink *devlink, + struct devlink_rate *devlink_rate, + enum devlink_command cmd, u32 portid, + u32 seq, int flags, + struct netlink_ext_ack *extack) +{ + void *hdr; + + hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, flags, cmd); + if (!hdr) + return -EMSGSIZE; + + if (devlink_nl_put_handle(msg, devlink)) + goto nla_put_failure; + + if (nla_put_u16(msg, DEVLINK_ATTR_RATE_TYPE, devlink_rate->type)) + goto nla_put_failure; + + if (devlink_rate_is_leaf(devlink_rate)) { + if (nla_put_u32(msg, DEVLINK_ATTR_PORT_INDEX, + devlink_rate->devlink_port->index)) + goto nla_put_failure; + } + + genlmsg_end(msg, hdr); + return 0; + +nla_put_failure: + genlmsg_cancel(msg, hdr); + return -EMSGSIZE; +} + static bool devlink_port_fn_state_valid(enum devlink_port_fn_state state) { @@ -920,6 +982,99 @@ static void devlink_port_notify(struct devlink_port *devlink_port, msg, 0, DEVLINK_MCGRP_CONFIG, GFP_KERNEL); } +static void devlink_rate_notify(struct devlink_rate *devlink_rate, + enum devlink_command cmd) +{ + struct devlink *devlink = devlink_rate->devlink; + struct sk_buff *msg; + int err; + + WARN_ON(cmd != DEVLINK_CMD_RATE_NEW && + cmd != DEVLINK_CMD_RATE_DEL); + + msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL); + if (!msg) + return; + + err = devlink_nl_rate_fill(msg, devlink, devlink_rate, + cmd, 0, 0, 0, NULL); + if (err) { + nlmsg_free(msg); + return; + } + + genlmsg_multicast_netns(&devlink_nl_family, devlink_net(devlink), + msg, 0, DEVLINK_MCGRP_CONFIG, GFP_KERNEL); +} + +static int devlink_nl_cmd_rate_get_dumpit(struct sk_buff *msg, + struct netlink_callback *cb) +{ + struct devlink_rate *devlink_rate; + struct devlink *devlink; + int start = cb->args[0]; + int idx = 0; + int err = 0; + + mutex_lock(&devlink_mutex); + list_for_each_entry(devlink, &devlink_list, list) { + if (!net_eq(devlink_net(devlink), sock_net(msg->sk))) + continue; + mutex_lock(&devlink->lock); + list_for_each_entry(devlink_rate, &devlink->rate_list, list) { + enum devlink_command cmd = DEVLINK_CMD_RATE_NEW; + u32 id = NETLINK_CB(cb->skb).portid; + + if (idx < start) { + idx++; + continue; + } + err = devlink_nl_rate_fill(msg, devlink, + devlink_rate, + cmd, id, + cb->nlh->nlmsg_seq, + NLM_F_MULTI, NULL); + if (err) { + mutex_unlock(&devlink->lock); + goto out; + } + idx++; + } + mutex_unlock(&devlink->lock); + } +out: + mutex_unlock(&devlink_mutex); + if (err != -EMSGSIZE) + return err; + + cb->args[0] = idx; + return msg->len; +} + +static int devlink_nl_cmd_rate_get_doit(struct sk_buff *skb, + struct genl_info *info) +{ + struct devlink_rate *devlink_rate = info->user_ptr[1]; + struct devlink *devlink = devlink_rate->devlink; + struct sk_buff *msg; + int err; + + msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL); + if (!msg) + return -ENOMEM; + + err = devlink_nl_rate_fill(msg, devlink, devlink_rate, + DEVLINK_CMD_RATE_NEW, + info->snd_portid, info->snd_seq, 0, + info->extack); + if (err) { + nlmsg_free(msg); + return err; + } + + return genlmsg_reply(msg, info); +} + static int devlink_nl_cmd_get_doit(struct sk_buff *skb, struct genl_info *info) { struct devlink *devlink = info->user_ptr[0]; @@ -7802,6 +7957,7 @@ static const struct nla_policy devlink_nl_policy[DEVLINK_ATTR_MAX + 1] = { [DEVLINK_ATTR_PORT_PCI_PF_NUMBER] = { .type = NLA_U16 }, [DEVLINK_ATTR_PORT_PCI_SF_NUMBER] = { .type = NLA_U32 }, [DEVLINK_ATTR_PORT_CONTROLLER_NUMBER] = { .type = NLA_U32 }, + [DEVLINK_ATTR_RATE_TYPE] = { .type = NLA_U16 }, }; static const struct genl_small_ops devlink_nl_ops[] = { @@ -7827,6 +7983,13 @@ static const struct genl_small_ops devlink_nl_ops[] = { .flags = GENL_ADMIN_PERM, .internal_flags = DEVLINK_NL_FLAG_NEED_PORT, }, + { + .cmd = DEVLINK_CMD_RATE_GET, + .doit = devlink_nl_cmd_rate_get_doit, + .dumpit = devlink_nl_cmd_rate_get_dumpit, + .internal_flags = DEVLINK_NL_FLAG_NEED_RATE, + /* can be retrieved by unprivileged users */ + }, { .cmd = DEVLINK_CMD_PORT_SPLIT, .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, @@ -8202,6 +8365,7 @@ struct devlink *devlink_alloc(const struct devlink_ops *ops, size_t priv_size) xa_init_flags(&devlink->snapshot_ids, XA_FLAGS_ALLOC); __devlink_net_set(devlink, &init_net); INIT_LIST_HEAD(&devlink->port_list); + INIT_LIST_HEAD(&devlink->rate_list); INIT_LIST_HEAD(&devlink->sb_list); INIT_LIST_HEAD_RCU(&devlink->dpipe_table_list); INIT_LIST_HEAD(&devlink->resource_list); @@ -8304,6 +8468,7 @@ void devlink_free(struct devlink *devlink) WARN_ON(!list_empty(&devlink->resource_list)); WARN_ON(!list_empty(&devlink->dpipe_table_list)); WARN_ON(!list_empty(&devlink->sb_list)); + WARN_ON(!list_empty(&devlink->rate_list)); WARN_ON(!list_empty(&devlink->port_list)); xa_destroy(&devlink->snapshot_ids); @@ -8620,6 +8785,68 @@ void devlink_port_attrs_pci_sf_set(struct devlink_port *devlink_port, u32 contro } EXPORT_SYMBOL_GPL(devlink_port_attrs_pci_sf_set); +/** + * devlink_rate_leaf_create - create devlink rate leaf + * + * @devlink_port: devlink port object to create rate object on + * @priv: driver private data + * + * Create devlink rate object of type leaf on provided @devlink_port. + * Throws call trace if @devlink_port already has a devlink rate object. + * + * Context: Takes and release devlink->lock . + * + * Return: -ENOMEM if failed to allocate rate object, 0 otherwise. + */ +int +devlink_rate_leaf_create(struct devlink_port *devlink_port, void *priv) +{ + struct devlink *devlink = devlink_port->devlink; + struct devlink_rate *devlink_rate; + + devlink_rate = kzalloc(sizeof(*devlink_rate), GFP_KERNEL); + if (!devlink_rate) + return -ENOMEM; + + mutex_lock(&devlink->lock); + WARN_ON(devlink_port->devlink_rate); + devlink_rate->type = DEVLINK_RATE_TYPE_LEAF; + devlink_rate->devlink = devlink; + devlink_rate->devlink_port = devlink_port; + devlink_rate->priv = priv; + list_add_tail(&devlink_rate->list, &devlink->rate_list); + devlink_port->devlink_rate = devlink_rate; + devlink_rate_notify(devlink_rate, DEVLINK_CMD_RATE_NEW); + mutex_unlock(&devlink->lock); + + return 0; +} +EXPORT_SYMBOL_GPL(devlink_rate_leaf_create); + +/** + * devlink_rate_leaf_destroy - destroy devlink rate leaf + * + * @devlink_port: devlink port linked to the rate object + * + * Context: Takes and release devlink->lock . + */ +void devlink_rate_leaf_destroy(struct devlink_port *devlink_port) +{ + struct devlink_rate *devlink_rate = devlink_port->devlink_rate; + struct devlink *devlink = devlink_port->devlink; + + if (!devlink_rate) + return; + + mutex_lock(&devlink->lock); + devlink_rate_notify(devlink_rate, DEVLINK_CMD_RATE_DEL); + list_del(&devlink_rate->list); + devlink_port->devlink_rate = NULL; + mutex_unlock(&devlink->lock); + kfree(devlink_rate); +} +EXPORT_SYMBOL_GPL(devlink_rate_leaf_destroy); + static int __devlink_port_phys_port_name_get(struct devlink_port *devlink_port, char *name, size_t len) { -- cgit v1.2.3 From 1897db2ec3109eb1dd07b357c95c5e03d54e41b9 Mon Sep 17 00:00:00 2001 From: Dmytro Linkin Date: Wed, 2 Jun 2021 15:17:22 +0300 Subject: devlink: Allow setting tx rate for devlink rate leaf objects Implement support for DEVLINK_CMD_RATE_SET command with new attributes DEVLINK_ATTR_RATE_TX_{SHARE|MAX} that are used to set devlink rate shared/max tx rate values. Extend devlink ops with new callbacks rate_leaf_tx_{share|max}_set() to allow supporting drivers to implement rate control through devlink. New attributes are optional. Driver implementations are allowed to support either or both of them. Shared rate example: $ devlink port function rate set netdevsim/netdevsim10/0 tx_share 10mbit $ devlink port function rate show netdevsim/netdevsim10/0 netdevsim/netdevsim10/0: type leaf tx_share 10mbit Max rate example: $ devlink port function rate set netdevsim/netdevsim10/0 tx_max 100mbit $ devlink port function rate show netdevsim/netdevsim10/0 netdevsim/netdevsim10/0: type leaf tx_max 100mbit Co-developed-by: Vlad Buslov Signed-off-by: Vlad Buslov Signed-off-by: Dmytro Linkin Reviewed-by: Jiri Pirko Signed-off-by: David S. Miller --- include/net/devlink.h | 10 ++++++ include/uapi/linux/devlink.h | 2 ++ net/core/devlink.c | 86 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 98 insertions(+) (limited to 'include/uapi') diff --git a/include/net/devlink.h b/include/net/devlink.h index 2f5954d96c3e..46d553545f83 100644 --- a/include/net/devlink.h +++ b/include/net/devlink.h @@ -139,6 +139,8 @@ struct devlink_rate { enum devlink_rate_type type; struct devlink *devlink; void *priv; + u64 tx_share; + u64 tx_max; struct devlink_port *devlink_port; }; @@ -1465,6 +1467,14 @@ struct devlink_ops { struct devlink_port *port, enum devlink_port_fn_state state, struct netlink_ext_ack *extack); + + /** + * Rate control callbacks. + */ + int (*rate_leaf_tx_share_set)(struct devlink_rate *devlink_rate, void *priv, + u64 tx_share, struct netlink_ext_ack *extack); + int (*rate_leaf_tx_max_set)(struct devlink_rate *devlink_rate, void *priv, + u64 tx_max, struct netlink_ext_ack *extack); }; static inline void *devlink_priv(struct devlink *devlink) diff --git a/include/uapi/linux/devlink.h b/include/uapi/linux/devlink.h index 0c27b45c47db..ae94cd2a1078 100644 --- a/include/uapi/linux/devlink.h +++ b/include/uapi/linux/devlink.h @@ -545,6 +545,8 @@ enum devlink_attr { DEVLINK_ATTR_PORT_PCI_SF_NUMBER, /* u32 */ DEVLINK_ATTR_RATE_TYPE, /* u16 */ + DEVLINK_ATTR_RATE_TX_SHARE, /* u64 */ + DEVLINK_ATTR_RATE_TX_MAX, /* u64 */ /* add new attributes above here, update the policy in devlink.c */ __DEVLINK_ATTR_MAX, diff --git a/net/core/devlink.c b/net/core/devlink.c index 3b785f51156f..37839fd5ca73 100644 --- a/net/core/devlink.c +++ b/net/core/devlink.c @@ -803,6 +803,14 @@ static int devlink_nl_rate_fill(struct sk_buff *msg, goto nla_put_failure; } + if (nla_put_u64_64bit(msg, DEVLINK_ATTR_RATE_TX_SHARE, + devlink_rate->tx_share, DEVLINK_ATTR_PAD)) + goto nla_put_failure; + + if (nla_put_u64_64bit(msg, DEVLINK_ATTR_RATE_TX_MAX, + devlink_rate->tx_max, DEVLINK_ATTR_PAD)) + goto nla_put_failure; + genlmsg_end(msg, hdr); return 0; @@ -1495,6 +1503,76 @@ static int devlink_nl_cmd_port_del_doit(struct sk_buff *skb, return devlink->ops->port_del(devlink, port_index, extack); } +static int devlink_nl_rate_set(struct devlink_rate *devlink_rate, + const struct devlink_ops *ops, + struct genl_info *info) +{ + struct nlattr **attrs = info->attrs; + u64 rate; + int err; + + if (attrs[DEVLINK_ATTR_RATE_TX_SHARE]) { + rate = nla_get_u64(attrs[DEVLINK_ATTR_RATE_TX_SHARE]); + err = ops->rate_leaf_tx_share_set(devlink_rate, devlink_rate->priv, + rate, info->extack); + if (err) + return err; + devlink_rate->tx_share = rate; + } + + if (attrs[DEVLINK_ATTR_RATE_TX_MAX]) { + rate = nla_get_u64(attrs[DEVLINK_ATTR_RATE_TX_MAX]); + err = ops->rate_leaf_tx_max_set(devlink_rate, devlink_rate->priv, + rate, info->extack); + if (err) + return err; + devlink_rate->tx_max = rate; + } + + return 0; +} + +static bool devlink_rate_set_ops_supported(const struct devlink_ops *ops, + struct genl_info *info, + enum devlink_rate_type type) +{ + struct nlattr **attrs = info->attrs; + + if (type == DEVLINK_RATE_TYPE_LEAF) { + if (attrs[DEVLINK_ATTR_RATE_TX_SHARE] && !ops->rate_leaf_tx_share_set) { + NL_SET_ERR_MSG_MOD(info->extack, "TX share set isn't supported for the leafs"); + return false; + } + if (attrs[DEVLINK_ATTR_RATE_TX_MAX] && !ops->rate_leaf_tx_max_set) { + NL_SET_ERR_MSG_MOD(info->extack, "TX max set isn't supported for the leafs"); + return false; + } + } else { + WARN_ON("Unknown type of rate object"); + return false; + } + + return true; +} + +static int devlink_nl_cmd_rate_set_doit(struct sk_buff *skb, + struct genl_info *info) +{ + struct devlink_rate *devlink_rate = info->user_ptr[1]; + struct devlink *devlink = devlink_rate->devlink; + const struct devlink_ops *ops = devlink->ops; + int err; + + if (!ops || !devlink_rate_set_ops_supported(ops, info, devlink_rate->type)) + return -EOPNOTSUPP; + + err = devlink_nl_rate_set(devlink_rate, ops, info); + + if (!err) + devlink_rate_notify(devlink_rate, DEVLINK_CMD_RATE_NEW); + return err; +} + static int devlink_nl_sb_fill(struct sk_buff *msg, struct devlink *devlink, struct devlink_sb *devlink_sb, enum devlink_command cmd, u32 portid, @@ -7958,6 +8036,8 @@ static const struct nla_policy devlink_nl_policy[DEVLINK_ATTR_MAX + 1] = { [DEVLINK_ATTR_PORT_PCI_SF_NUMBER] = { .type = NLA_U32 }, [DEVLINK_ATTR_PORT_CONTROLLER_NUMBER] = { .type = NLA_U32 }, [DEVLINK_ATTR_RATE_TYPE] = { .type = NLA_U16 }, + [DEVLINK_ATTR_RATE_TX_SHARE] = { .type = NLA_U64 }, + [DEVLINK_ATTR_RATE_TX_MAX] = { .type = NLA_U64 }, }; static const struct genl_small_ops devlink_nl_ops[] = { @@ -7990,6 +8070,12 @@ static const struct genl_small_ops devlink_nl_ops[] = { .internal_flags = DEVLINK_NL_FLAG_NEED_RATE, /* can be retrieved by unprivileged users */ }, + { + .cmd = DEVLINK_CMD_RATE_SET, + .doit = devlink_nl_cmd_rate_set_doit, + .flags = GENL_ADMIN_PERM, + .internal_flags = DEVLINK_NL_FLAG_NEED_RATE, + }, { .cmd = DEVLINK_CMD_PORT_SPLIT, .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, -- cgit v1.2.3 From a8ecb93ef03de4c59fb6289f99bc9616a852c917 Mon Sep 17 00:00:00 2001 From: Dmytro Linkin Date: Wed, 2 Jun 2021 15:17:25 +0300 Subject: devlink: Introduce rate nodes Implement support for DEVLINK_CMD_RATE_{NEW|DEL} commands that are used to create and delete devlink rate nodes. Add new attribute DEVLINK_ATTR_RATE_NODE_NAME that specify node name string. The node name is an alphanumeric identifier. No valid node name can be a devlink port index, eg. decimal number. Extend devlink ops with new callbacks rate_node_{new|del}() and rate_node_tx_{share|max}_set() to allow supporting drivers to implement ports rate grouping and setting tx rate of rate nodes through devlink. Expose devlink_rate_nodes_destroy() function to allow vendor driver do proper cleanup of internally allocated resources for the nodes if the driver goes down or due to any other reasons which requires nodes to be destroyed. Disallow moving device from switchdev to legacy mode if any node exists on that device. User must explicitly delete nodes before switching mode. Example: $ devlink port function rate add netdevsim/netdevsim10/group1 $ devlink port function rate set netdevsim/netdevsim10/group1 \ tx_share 10mbit tx_max 100mbit Add + set command can be combined: $ devlink port function rate add netdevsim/netdevsim10/group1 \ tx_share 10mbit tx_max 100mbit $ devlink port function rate show netdevsim/netdevsim10/group1 netdevsim/netdevsim10/group1: type node tx_share 10mbit tx_max 100mbit $ devlink port function rate del netdevsim/netdevsim10/group1 Co-developed-by: Vlad Buslov Signed-off-by: Vlad Buslov Signed-off-by: Dmytro Linkin Reviewed-by: Jiri Pirko Signed-off-by: David S. Miller --- include/net/devlink.h | 14 ++- include/uapi/linux/devlink.h | 3 + net/core/devlink.c | 238 +++++++++++++++++++++++++++++++++++++++++-- 3 files changed, 247 insertions(+), 8 deletions(-) (limited to 'include/uapi') diff --git a/include/net/devlink.h b/include/net/devlink.h index 46d553545f83..13162b579124 100644 --- a/include/net/devlink.h +++ b/include/net/devlink.h @@ -142,7 +142,10 @@ struct devlink_rate { u64 tx_share; u64 tx_max; - struct devlink_port *devlink_port; + union { + struct devlink_port *devlink_port; + char *name; + }; }; struct devlink_port { @@ -1475,6 +1478,14 @@ struct devlink_ops { u64 tx_share, struct netlink_ext_ack *extack); int (*rate_leaf_tx_max_set)(struct devlink_rate *devlink_rate, void *priv, u64 tx_max, struct netlink_ext_ack *extack); + int (*rate_node_tx_share_set)(struct devlink_rate *devlink_rate, void *priv, + u64 tx_share, struct netlink_ext_ack *extack); + int (*rate_node_tx_max_set)(struct devlink_rate *devlink_rate, void *priv, + u64 tx_max, struct netlink_ext_ack *extack); + int (*rate_node_new)(struct devlink_rate *rate_node, void **priv, + struct netlink_ext_ack *extack); + int (*rate_node_del)(struct devlink_rate *rate_node, void *priv, + struct netlink_ext_ack *extack); }; static inline void *devlink_priv(struct devlink *devlink) @@ -1536,6 +1547,7 @@ void devlink_port_attrs_pci_sf_set(struct devlink_port *devlink_port, bool external); int devlink_rate_leaf_create(struct devlink_port *port, void *priv); void devlink_rate_leaf_destroy(struct devlink_port *devlink_port); +void devlink_rate_nodes_destroy(struct devlink *devlink); int devlink_sb_register(struct devlink *devlink, unsigned int sb_index, u32 size, u16 ingress_pools_count, u16 egress_pools_count, u16 ingress_tc_count, diff --git a/include/uapi/linux/devlink.h b/include/uapi/linux/devlink.h index ae94cd2a1078..7e15853b77fe 100644 --- a/include/uapi/linux/devlink.h +++ b/include/uapi/linux/devlink.h @@ -213,6 +213,7 @@ enum devlink_port_flavour { enum devlink_rate_type { DEVLINK_RATE_TYPE_LEAF, + DEVLINK_RATE_TYPE_NODE, }; enum devlink_param_cmode { @@ -547,6 +548,8 @@ enum devlink_attr { DEVLINK_ATTR_RATE_TYPE, /* u16 */ DEVLINK_ATTR_RATE_TX_SHARE, /* u64 */ DEVLINK_ATTR_RATE_TX_MAX, /* u64 */ + DEVLINK_ATTR_RATE_NODE_NAME, /* string */ + /* add new attributes above here, update the policy in devlink.c */ __DEVLINK_ATTR_MAX, diff --git a/net/core/devlink.c b/net/core/devlink.c index 37839fd5ca73..589d750b70e4 100644 --- a/net/core/devlink.c +++ b/net/core/devlink.c @@ -196,6 +196,12 @@ devlink_rate_is_leaf(struct devlink_rate *devlink_rate) return devlink_rate->type == DEVLINK_RATE_TYPE_LEAF; } +static inline bool +devlink_rate_is_node(struct devlink_rate *devlink_rate) +{ + return devlink_rate->type == DEVLINK_RATE_TYPE_NODE; +} + static struct devlink_rate * devlink_rate_leaf_get_from_info(struct devlink *devlink, struct genl_info *info) { @@ -209,6 +215,55 @@ devlink_rate_leaf_get_from_info(struct devlink *devlink, struct genl_info *info) return devlink_rate ?: ERR_PTR(-ENODEV); } +static struct devlink_rate * +devlink_rate_node_get_by_name(struct devlink *devlink, const char *node_name) +{ + static struct devlink_rate *devlink_rate; + + list_for_each_entry(devlink_rate, &devlink->rate_list, list) { + if (devlink_rate_is_node(devlink_rate) && + !strcmp(node_name, devlink_rate->name)) + return devlink_rate; + } + return ERR_PTR(-ENODEV); +} + +static struct devlink_rate * +devlink_rate_node_get_from_attrs(struct devlink *devlink, struct nlattr **attrs) +{ + const char *rate_node_name; + size_t len; + + if (!attrs[DEVLINK_ATTR_RATE_NODE_NAME]) + return ERR_PTR(-EINVAL); + rate_node_name = nla_data(attrs[DEVLINK_ATTR_RATE_NODE_NAME]); + len = strlen(rate_node_name); + /* Name cannot be empty or decimal number */ + if (!len || strspn(rate_node_name, "0123456789") == len) + return ERR_PTR(-EINVAL); + + return devlink_rate_node_get_by_name(devlink, rate_node_name); +} + +static struct devlink_rate * +devlink_rate_node_get_from_info(struct devlink *devlink, struct genl_info *info) +{ + return devlink_rate_node_get_from_attrs(devlink, info->attrs); +} + +static struct devlink_rate * +devlink_rate_get_from_info(struct devlink *devlink, struct genl_info *info) +{ + struct nlattr **attrs = info->attrs; + + if (attrs[DEVLINK_ATTR_PORT_INDEX]) + return devlink_rate_leaf_get_from_info(devlink, info); + else if (attrs[DEVLINK_ATTR_RATE_NODE_NAME]) + return devlink_rate_node_get_from_info(devlink, info); + else + return ERR_PTR(-EINVAL); +} + struct devlink_sb { struct list_head list; unsigned int index; @@ -428,12 +483,13 @@ devlink_region_snapshot_get_by_id(struct devlink_region *region, u32 id) #define DEVLINK_NL_FLAG_NEED_PORT BIT(0) #define DEVLINK_NL_FLAG_NEED_DEVLINK_OR_PORT BIT(1) #define DEVLINK_NL_FLAG_NEED_RATE BIT(2) +#define DEVLINK_NL_FLAG_NEED_RATE_NODE BIT(3) /* The per devlink instance lock is taken by default in the pre-doit * operation, yet several commands do not require this. The global * devlink lock is taken and protects from disruption by user-calls. */ -#define DEVLINK_NL_FLAG_NO_LOCK BIT(3) +#define DEVLINK_NL_FLAG_NO_LOCK BIT(4) static int devlink_nl_pre_doit(const struct genl_ops *ops, struct sk_buff *skb, struct genl_info *info) @@ -465,12 +521,21 @@ static int devlink_nl_pre_doit(const struct genl_ops *ops, } else if (ops->internal_flags & DEVLINK_NL_FLAG_NEED_RATE) { struct devlink_rate *devlink_rate; - devlink_rate = devlink_rate_leaf_get_from_info(devlink, info); + devlink_rate = devlink_rate_get_from_info(devlink, info); if (IS_ERR(devlink_rate)) { err = PTR_ERR(devlink_rate); goto unlock; } info->user_ptr[1] = devlink_rate; + } else if (ops->internal_flags & DEVLINK_NL_FLAG_NEED_RATE_NODE) { + struct devlink_rate *rate_node; + + rate_node = devlink_rate_node_get_from_info(devlink, info); + if (IS_ERR(rate_node)) { + err = PTR_ERR(rate_node); + goto unlock; + } + info->user_ptr[1] = rate_node; } return 0; @@ -801,6 +866,10 @@ static int devlink_nl_rate_fill(struct sk_buff *msg, if (nla_put_u32(msg, DEVLINK_ATTR_PORT_INDEX, devlink_rate->devlink_port->index)) goto nla_put_failure; + } else if (devlink_rate_is_node(devlink_rate)) { + if (nla_put_string(msg, DEVLINK_ATTR_RATE_NODE_NAME, + devlink_rate->name)) + goto nla_put_failure; } if (nla_put_u64_64bit(msg, DEVLINK_ATTR_RATE_TX_SHARE, @@ -1508,13 +1577,17 @@ static int devlink_nl_rate_set(struct devlink_rate *devlink_rate, struct genl_info *info) { struct nlattr **attrs = info->attrs; + int err = -EOPNOTSUPP; u64 rate; - int err; if (attrs[DEVLINK_ATTR_RATE_TX_SHARE]) { rate = nla_get_u64(attrs[DEVLINK_ATTR_RATE_TX_SHARE]); - err = ops->rate_leaf_tx_share_set(devlink_rate, devlink_rate->priv, - rate, info->extack); + if (devlink_rate_is_leaf(devlink_rate)) + err = ops->rate_leaf_tx_share_set(devlink_rate, devlink_rate->priv, + rate, info->extack); + else if (devlink_rate_is_node(devlink_rate)) + err = ops->rate_node_tx_share_set(devlink_rate, devlink_rate->priv, + rate, info->extack); if (err) return err; devlink_rate->tx_share = rate; @@ -1522,8 +1595,12 @@ static int devlink_nl_rate_set(struct devlink_rate *devlink_rate, if (attrs[DEVLINK_ATTR_RATE_TX_MAX]) { rate = nla_get_u64(attrs[DEVLINK_ATTR_RATE_TX_MAX]); - err = ops->rate_leaf_tx_max_set(devlink_rate, devlink_rate->priv, - rate, info->extack); + if (devlink_rate_is_leaf(devlink_rate)) + err = ops->rate_leaf_tx_max_set(devlink_rate, devlink_rate->priv, + rate, info->extack); + else if (devlink_rate_is_node(devlink_rate)) + err = ops->rate_node_tx_max_set(devlink_rate, devlink_rate->priv, + rate, info->extack); if (err) return err; devlink_rate->tx_max = rate; @@ -1547,6 +1624,15 @@ static bool devlink_rate_set_ops_supported(const struct devlink_ops *ops, NL_SET_ERR_MSG_MOD(info->extack, "TX max set isn't supported for the leafs"); return false; } + } else if (type == DEVLINK_RATE_TYPE_NODE) { + if (attrs[DEVLINK_ATTR_RATE_TX_SHARE] && !ops->rate_node_tx_share_set) { + NL_SET_ERR_MSG_MOD(info->extack, "TX share set isn't supported for the nodes"); + return false; + } + if (attrs[DEVLINK_ATTR_RATE_TX_MAX] && !ops->rate_node_tx_max_set) { + NL_SET_ERR_MSG_MOD(info->extack, "TX max set isn't supported for the nodes"); + return false; + } } else { WARN_ON("Unknown type of rate object"); return false; @@ -1573,6 +1659,78 @@ static int devlink_nl_cmd_rate_set_doit(struct sk_buff *skb, return err; } +static int devlink_nl_cmd_rate_new_doit(struct sk_buff *skb, + struct genl_info *info) +{ + struct devlink *devlink = info->user_ptr[0]; + struct devlink_rate *rate_node; + const struct devlink_ops *ops; + int err; + + ops = devlink->ops; + if (!ops || !ops->rate_node_new || !ops->rate_node_del) { + NL_SET_ERR_MSG_MOD(info->extack, "Rate nodes aren't supported"); + return -EOPNOTSUPP; + } + + if (!devlink_rate_set_ops_supported(ops, info, DEVLINK_RATE_TYPE_NODE)) + return -EOPNOTSUPP; + + rate_node = devlink_rate_node_get_from_attrs(devlink, info->attrs); + if (!IS_ERR(rate_node)) + return -EEXIST; + else if (rate_node == ERR_PTR(-EINVAL)) + return -EINVAL; + + rate_node = kzalloc(sizeof(*rate_node), GFP_KERNEL); + if (!rate_node) + return -ENOMEM; + + rate_node->devlink = devlink; + rate_node->type = DEVLINK_RATE_TYPE_NODE; + rate_node->name = nla_strdup(info->attrs[DEVLINK_ATTR_RATE_NODE_NAME], GFP_KERNEL); + if (!rate_node->name) { + err = -ENOMEM; + goto err_strdup; + } + + err = ops->rate_node_new(rate_node, &rate_node->priv, info->extack); + if (err) + goto err_node_new; + + err = devlink_nl_rate_set(rate_node, ops, info); + if (err) + goto err_rate_set; + + list_add(&rate_node->list, &devlink->rate_list); + devlink_rate_notify(rate_node, DEVLINK_CMD_RATE_NEW); + return 0; + +err_rate_set: + ops->rate_node_del(rate_node, rate_node->priv, info->extack); +err_node_new: + kfree(rate_node->name); +err_strdup: + kfree(rate_node); + return err; +} + +static int devlink_nl_cmd_rate_del_doit(struct sk_buff *skb, + struct genl_info *info) +{ + struct devlink_rate *rate_node = info->user_ptr[1]; + struct devlink *devlink = rate_node->devlink; + const struct devlink_ops *ops = devlink->ops; + int err; + + devlink_rate_notify(rate_node, DEVLINK_CMD_RATE_DEL); + err = ops->rate_node_del(rate_node, rate_node->priv, info->extack); + list_del(&rate_node->list); + kfree(rate_node->name); + kfree(rate_node); + return err; +} + static int devlink_nl_sb_fill(struct sk_buff *msg, struct devlink *devlink, struct devlink_sb *devlink_sb, enum devlink_command cmd, u32 portid, @@ -2441,6 +2599,30 @@ static int devlink_nl_cmd_eswitch_get_doit(struct sk_buff *skb, return genlmsg_reply(msg, info); } +static int devlink_rate_nodes_check(struct devlink *devlink, u16 mode, + struct netlink_ext_ack *extack) +{ + struct devlink_rate *devlink_rate; + u16 old_mode; + int err; + + if (!devlink->ops->eswitch_mode_get) + return -EOPNOTSUPP; + err = devlink->ops->eswitch_mode_get(devlink, &old_mode); + if (err) + return err; + + if (old_mode == mode) + return 0; + + list_for_each_entry(devlink_rate, &devlink->rate_list, list) + if (devlink_rate_is_node(devlink_rate)) { + NL_SET_ERR_MSG_MOD(extack, "Rate node(s) exists."); + return -EBUSY; + } + return 0; +} + static int devlink_nl_cmd_eswitch_set_doit(struct sk_buff *skb, struct genl_info *info) { @@ -2455,6 +2637,9 @@ static int devlink_nl_cmd_eswitch_set_doit(struct sk_buff *skb, if (!ops->eswitch_mode_set) return -EOPNOTSUPP; mode = nla_get_u16(info->attrs[DEVLINK_ATTR_ESWITCH_MODE]); + err = devlink_rate_nodes_check(devlink, mode, info->extack); + if (err) + return err; err = ops->eswitch_mode_set(devlink, mode, info->extack); if (err) return err; @@ -8038,6 +8223,7 @@ static const struct nla_policy devlink_nl_policy[DEVLINK_ATTR_MAX + 1] = { [DEVLINK_ATTR_RATE_TYPE] = { .type = NLA_U16 }, [DEVLINK_ATTR_RATE_TX_SHARE] = { .type = NLA_U64 }, [DEVLINK_ATTR_RATE_TX_MAX] = { .type = NLA_U64 }, + [DEVLINK_ATTR_RATE_NODE_NAME] = { .type = NLA_NUL_STRING }, }; static const struct genl_small_ops devlink_nl_ops[] = { @@ -8076,6 +8262,17 @@ static const struct genl_small_ops devlink_nl_ops[] = { .flags = GENL_ADMIN_PERM, .internal_flags = DEVLINK_NL_FLAG_NEED_RATE, }, + { + .cmd = DEVLINK_CMD_RATE_NEW, + .doit = devlink_nl_cmd_rate_new_doit, + .flags = GENL_ADMIN_PERM, + }, + { + .cmd = DEVLINK_CMD_RATE_DEL, + .doit = devlink_nl_cmd_rate_del_doit, + .flags = GENL_ADMIN_PERM, + .internal_flags = DEVLINK_NL_FLAG_NEED_RATE_NODE, + }, { .cmd = DEVLINK_CMD_PORT_SPLIT, .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, @@ -8933,6 +9130,33 @@ void devlink_rate_leaf_destroy(struct devlink_port *devlink_port) } EXPORT_SYMBOL_GPL(devlink_rate_leaf_destroy); +/** + * devlink_rate_nodes_destroy - destroy all devlink rate nodes on device + * + * @devlink: devlink instance + * + * Destroy all rate nodes on specified device + * + * Context: Takes and release devlink->lock . + */ +void devlink_rate_nodes_destroy(struct devlink *devlink) +{ + static struct devlink_rate *devlink_rate, *tmp; + const struct devlink_ops *ops = devlink->ops; + + mutex_lock(&devlink->lock); + list_for_each_entry_safe(devlink_rate, tmp, &devlink->rate_list, list) { + if (devlink_rate_is_node(devlink_rate)) { + ops->rate_node_del(devlink_rate, devlink_rate->priv, NULL); + list_del(&devlink_rate->list); + kfree(devlink_rate->name); + kfree(devlink_rate); + } + } + mutex_unlock(&devlink->lock); +} +EXPORT_SYMBOL_GPL(devlink_rate_nodes_destroy); + static int __devlink_port_phys_port_name_get(struct devlink_port *devlink_port, char *name, size_t len) { -- cgit v1.2.3 From d7555984507822458b32a6405881038241d140be Mon Sep 17 00:00:00 2001 From: Dmytro Linkin Date: Wed, 2 Jun 2021 15:17:28 +0300 Subject: devlink: Allow setting parent node of rate objects Refactor DEVLINK_CMD_RATE_{GET|SET} command handlers to support setting a node as a parent for another rate object (leaf or node) by means of new attribute DEVLINK_ATTR_RATE_PARENT_NODE_NAME. Extend devlink ops with new callbacks rate_{leaf|node}_parent_set() to set node as a parent for rate object to allow supporting drivers to implement rate grouping through devlink. Driver implementations are allowed to support leafs or node children only. Invoking callback with NULL as parent should be threated by the driver as unset parent action. Extend rate object struct with reference counter to disallow deleting a node with any child pointing to it. User should unset parent for the child explicitly. Example: $ devlink port function rate add netdevsim/netdevsim10/group1 $ devlink port function rate add netdevsim/netdevsim10/group2 $ devlink port function rate set netdevsim/netdevsim10/group1 parent group2 $ devlink port function rate show netdevsim/netdevsim10/group1 netdevsim/netdevsim10/group1: type node parent group2 $ devlink port function rate set netdevsim/netdevsim10/group1 noparent Co-developed-by: Vlad Buslov Signed-off-by: Vlad Buslov Signed-off-by: Dmytro Linkin Reviewed-by: Jiri Pirko Signed-off-by: David S. Miller --- include/net/devlink.h | 14 ++++- include/uapi/linux/devlink.h | 1 + net/core/devlink.c | 125 ++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 137 insertions(+), 3 deletions(-) (limited to 'include/uapi') diff --git a/include/net/devlink.h b/include/net/devlink.h index 13162b579124..eb045f1b5d1d 100644 --- a/include/net/devlink.h +++ b/include/net/devlink.h @@ -142,9 +142,13 @@ struct devlink_rate { u64 tx_share; u64 tx_max; + struct devlink_rate *parent; union { struct devlink_port *devlink_port; - char *name; + struct { + char *name; + refcount_t refcnt; + }; }; }; @@ -1486,6 +1490,14 @@ struct devlink_ops { struct netlink_ext_ack *extack); int (*rate_node_del)(struct devlink_rate *rate_node, void *priv, struct netlink_ext_ack *extack); + int (*rate_leaf_parent_set)(struct devlink_rate *child, + struct devlink_rate *parent, + void *priv_child, void *priv_parent, + struct netlink_ext_ack *extack); + int (*rate_node_parent_set)(struct devlink_rate *child, + struct devlink_rate *parent, + void *priv_child, void *priv_parent, + struct netlink_ext_ack *extack); }; static inline void *devlink_priv(struct devlink *devlink) diff --git a/include/uapi/linux/devlink.h b/include/uapi/linux/devlink.h index 7e15853b77fe..32f53a0069d6 100644 --- a/include/uapi/linux/devlink.h +++ b/include/uapi/linux/devlink.h @@ -549,6 +549,7 @@ enum devlink_attr { DEVLINK_ATTR_RATE_TX_SHARE, /* u64 */ DEVLINK_ATTR_RATE_TX_MAX, /* u64 */ DEVLINK_ATTR_RATE_NODE_NAME, /* string */ + DEVLINK_ATTR_RATE_PARENT_NODE_NAME, /* string */ /* add new attributes above here, update the policy in devlink.c */ diff --git a/net/core/devlink.c b/net/core/devlink.c index 589d750b70e4..464f56408247 100644 --- a/net/core/devlink.c +++ b/net/core/devlink.c @@ -880,6 +880,11 @@ static int devlink_nl_rate_fill(struct sk_buff *msg, devlink_rate->tx_max, DEVLINK_ATTR_PAD)) goto nla_put_failure; + if (devlink_rate->parent) + if (nla_put_string(msg, DEVLINK_ATTR_RATE_PARENT_NODE_NAME, + devlink_rate->parent->name)) + goto nla_put_failure; + genlmsg_end(msg, hdr); return 0; @@ -1152,6 +1157,18 @@ static int devlink_nl_cmd_rate_get_doit(struct sk_buff *skb, return genlmsg_reply(msg, info); } +static bool +devlink_rate_is_parent_node(struct devlink_rate *devlink_rate, + struct devlink_rate *parent) +{ + while (parent) { + if (parent == devlink_rate) + return true; + parent = parent->parent; + } + return false; +} + static int devlink_nl_cmd_get_doit(struct sk_buff *skb, struct genl_info *info) { struct devlink *devlink = info->user_ptr[0]; @@ -1572,11 +1589,75 @@ static int devlink_nl_cmd_port_del_doit(struct sk_buff *skb, return devlink->ops->port_del(devlink, port_index, extack); } +static int +devlink_nl_rate_parent_node_set(struct devlink_rate *devlink_rate, + struct genl_info *info, + struct nlattr *nla_parent) +{ + struct devlink *devlink = devlink_rate->devlink; + const char *parent_name = nla_data(nla_parent); + const struct devlink_ops *ops = devlink->ops; + size_t len = strlen(parent_name); + struct devlink_rate *parent; + int err = -EOPNOTSUPP; + + parent = devlink_rate->parent; + if (parent && len) { + NL_SET_ERR_MSG_MOD(info->extack, "Rate object already has parent."); + return -EBUSY; + } else if (parent && !len) { + if (devlink_rate_is_leaf(devlink_rate)) + err = ops->rate_leaf_parent_set(devlink_rate, NULL, + devlink_rate->priv, NULL, + info->extack); + else if (devlink_rate_is_node(devlink_rate)) + err = ops->rate_node_parent_set(devlink_rate, NULL, + devlink_rate->priv, NULL, + info->extack); + if (err) + return err; + + refcount_dec(&parent->refcnt); + devlink_rate->parent = NULL; + } else if (!parent && len) { + parent = devlink_rate_node_get_by_name(devlink, parent_name); + if (IS_ERR(parent)) + return -ENODEV; + + if (parent == devlink_rate) { + NL_SET_ERR_MSG_MOD(info->extack, "Parent to self is not allowed"); + return -EINVAL; + } + + if (devlink_rate_is_node(devlink_rate) && + devlink_rate_is_parent_node(devlink_rate, parent->parent)) { + NL_SET_ERR_MSG_MOD(info->extack, "Node is already a parent of parent node."); + return -EEXIST; + } + + if (devlink_rate_is_leaf(devlink_rate)) + err = ops->rate_leaf_parent_set(devlink_rate, parent, + devlink_rate->priv, parent->priv, + info->extack); + else if (devlink_rate_is_node(devlink_rate)) + err = ops->rate_node_parent_set(devlink_rate, parent, + devlink_rate->priv, parent->priv, + info->extack); + if (err) + return err; + + refcount_inc(&parent->refcnt); + devlink_rate->parent = parent; + } + + return 0; +} + static int devlink_nl_rate_set(struct devlink_rate *devlink_rate, const struct devlink_ops *ops, struct genl_info *info) { - struct nlattr **attrs = info->attrs; + struct nlattr *nla_parent, **attrs = info->attrs; int err = -EOPNOTSUPP; u64 rate; @@ -1606,6 +1687,14 @@ static int devlink_nl_rate_set(struct devlink_rate *devlink_rate, devlink_rate->tx_max = rate; } + nla_parent = attrs[DEVLINK_ATTR_RATE_PARENT_NODE_NAME]; + if (nla_parent) { + err = devlink_nl_rate_parent_node_set(devlink_rate, info, + nla_parent); + if (err) + return err; + } + return 0; } @@ -1624,6 +1713,11 @@ static bool devlink_rate_set_ops_supported(const struct devlink_ops *ops, NL_SET_ERR_MSG_MOD(info->extack, "TX max set isn't supported for the leafs"); return false; } + if (attrs[DEVLINK_ATTR_RATE_PARENT_NODE_NAME] && + !ops->rate_leaf_parent_set) { + NL_SET_ERR_MSG_MOD(info->extack, "Parent set isn't supported for the leafs"); + return false; + } } else if (type == DEVLINK_RATE_TYPE_NODE) { if (attrs[DEVLINK_ATTR_RATE_TX_SHARE] && !ops->rate_node_tx_share_set) { NL_SET_ERR_MSG_MOD(info->extack, "TX share set isn't supported for the nodes"); @@ -1633,6 +1727,11 @@ static bool devlink_rate_set_ops_supported(const struct devlink_ops *ops, NL_SET_ERR_MSG_MOD(info->extack, "TX max set isn't supported for the nodes"); return false; } + if (attrs[DEVLINK_ATTR_RATE_PARENT_NODE_NAME] && + !ops->rate_node_parent_set) { + NL_SET_ERR_MSG_MOD(info->extack, "Parent set isn't supported for the nodes"); + return false; + } } else { WARN_ON("Unknown type of rate object"); return false; @@ -1702,6 +1801,7 @@ static int devlink_nl_cmd_rate_new_doit(struct sk_buff *skb, if (err) goto err_rate_set; + refcount_set(&rate_node->refcnt, 1); list_add(&rate_node->list, &devlink->rate_list); devlink_rate_notify(rate_node, DEVLINK_CMD_RATE_NEW); return 0; @@ -1723,8 +1823,15 @@ static int devlink_nl_cmd_rate_del_doit(struct sk_buff *skb, const struct devlink_ops *ops = devlink->ops; int err; + if (refcount_read(&rate_node->refcnt) > 1) { + NL_SET_ERR_MSG_MOD(info->extack, "Node has children. Cannot delete node."); + return -EBUSY; + } + devlink_rate_notify(rate_node, DEVLINK_CMD_RATE_DEL); err = ops->rate_node_del(rate_node, rate_node->priv, info->extack); + if (rate_node->parent) + refcount_dec(&rate_node->parent->refcnt); list_del(&rate_node->list); kfree(rate_node->name); kfree(rate_node); @@ -8224,6 +8331,7 @@ static const struct nla_policy devlink_nl_policy[DEVLINK_ATTR_MAX + 1] = { [DEVLINK_ATTR_RATE_TX_SHARE] = { .type = NLA_U64 }, [DEVLINK_ATTR_RATE_TX_MAX] = { .type = NLA_U64 }, [DEVLINK_ATTR_RATE_NODE_NAME] = { .type = NLA_NUL_STRING }, + [DEVLINK_ATTR_RATE_PARENT_NODE_NAME] = { .type = NLA_NUL_STRING }, }; static const struct genl_small_ops devlink_nl_ops[] = { @@ -9135,7 +9243,8 @@ EXPORT_SYMBOL_GPL(devlink_rate_leaf_destroy); * * @devlink: devlink instance * - * Destroy all rate nodes on specified device + * Unset parent for all rate objects and destroy all rate nodes + * on specified device. * * Context: Takes and release devlink->lock . */ @@ -9145,6 +9254,18 @@ void devlink_rate_nodes_destroy(struct devlink *devlink) const struct devlink_ops *ops = devlink->ops; mutex_lock(&devlink->lock); + list_for_each_entry(devlink_rate, &devlink->rate_list, list) { + if (!devlink_rate->parent) + continue; + + refcount_dec(&devlink_rate->parent->refcnt); + if (devlink_rate_is_leaf(devlink_rate)) + ops->rate_leaf_parent_set(devlink_rate, NULL, devlink_rate->priv, + NULL, NULL); + else if (devlink_rate_is_node(devlink_rate)) + ops->rate_node_parent_set(devlink_rate, NULL, devlink_rate->priv, + NULL, NULL); + } list_for_each_entry_safe(devlink_rate, tmp, &devlink->rate_list, list) { if (devlink_rate_is_node(devlink_rate)) { ops->rate_node_del(devlink_rate, devlink_rate->priv, NULL); -- cgit v1.2.3 From b48c24c2d710cf34810c555dcef883a3d35a9c08 Mon Sep 17 00:00:00 2001 From: Mustafa Ismail Date: Wed, 2 Jun 2021 15:51:31 -0500 Subject: RDMA/irdma: Implement device supported verb APIs Implement device supported verb APIs. The supported APIs vary based on the underlying transport the ibdev is registered as (i.e. iWARP or RoCEv2). Link: https://lore.kernel.org/r/20210602205138.889-10-shiraz.saleem@intel.com Signed-off-by: Mustafa Ismail Signed-off-by: Shiraz Saleem Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/irdma/verbs.c | 4527 +++++++++++++++++++++++++++++++ drivers/infiniband/hw/irdma/verbs.h | 225 ++ include/uapi/rdma/ib_user_ioctl_verbs.h | 1 + 3 files changed, 4753 insertions(+) create mode 100644 drivers/infiniband/hw/irdma/verbs.c create mode 100644 drivers/infiniband/hw/irdma/verbs.h (limited to 'include/uapi') diff --git a/drivers/infiniband/hw/irdma/verbs.c b/drivers/infiniband/hw/irdma/verbs.c new file mode 100644 index 000000000000..294155293243 --- /dev/null +++ b/drivers/infiniband/hw/irdma/verbs.c @@ -0,0 +1,4527 @@ +// SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB +/* Copyright (c) 2015 - 2021 Intel Corporation */ +#include "main.h" + +/** + * irdma_query_device - get device attributes + * @ibdev: device pointer from stack + * @props: returning device attributes + * @udata: user data + */ +static int irdma_query_device(struct ib_device *ibdev, + struct ib_device_attr *props, + struct ib_udata *udata) +{ + struct irdma_device *iwdev = to_iwdev(ibdev); + struct irdma_pci_f *rf = iwdev->rf; + struct pci_dev *pcidev = iwdev->rf->pcidev; + struct irdma_hw_attrs *hw_attrs = &rf->sc_dev.hw_attrs; + + if (udata->inlen || udata->outlen) + return -EINVAL; + + memset(props, 0, sizeof(*props)); + ether_addr_copy((u8 *)&props->sys_image_guid, iwdev->netdev->dev_addr); + props->fw_ver = (u64)irdma_fw_major_ver(&rf->sc_dev) << 32 | + irdma_fw_minor_ver(&rf->sc_dev); + props->device_cap_flags = iwdev->device_cap_flags; + props->vendor_id = pcidev->vendor; + props->vendor_part_id = pcidev->device; + + props->hw_ver = rf->pcidev->revision; + props->page_size_cap = SZ_4K | SZ_2M | SZ_1G; + props->max_mr_size = hw_attrs->max_mr_size; + props->max_qp = rf->max_qp - rf->used_qps; + props->max_qp_wr = hw_attrs->max_qp_wr; + props->max_send_sge = hw_attrs->uk_attrs.max_hw_wq_frags; + props->max_recv_sge = hw_attrs->uk_attrs.max_hw_wq_frags; + props->max_cq = rf->max_cq - rf->used_cqs; + props->max_cqe = rf->max_cqe; + props->max_mr = rf->max_mr - rf->used_mrs; + props->max_mw = props->max_mr; + props->max_pd = rf->max_pd - rf->used_pds; + props->max_sge_rd = hw_attrs->uk_attrs.max_hw_read_sges; + props->max_qp_rd_atom = hw_attrs->max_hw_ird; + props->max_qp_init_rd_atom = hw_attrs->max_hw_ord; + if (rdma_protocol_roce(ibdev, 1)) + props->max_pkeys = IRDMA_PKEY_TBL_SZ; + props->max_ah = rf->max_ah; + props->max_mcast_grp = rf->max_mcg; + props->max_mcast_qp_attach = IRDMA_MAX_MGS_PER_CTX; + props->max_total_mcast_qp_attach = rf->max_qp * IRDMA_MAX_MGS_PER_CTX; + props->max_fast_reg_page_list_len = IRDMA_MAX_PAGES_PER_FMR; +#define HCA_CLOCK_TIMESTAMP_MASK 0x1ffff + if (hw_attrs->uk_attrs.hw_rev >= IRDMA_GEN_2) + props->timestamp_mask = HCA_CLOCK_TIMESTAMP_MASK; + + return 0; +} + +/** + * irdma_get_eth_speed_and_width - Get IB port speed and width from netdev speed + * @link_speed: netdev phy link speed + * @active_speed: IB port speed + * @active_width: IB port width + */ +static void irdma_get_eth_speed_and_width(u32 link_speed, u16 *active_speed, + u8 *active_width) +{ + if (link_speed <= SPEED_1000) { + *active_width = IB_WIDTH_1X; + *active_speed = IB_SPEED_SDR; + } else if (link_speed <= SPEED_10000) { + *active_width = IB_WIDTH_1X; + *active_speed = IB_SPEED_FDR10; + } else if (link_speed <= SPEED_20000) { + *active_width = IB_WIDTH_4X; + *active_speed = IB_SPEED_DDR; + } else if (link_speed <= SPEED_25000) { + *active_width = IB_WIDTH_1X; + *active_speed = IB_SPEED_EDR; + } else if (link_speed <= SPEED_40000) { + *active_width = IB_WIDTH_4X; + *active_speed = IB_SPEED_FDR10; + } else { + *active_width = IB_WIDTH_4X; + *active_speed = IB_SPEED_EDR; + } +} + +/** + * irdma_query_port - get port attributes + * @ibdev: device pointer from stack + * @port: port number for query + * @props: returning device attributes + */ +static int irdma_query_port(struct ib_device *ibdev, u32 port, + struct ib_port_attr *props) +{ + struct irdma_device *iwdev = to_iwdev(ibdev); + struct net_device *netdev = iwdev->netdev; + + /* no need to zero out pros here. done by caller */ + + props->max_mtu = IB_MTU_4096; + props->active_mtu = ib_mtu_int_to_enum(netdev->mtu); + props->lid = 1; + props->lmc = 0; + props->sm_lid = 0; + props->sm_sl = 0; + if (netif_carrier_ok(netdev) && netif_running(netdev)) { + props->state = IB_PORT_ACTIVE; + props->phys_state = IB_PORT_PHYS_STATE_LINK_UP; + } else { + props->state = IB_PORT_DOWN; + props->phys_state = IB_PORT_PHYS_STATE_DISABLED; + } + irdma_get_eth_speed_and_width(SPEED_100000, &props->active_speed, + &props->active_width); + + if (rdma_protocol_roce(ibdev, 1)) { + props->gid_tbl_len = 32; + props->ip_gids = true; + props->pkey_tbl_len = IRDMA_PKEY_TBL_SZ; + } else { + props->gid_tbl_len = 1; + } + props->qkey_viol_cntr = 0; + props->port_cap_flags |= IB_PORT_CM_SUP | IB_PORT_REINIT_SUP; + props->max_msg_sz = iwdev->rf->sc_dev.hw_attrs.max_hw_outbound_msg_size; + + return 0; +} + +/** + * irdma_disassociate_ucontext - Disassociate user context + * @context: ib user context + */ +static void irdma_disassociate_ucontext(struct ib_ucontext *context) +{ +} + +static int irdma_mmap_legacy(struct irdma_ucontext *ucontext, + struct vm_area_struct *vma) +{ + u64 pfn; + + if (vma->vm_pgoff || vma->vm_end - vma->vm_start != PAGE_SIZE) + return -EINVAL; + + vma->vm_private_data = ucontext; + pfn = ((uintptr_t)ucontext->iwdev->rf->sc_dev.hw_regs[IRDMA_DB_ADDR_OFFSET] + + pci_resource_start(ucontext->iwdev->rf->pcidev, 0)) >> PAGE_SHIFT; + + return rdma_user_mmap_io(&ucontext->ibucontext, vma, pfn, PAGE_SIZE, + pgprot_noncached(vma->vm_page_prot), NULL); +} + +static void irdma_mmap_free(struct rdma_user_mmap_entry *rdma_entry) +{ + struct irdma_user_mmap_entry *entry = to_irdma_mmap_entry(rdma_entry); + + kfree(entry); +} + +static struct rdma_user_mmap_entry* +irdma_user_mmap_entry_insert(struct irdma_ucontext *ucontext, u64 bar_offset, + enum irdma_mmap_flag mmap_flag, u64 *mmap_offset) +{ + struct irdma_user_mmap_entry *entry = kzalloc(sizeof(*entry), GFP_KERNEL); + int ret; + + if (!entry) + return NULL; + + entry->bar_offset = bar_offset; + entry->mmap_flag = mmap_flag; + + ret = rdma_user_mmap_entry_insert(&ucontext->ibucontext, + &entry->rdma_entry, PAGE_SIZE); + if (ret) { + kfree(entry); + return NULL; + } + *mmap_offset = rdma_user_mmap_get_offset(&entry->rdma_entry); + + return &entry->rdma_entry; +} + +/** + * irdma_mmap - user memory map + * @context: context created during alloc + * @vma: kernel info for user memory map + */ +static int irdma_mmap(struct ib_ucontext *context, struct vm_area_struct *vma) +{ + struct rdma_user_mmap_entry *rdma_entry; + struct irdma_user_mmap_entry *entry; + struct irdma_ucontext *ucontext; + u64 pfn; + int ret; + + ucontext = to_ucontext(context); + + /* Legacy support for libi40iw with hard-coded mmap key */ + if (ucontext->legacy_mode) + return irdma_mmap_legacy(ucontext, vma); + + rdma_entry = rdma_user_mmap_entry_get(&ucontext->ibucontext, vma); + if (!rdma_entry) { + ibdev_dbg(&ucontext->iwdev->ibdev, + "VERBS: pgoff[0x%lx] does not have valid entry\n", + vma->vm_pgoff); + return -EINVAL; + } + + entry = to_irdma_mmap_entry(rdma_entry); + ibdev_dbg(&ucontext->iwdev->ibdev, + "VERBS: bar_offset [0x%llx] mmap_flag [%d]\n", + entry->bar_offset, entry->mmap_flag); + + pfn = (entry->bar_offset + + pci_resource_start(ucontext->iwdev->rf->pcidev, 0)) >> PAGE_SHIFT; + + switch (entry->mmap_flag) { + case IRDMA_MMAP_IO_NC: + ret = rdma_user_mmap_io(context, vma, pfn, PAGE_SIZE, + pgprot_noncached(vma->vm_page_prot), + rdma_entry); + break; + case IRDMA_MMAP_IO_WC: + ret = rdma_user_mmap_io(context, vma, pfn, PAGE_SIZE, + pgprot_writecombine(vma->vm_page_prot), + rdma_entry); + break; + default: + ret = -EINVAL; + } + + if (ret) + ibdev_dbg(&ucontext->iwdev->ibdev, + "VERBS: bar_offset [0x%llx] mmap_flag[%d] err[%d]\n", + entry->bar_offset, entry->mmap_flag, ret); + rdma_user_mmap_entry_put(rdma_entry); + + return ret; +} + +/** + * irdma_alloc_push_page - allocate a push page for qp + * @iwqp: qp pointer + */ +static void irdma_alloc_push_page(struct irdma_qp *iwqp) +{ + struct irdma_cqp_request *cqp_request; + struct cqp_cmds_info *cqp_info; + struct irdma_device *iwdev = iwqp->iwdev; + struct irdma_sc_qp *qp = &iwqp->sc_qp; + enum irdma_status_code status; + + cqp_request = irdma_alloc_and_get_cqp_request(&iwdev->rf->cqp, true); + if (!cqp_request) + return; + + cqp_info = &cqp_request->info; + cqp_info->cqp_cmd = IRDMA_OP_MANAGE_PUSH_PAGE; + cqp_info->post_sq = 1; + cqp_info->in.u.manage_push_page.info.push_idx = 0; + cqp_info->in.u.manage_push_page.info.qs_handle = + qp->vsi->qos[qp->user_pri].qs_handle; + cqp_info->in.u.manage_push_page.info.free_page = 0; + cqp_info->in.u.manage_push_page.info.push_page_type = 0; + cqp_info->in.u.manage_push_page.cqp = &iwdev->rf->cqp.sc_cqp; + cqp_info->in.u.manage_push_page.scratch = (uintptr_t)cqp_request; + + status = irdma_handle_cqp_op(iwdev->rf, cqp_request); + if (!status && cqp_request->compl_info.op_ret_val < + iwdev->rf->sc_dev.hw_attrs.max_hw_device_pages) { + qp->push_idx = cqp_request->compl_info.op_ret_val; + qp->push_offset = 0; + } + + irdma_put_cqp_request(&iwdev->rf->cqp, cqp_request); +} + +/** + * irdma_alloc_ucontext - Allocate the user context data structure + * @uctx: uverbs context pointer + * @udata: user data + * + * This keeps track of all objects associated with a particular + * user-mode client. + */ +static int irdma_alloc_ucontext(struct ib_ucontext *uctx, + struct ib_udata *udata) +{ + struct ib_device *ibdev = uctx->device; + struct irdma_device *iwdev = to_iwdev(ibdev); + struct irdma_alloc_ucontext_req req; + struct irdma_alloc_ucontext_resp uresp = {}; + struct irdma_ucontext *ucontext = to_ucontext(uctx); + struct irdma_uk_attrs *uk_attrs; + + if (ib_copy_from_udata(&req, udata, min(sizeof(req), udata->inlen))) + return -EINVAL; + + if (req.userspace_ver < 4 || req.userspace_ver > IRDMA_ABI_VER) + goto ver_error; + + ucontext->iwdev = iwdev; + ucontext->abi_ver = req.userspace_ver; + + uk_attrs = &iwdev->rf->sc_dev.hw_attrs.uk_attrs; + /* GEN_1 legacy support with libi40iw */ + if (udata->outlen < sizeof(uresp)) { + if (uk_attrs->hw_rev != IRDMA_GEN_1) + return -EOPNOTSUPP; + + ucontext->legacy_mode = true; + uresp.max_qps = iwdev->rf->max_qp; + uresp.max_pds = iwdev->rf->sc_dev.hw_attrs.max_hw_pds; + uresp.wq_size = iwdev->rf->sc_dev.hw_attrs.max_qp_wr * 2; + uresp.kernel_ver = req.userspace_ver; + if (ib_copy_to_udata(udata, &uresp, + min(sizeof(uresp), udata->outlen))) + return -EFAULT; + } else { + u64 bar_off = (uintptr_t)iwdev->rf->sc_dev.hw_regs[IRDMA_DB_ADDR_OFFSET]; + + ucontext->db_mmap_entry = + irdma_user_mmap_entry_insert(ucontext, bar_off, + IRDMA_MMAP_IO_NC, + &uresp.db_mmap_key); + if (!ucontext->db_mmap_entry) + return -ENOMEM; + + uresp.kernel_ver = IRDMA_ABI_VER; + uresp.feature_flags = uk_attrs->feature_flags; + uresp.max_hw_wq_frags = uk_attrs->max_hw_wq_frags; + uresp.max_hw_read_sges = uk_attrs->max_hw_read_sges; + uresp.max_hw_inline = uk_attrs->max_hw_inline; + uresp.max_hw_rq_quanta = uk_attrs->max_hw_rq_quanta; + uresp.max_hw_wq_quanta = uk_attrs->max_hw_wq_quanta; + uresp.max_hw_sq_chunk = uk_attrs->max_hw_sq_chunk; + uresp.max_hw_cq_size = uk_attrs->max_hw_cq_size; + uresp.min_hw_cq_size = uk_attrs->min_hw_cq_size; + uresp.hw_rev = uk_attrs->hw_rev; + if (ib_copy_to_udata(udata, &uresp, + min(sizeof(uresp), udata->outlen))) { + rdma_user_mmap_entry_remove(ucontext->db_mmap_entry); + return -EFAULT; + } + } + + INIT_LIST_HEAD(&ucontext->cq_reg_mem_list); + spin_lock_init(&ucontext->cq_reg_mem_list_lock); + INIT_LIST_HEAD(&ucontext->qp_reg_mem_list); + spin_lock_init(&ucontext->qp_reg_mem_list_lock); + + return 0; + +ver_error: + ibdev_err(&iwdev->ibdev, + "Invalid userspace driver version detected. Detected version %d, should be %d\n", + req.userspace_ver, IRDMA_ABI_VER); + return -EINVAL; +} + +/** + * irdma_dealloc_ucontext - deallocate the user context data structure + * @context: user context created during alloc + */ +static void irdma_dealloc_ucontext(struct ib_ucontext *context) +{ + struct irdma_ucontext *ucontext = to_ucontext(context); + + rdma_user_mmap_entry_remove(ucontext->db_mmap_entry); +} + +/** + * irdma_alloc_pd - allocate protection domain + * @pd: PD pointer + * @udata: user data + */ +static int irdma_alloc_pd(struct ib_pd *pd, struct ib_udata *udata) +{ + struct irdma_pd *iwpd = to_iwpd(pd); + struct irdma_device *iwdev = to_iwdev(pd->device); + struct irdma_sc_dev *dev = &iwdev->rf->sc_dev; + struct irdma_pci_f *rf = iwdev->rf; + struct irdma_alloc_pd_resp uresp = {}; + struct irdma_sc_pd *sc_pd; + u32 pd_id = 0; + int err; + + err = irdma_alloc_rsrc(rf, rf->allocated_pds, rf->max_pd, &pd_id, + &rf->next_pd); + if (err) + return err; + + sc_pd = &iwpd->sc_pd; + if (udata) { + struct irdma_ucontext *ucontext = + rdma_udata_to_drv_context(udata, struct irdma_ucontext, + ibucontext); + irdma_sc_pd_init(dev, sc_pd, pd_id, ucontext->abi_ver); + uresp.pd_id = pd_id; + if (ib_copy_to_udata(udata, &uresp, + min(sizeof(uresp), udata->outlen))) { + err = -EFAULT; + goto error; + } + } else { + irdma_sc_pd_init(dev, sc_pd, pd_id, IRDMA_ABI_VER); + } + + return 0; +error: + irdma_free_rsrc(rf, rf->allocated_pds, pd_id); + + return err; +} + +/** + * irdma_dealloc_pd - deallocate pd + * @ibpd: ptr of pd to be deallocated + * @udata: user data + */ +static int irdma_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) +{ + struct irdma_pd *iwpd = to_iwpd(ibpd); + struct irdma_device *iwdev = to_iwdev(ibpd->device); + + irdma_free_rsrc(iwdev->rf, iwdev->rf->allocated_pds, iwpd->sc_pd.pd_id); + + return 0; +} + +/** + * irdma_get_pbl - Retrieve pbl from a list given a virtual + * address + * @va: user virtual address + * @pbl_list: pbl list to search in (QP's or CQ's) + */ +static struct irdma_pbl *irdma_get_pbl(unsigned long va, + struct list_head *pbl_list) +{ + struct irdma_pbl *iwpbl; + + list_for_each_entry (iwpbl, pbl_list, list) { + if (iwpbl->user_base == va) { + list_del(&iwpbl->list); + iwpbl->on_list = false; + return iwpbl; + } + } + + return NULL; +} + +/** + * irdma_clean_cqes - clean cq entries for qp + * @iwqp: qp ptr (user or kernel) + * @iwcq: cq ptr + */ +static void irdma_clean_cqes(struct irdma_qp *iwqp, struct irdma_cq *iwcq) +{ + struct irdma_cq_uk *ukcq = &iwcq->sc_cq.cq_uk; + unsigned long flags; + + spin_lock_irqsave(&iwcq->lock, flags); + irdma_uk_clean_cq(&iwqp->sc_qp.qp_uk, ukcq); + spin_unlock_irqrestore(&iwcq->lock, flags); +} + +static void irdma_remove_push_mmap_entries(struct irdma_qp *iwqp) +{ + if (iwqp->push_db_mmap_entry) { + rdma_user_mmap_entry_remove(iwqp->push_db_mmap_entry); + iwqp->push_db_mmap_entry = NULL; + } + if (iwqp->push_wqe_mmap_entry) { + rdma_user_mmap_entry_remove(iwqp->push_wqe_mmap_entry); + iwqp->push_wqe_mmap_entry = NULL; + } +} + +static int irdma_setup_push_mmap_entries(struct irdma_ucontext *ucontext, + struct irdma_qp *iwqp, + u64 *push_wqe_mmap_key, + u64 *push_db_mmap_key) +{ + struct irdma_device *iwdev = ucontext->iwdev; + u64 rsvd, bar_off; + + rsvd = IRDMA_PF_BAR_RSVD; + bar_off = (uintptr_t)iwdev->rf->sc_dev.hw_regs[IRDMA_DB_ADDR_OFFSET]; + /* skip over db page */ + bar_off += IRDMA_HW_PAGE_SIZE; + /* push wqe page */ + bar_off += rsvd + iwqp->sc_qp.push_idx * IRDMA_HW_PAGE_SIZE; + iwqp->push_wqe_mmap_entry = irdma_user_mmap_entry_insert(ucontext, + bar_off, IRDMA_MMAP_IO_WC, + push_wqe_mmap_key); + if (!iwqp->push_wqe_mmap_entry) + return -ENOMEM; + + /* push doorbell page */ + bar_off += IRDMA_HW_PAGE_SIZE; + iwqp->push_db_mmap_entry = irdma_user_mmap_entry_insert(ucontext, + bar_off, IRDMA_MMAP_IO_NC, + push_db_mmap_key); + if (!iwqp->push_db_mmap_entry) { + rdma_user_mmap_entry_remove(iwqp->push_wqe_mmap_entry); + return -ENOMEM; + } + + return 0; +} + +/** + * irdma_destroy_qp - destroy qp + * @ibqp: qp's ib pointer also to get to device's qp address + * @udata: user data + */ +static int irdma_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata) +{ + struct irdma_qp *iwqp = to_iwqp(ibqp); + struct irdma_device *iwdev = iwqp->iwdev; + + iwqp->sc_qp.qp_uk.destroy_pending = true; + + if (iwqp->iwarp_state == IRDMA_QP_STATE_RTS) + irdma_modify_qp_to_err(&iwqp->sc_qp); + + irdma_qp_rem_ref(&iwqp->ibqp); + wait_for_completion(&iwqp->free_qp); + irdma_free_lsmm_rsrc(iwqp); + if (!iwdev->reset) + irdma_cqp_qp_destroy_cmd(&iwdev->rf->sc_dev, &iwqp->sc_qp); + + if (!iwqp->user_mode) { + if (iwqp->iwscq) { + irdma_clean_cqes(iwqp, iwqp->iwscq); + if (iwqp->iwrcq != iwqp->iwscq) + irdma_clean_cqes(iwqp, iwqp->iwrcq); + } + } + irdma_remove_push_mmap_entries(iwqp); + irdma_free_qp_rsrc(iwqp); + + return 0; +} + +/** + * irdma_setup_virt_qp - setup for allocation of virtual qp + * @iwdev: irdma device + * @iwqp: qp ptr + * @init_info: initialize info to return + */ +static int irdma_setup_virt_qp(struct irdma_device *iwdev, + struct irdma_qp *iwqp, + struct irdma_qp_init_info *init_info) +{ + struct irdma_pbl *iwpbl = iwqp->iwpbl; + struct irdma_qp_mr *qpmr = &iwpbl->qp_mr; + + iwqp->page = qpmr->sq_page; + init_info->shadow_area_pa = qpmr->shadow; + if (iwpbl->pbl_allocated) { + init_info->virtual_map = true; + init_info->sq_pa = qpmr->sq_pbl.idx; + init_info->rq_pa = qpmr->rq_pbl.idx; + } else { + init_info->sq_pa = qpmr->sq_pbl.addr; + init_info->rq_pa = qpmr->rq_pbl.addr; + } + + return 0; +} + +/** + * irdma_setup_kmode_qp - setup initialization for kernel mode qp + * @iwdev: iwarp device + * @iwqp: qp ptr (user or kernel) + * @info: initialize info to return + * @init_attr: Initial QP create attributes + */ +static int irdma_setup_kmode_qp(struct irdma_device *iwdev, + struct irdma_qp *iwqp, + struct irdma_qp_init_info *info, + struct ib_qp_init_attr *init_attr) +{ + struct irdma_dma_mem *mem = &iwqp->kqp.dma_mem; + u32 sqdepth, rqdepth; + u8 sqshift, rqshift; + u32 size; + enum irdma_status_code status; + struct irdma_qp_uk_init_info *ukinfo = &info->qp_uk_init_info; + struct irdma_uk_attrs *uk_attrs = &iwdev->rf->sc_dev.hw_attrs.uk_attrs; + + irdma_get_wqe_shift(uk_attrs, + uk_attrs->hw_rev >= IRDMA_GEN_2 ? ukinfo->max_sq_frag_cnt + 1 : + ukinfo->max_sq_frag_cnt, + ukinfo->max_inline_data, &sqshift); + status = irdma_get_sqdepth(uk_attrs, ukinfo->sq_size, sqshift, + &sqdepth); + if (status) + return -ENOMEM; + + if (uk_attrs->hw_rev == IRDMA_GEN_1) + rqshift = IRDMA_MAX_RQ_WQE_SHIFT_GEN1; + else + irdma_get_wqe_shift(uk_attrs, ukinfo->max_rq_frag_cnt, 0, + &rqshift); + + status = irdma_get_rqdepth(uk_attrs, ukinfo->rq_size, rqshift, + &rqdepth); + if (status) + return -ENOMEM; + + iwqp->kqp.sq_wrid_mem = + kcalloc(sqdepth, sizeof(*iwqp->kqp.sq_wrid_mem), GFP_KERNEL); + if (!iwqp->kqp.sq_wrid_mem) + return -ENOMEM; + + iwqp->kqp.rq_wrid_mem = + kcalloc(rqdepth, sizeof(*iwqp->kqp.rq_wrid_mem), GFP_KERNEL); + if (!iwqp->kqp.rq_wrid_mem) { + kfree(iwqp->kqp.sq_wrid_mem); + iwqp->kqp.sq_wrid_mem = NULL; + return -ENOMEM; + } + + ukinfo->sq_wrtrk_array = iwqp->kqp.sq_wrid_mem; + ukinfo->rq_wrid_array = iwqp->kqp.rq_wrid_mem; + + size = (sqdepth + rqdepth) * IRDMA_QP_WQE_MIN_SIZE; + size += (IRDMA_SHADOW_AREA_SIZE << 3); + + mem->size = ALIGN(size, 256); + mem->va = dma_alloc_coherent(iwdev->rf->hw.device, mem->size, + &mem->pa, GFP_KERNEL); + if (!mem->va) { + kfree(iwqp->kqp.sq_wrid_mem); + iwqp->kqp.sq_wrid_mem = NULL; + kfree(iwqp->kqp.rq_wrid_mem); + iwqp->kqp.rq_wrid_mem = NULL; + return -ENOMEM; + } + + ukinfo->sq = mem->va; + info->sq_pa = mem->pa; + ukinfo->rq = &ukinfo->sq[sqdepth]; + info->rq_pa = info->sq_pa + (sqdepth * IRDMA_QP_WQE_MIN_SIZE); + ukinfo->shadow_area = ukinfo->rq[rqdepth].elem; + info->shadow_area_pa = info->rq_pa + (rqdepth * IRDMA_QP_WQE_MIN_SIZE); + ukinfo->sq_size = sqdepth >> sqshift; + ukinfo->rq_size = rqdepth >> rqshift; + ukinfo->qp_id = iwqp->ibqp.qp_num; + + init_attr->cap.max_send_wr = (sqdepth - IRDMA_SQ_RSVD) >> sqshift; + init_attr->cap.max_recv_wr = (rqdepth - IRDMA_RQ_RSVD) >> rqshift; + + return 0; +} + +static int irdma_cqp_create_qp_cmd(struct irdma_qp *iwqp) +{ + struct irdma_pci_f *rf = iwqp->iwdev->rf; + struct irdma_cqp_request *cqp_request; + struct cqp_cmds_info *cqp_info; + struct irdma_create_qp_info *qp_info; + enum irdma_status_code status; + + cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true); + if (!cqp_request) + return -ENOMEM; + + cqp_info = &cqp_request->info; + qp_info = &cqp_request->info.in.u.qp_create.info; + memset(qp_info, 0, sizeof(*qp_info)); + qp_info->mac_valid = true; + qp_info->cq_num_valid = true; + qp_info->next_iwarp_state = IRDMA_QP_STATE_IDLE; + + cqp_info->cqp_cmd = IRDMA_OP_QP_CREATE; + cqp_info->post_sq = 1; + cqp_info->in.u.qp_create.qp = &iwqp->sc_qp; + cqp_info->in.u.qp_create.scratch = (uintptr_t)cqp_request; + status = irdma_handle_cqp_op(rf, cqp_request); + irdma_put_cqp_request(&rf->cqp, cqp_request); + + return status ? -ENOMEM : 0; +} + +static void irdma_roce_fill_and_set_qpctx_info(struct irdma_qp *iwqp, + struct irdma_qp_host_ctx_info *ctx_info) +{ + struct irdma_device *iwdev = iwqp->iwdev; + struct irdma_sc_dev *dev = &iwdev->rf->sc_dev; + struct irdma_roce_offload_info *roce_info; + struct irdma_udp_offload_info *udp_info; + + udp_info = &iwqp->udp_info; + udp_info->snd_mss = ib_mtu_enum_to_int(ib_mtu_int_to_enum(iwdev->vsi.mtu)); + udp_info->cwnd = iwdev->roce_cwnd; + udp_info->rexmit_thresh = 2; + udp_info->rnr_nak_thresh = 2; + udp_info->src_port = 0xc000; + udp_info->dst_port = ROCE_V2_UDP_DPORT; + roce_info = &iwqp->roce_info; + ether_addr_copy(roce_info->mac_addr, iwdev->netdev->dev_addr); + + roce_info->rd_en = true; + roce_info->wr_rdresp_en = true; + roce_info->bind_en = true; + roce_info->dcqcn_en = false; + roce_info->rtomin = 5; + + roce_info->ack_credits = iwdev->roce_ackcreds; + roce_info->ird_size = dev->hw_attrs.max_hw_ird; + roce_info->ord_size = dev->hw_attrs.max_hw_ord; + + if (!iwqp->user_mode) { + roce_info->priv_mode_en = true; + roce_info->fast_reg_en = true; + roce_info->udprivcq_en = true; + } + roce_info->roce_tver = 0; + + ctx_info->roce_info = &iwqp->roce_info; + ctx_info->udp_info = &iwqp->udp_info; + irdma_sc_qp_setctx_roce(&iwqp->sc_qp, iwqp->host_ctx.va, ctx_info); +} + +static void irdma_iw_fill_and_set_qpctx_info(struct irdma_qp *iwqp, + struct irdma_qp_host_ctx_info *ctx_info) +{ + struct irdma_device *iwdev = iwqp->iwdev; + struct irdma_sc_dev *dev = &iwdev->rf->sc_dev; + struct irdma_iwarp_offload_info *iwarp_info; + + iwarp_info = &iwqp->iwarp_info; + ether_addr_copy(iwarp_info->mac_addr, iwdev->netdev->dev_addr); + iwarp_info->rd_en = true; + iwarp_info->wr_rdresp_en = true; + iwarp_info->bind_en = true; + iwarp_info->ecn_en = true; + iwarp_info->rtomin = 5; + + if (dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) + iwarp_info->ib_rd_en = true; + if (!iwqp->user_mode) { + iwarp_info->priv_mode_en = true; + iwarp_info->fast_reg_en = true; + } + iwarp_info->ddp_ver = 1; + iwarp_info->rdmap_ver = 1; + + ctx_info->iwarp_info = &iwqp->iwarp_info; + ctx_info->iwarp_info_valid = true; + irdma_sc_qp_setctx(&iwqp->sc_qp, iwqp->host_ctx.va, ctx_info); + ctx_info->iwarp_info_valid = false; +} + +static int irdma_validate_qp_attrs(struct ib_qp_init_attr *init_attr, + struct irdma_device *iwdev) +{ + struct irdma_sc_dev *dev = &iwdev->rf->sc_dev; + struct irdma_uk_attrs *uk_attrs = &dev->hw_attrs.uk_attrs; + + if (init_attr->create_flags) + return -EOPNOTSUPP; + + if (init_attr->cap.max_inline_data > uk_attrs->max_hw_inline || + init_attr->cap.max_send_sge > uk_attrs->max_hw_wq_frags || + init_attr->cap.max_recv_sge > uk_attrs->max_hw_wq_frags) + return -EINVAL; + + if (rdma_protocol_roce(&iwdev->ibdev, 1)) { + if (init_attr->qp_type != IB_QPT_RC && + init_attr->qp_type != IB_QPT_UD && + init_attr->qp_type != IB_QPT_GSI) + return -EOPNOTSUPP; + } else { + if (init_attr->qp_type != IB_QPT_RC) + return -EOPNOTSUPP; + } + + return 0; +} + +/** + * irdma_create_qp - create qp + * @ibpd: ptr of pd + * @init_attr: attributes for qp + * @udata: user data for create qp + */ +static struct ib_qp *irdma_create_qp(struct ib_pd *ibpd, + struct ib_qp_init_attr *init_attr, + struct ib_udata *udata) +{ + struct irdma_pd *iwpd = to_iwpd(ibpd); + struct irdma_device *iwdev = to_iwdev(ibpd->device); + struct irdma_pci_f *rf = iwdev->rf; + struct irdma_qp *iwqp; + struct irdma_create_qp_req req; + struct irdma_create_qp_resp uresp = {}; + u32 qp_num = 0; + enum irdma_status_code ret; + int err_code; + int sq_size; + int rq_size; + struct irdma_sc_qp *qp; + struct irdma_sc_dev *dev = &rf->sc_dev; + struct irdma_uk_attrs *uk_attrs = &dev->hw_attrs.uk_attrs; + struct irdma_qp_init_info init_info = {}; + struct irdma_qp_host_ctx_info *ctx_info; + unsigned long flags; + + err_code = irdma_validate_qp_attrs(init_attr, iwdev); + if (err_code) + return ERR_PTR(err_code); + + sq_size = init_attr->cap.max_send_wr; + rq_size = init_attr->cap.max_recv_wr; + + init_info.vsi = &iwdev->vsi; + init_info.qp_uk_init_info.uk_attrs = uk_attrs; + init_info.qp_uk_init_info.sq_size = sq_size; + init_info.qp_uk_init_info.rq_size = rq_size; + init_info.qp_uk_init_info.max_sq_frag_cnt = init_attr->cap.max_send_sge; + init_info.qp_uk_init_info.max_rq_frag_cnt = init_attr->cap.max_recv_sge; + init_info.qp_uk_init_info.max_inline_data = init_attr->cap.max_inline_data; + + iwqp = kzalloc(sizeof(*iwqp), GFP_KERNEL); + if (!iwqp) + return ERR_PTR(-ENOMEM); + + qp = &iwqp->sc_qp; + qp->qp_uk.back_qp = iwqp; + qp->qp_uk.lock = &iwqp->lock; + qp->push_idx = IRDMA_INVALID_PUSH_PAGE_INDEX; + + iwqp->iwdev = iwdev; + iwqp->q2_ctx_mem.size = ALIGN(IRDMA_Q2_BUF_SIZE + IRDMA_QP_CTX_SIZE, + 256); + iwqp->q2_ctx_mem.va = dma_alloc_coherent(dev->hw->device, + iwqp->q2_ctx_mem.size, + &iwqp->q2_ctx_mem.pa, + GFP_KERNEL); + if (!iwqp->q2_ctx_mem.va) { + err_code = -ENOMEM; + goto error; + } + + init_info.q2 = iwqp->q2_ctx_mem.va; + init_info.q2_pa = iwqp->q2_ctx_mem.pa; + init_info.host_ctx = (__le64 *)(init_info.q2 + IRDMA_Q2_BUF_SIZE); + init_info.host_ctx_pa = init_info.q2_pa + IRDMA_Q2_BUF_SIZE; + + if (init_attr->qp_type == IB_QPT_GSI) + qp_num = 1; + else + err_code = irdma_alloc_rsrc(rf, rf->allocated_qps, rf->max_qp, + &qp_num, &rf->next_qp); + if (err_code) + goto error; + + iwqp->iwpd = iwpd; + iwqp->ibqp.qp_num = qp_num; + qp = &iwqp->sc_qp; + iwqp->iwscq = to_iwcq(init_attr->send_cq); + iwqp->iwrcq = to_iwcq(init_attr->recv_cq); + iwqp->host_ctx.va = init_info.host_ctx; + iwqp->host_ctx.pa = init_info.host_ctx_pa; + iwqp->host_ctx.size = IRDMA_QP_CTX_SIZE; + + init_info.pd = &iwpd->sc_pd; + init_info.qp_uk_init_info.qp_id = iwqp->ibqp.qp_num; + if (!rdma_protocol_roce(&iwdev->ibdev, 1)) + init_info.qp_uk_init_info.first_sq_wq = 1; + iwqp->ctx_info.qp_compl_ctx = (uintptr_t)qp; + init_waitqueue_head(&iwqp->waitq); + init_waitqueue_head(&iwqp->mod_qp_waitq); + + if (udata) { + err_code = ib_copy_from_udata(&req, udata, + min(sizeof(req), udata->inlen)); + if (err_code) { + ibdev_dbg(&iwdev->ibdev, + "VERBS: ib_copy_from_data fail\n"); + goto error; + } + + iwqp->ctx_info.qp_compl_ctx = req.user_compl_ctx; + iwqp->user_mode = 1; + if (req.user_wqe_bufs) { + struct irdma_ucontext *ucontext = + rdma_udata_to_drv_context(udata, + struct irdma_ucontext, + ibucontext); + + init_info.qp_uk_init_info.legacy_mode = ucontext->legacy_mode; + spin_lock_irqsave(&ucontext->qp_reg_mem_list_lock, flags); + iwqp->iwpbl = irdma_get_pbl((unsigned long)req.user_wqe_bufs, + &ucontext->qp_reg_mem_list); + spin_unlock_irqrestore(&ucontext->qp_reg_mem_list_lock, flags); + + if (!iwqp->iwpbl) { + err_code = -ENODATA; + ibdev_dbg(&iwdev->ibdev, "VERBS: no pbl info\n"); + goto error; + } + } + init_info.qp_uk_init_info.abi_ver = iwpd->sc_pd.abi_ver; + err_code = irdma_setup_virt_qp(iwdev, iwqp, &init_info); + } else { + init_info.qp_uk_init_info.abi_ver = IRDMA_ABI_VER; + err_code = irdma_setup_kmode_qp(iwdev, iwqp, &init_info, init_attr); + } + + if (err_code) { + ibdev_dbg(&iwdev->ibdev, "VERBS: setup qp failed\n"); + goto error; + } + + if (rdma_protocol_roce(&iwdev->ibdev, 1)) { + if (init_attr->qp_type == IB_QPT_RC) { + init_info.qp_uk_init_info.type = IRDMA_QP_TYPE_ROCE_RC; + init_info.qp_uk_init_info.qp_caps = IRDMA_SEND_WITH_IMM | + IRDMA_WRITE_WITH_IMM | + IRDMA_ROCE; + } else { + init_info.qp_uk_init_info.type = IRDMA_QP_TYPE_ROCE_UD; + init_info.qp_uk_init_info.qp_caps = IRDMA_SEND_WITH_IMM | + IRDMA_ROCE; + } + } else { + init_info.qp_uk_init_info.type = IRDMA_QP_TYPE_IWARP; + init_info.qp_uk_init_info.qp_caps = IRDMA_WRITE_WITH_IMM; + } + + if (dev->hw_attrs.uk_attrs.hw_rev > IRDMA_GEN_1) + init_info.qp_uk_init_info.qp_caps |= IRDMA_PUSH_MODE; + + ret = irdma_sc_qp_init(qp, &init_info); + if (ret) { + err_code = -EPROTO; + ibdev_dbg(&iwdev->ibdev, "VERBS: qp_init fail\n"); + goto error; + } + + ctx_info = &iwqp->ctx_info; + ctx_info->send_cq_num = iwqp->iwscq->sc_cq.cq_uk.cq_id; + ctx_info->rcv_cq_num = iwqp->iwrcq->sc_cq.cq_uk.cq_id; + + if (rdma_protocol_roce(&iwdev->ibdev, 1)) + irdma_roce_fill_and_set_qpctx_info(iwqp, ctx_info); + else + irdma_iw_fill_and_set_qpctx_info(iwqp, ctx_info); + + err_code = irdma_cqp_create_qp_cmd(iwqp); + if (err_code) + goto error; + + refcount_set(&iwqp->refcnt, 1); + spin_lock_init(&iwqp->lock); + spin_lock_init(&iwqp->sc_qp.pfpdu.lock); + iwqp->sig_all = (init_attr->sq_sig_type == IB_SIGNAL_ALL_WR) ? 1 : 0; + rf->qp_table[qp_num] = iwqp; + iwqp->max_send_wr = sq_size; + iwqp->max_recv_wr = rq_size; + + if (rdma_protocol_roce(&iwdev->ibdev, 1)) { + if (dev->ws_add(&iwdev->vsi, 0)) { + irdma_cqp_qp_destroy_cmd(&rf->sc_dev, &iwqp->sc_qp); + err_code = -EINVAL; + goto error; + } + + irdma_qp_add_qos(&iwqp->sc_qp); + } + + if (udata) { + /* GEN_1 legacy support with libi40iw does not have expanded uresp struct */ + if (udata->outlen < sizeof(uresp)) { + uresp.lsmm = 1; + uresp.push_idx = IRDMA_INVALID_PUSH_PAGE_INDEX_GEN_1; + } else { + if (rdma_protocol_iwarp(&iwdev->ibdev, 1)) + uresp.lsmm = 1; + } + uresp.actual_sq_size = sq_size; + uresp.actual_rq_size = rq_size; + uresp.qp_id = qp_num; + uresp.qp_caps = qp->qp_uk.qp_caps; + + err_code = ib_copy_to_udata(udata, &uresp, + min(sizeof(uresp), udata->outlen)); + if (err_code) { + ibdev_dbg(&iwdev->ibdev, "VERBS: copy_to_udata failed\n"); + irdma_destroy_qp(&iwqp->ibqp, udata); + return ERR_PTR(err_code); + } + } + + init_completion(&iwqp->free_qp); + return &iwqp->ibqp; + +error: + irdma_free_qp_rsrc(iwqp); + + return ERR_PTR(err_code); +} + +static int irdma_get_ib_acc_flags(struct irdma_qp *iwqp) +{ + int acc_flags = 0; + + if (rdma_protocol_roce(iwqp->ibqp.device, 1)) { + if (iwqp->roce_info.wr_rdresp_en) { + acc_flags |= IB_ACCESS_LOCAL_WRITE; + acc_flags |= IB_ACCESS_REMOTE_WRITE; + } + if (iwqp->roce_info.rd_en) + acc_flags |= IB_ACCESS_REMOTE_READ; + if (iwqp->roce_info.bind_en) + acc_flags |= IB_ACCESS_MW_BIND; + } else { + if (iwqp->iwarp_info.wr_rdresp_en) { + acc_flags |= IB_ACCESS_LOCAL_WRITE; + acc_flags |= IB_ACCESS_REMOTE_WRITE; + } + if (iwqp->iwarp_info.rd_en) + acc_flags |= IB_ACCESS_REMOTE_READ; + if (iwqp->iwarp_info.bind_en) + acc_flags |= IB_ACCESS_MW_BIND; + } + return acc_flags; +} + +/** + * irdma_query_qp - query qp attributes + * @ibqp: qp pointer + * @attr: attributes pointer + * @attr_mask: Not used + * @init_attr: qp attributes to return + */ +static int irdma_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, + int attr_mask, struct ib_qp_init_attr *init_attr) +{ + struct irdma_qp *iwqp = to_iwqp(ibqp); + struct irdma_sc_qp *qp = &iwqp->sc_qp; + + memset(attr, 0, sizeof(*attr)); + memset(init_attr, 0, sizeof(*init_attr)); + + attr->qp_state = iwqp->ibqp_state; + attr->cur_qp_state = iwqp->ibqp_state; + attr->cap.max_send_wr = iwqp->max_send_wr; + attr->cap.max_recv_wr = iwqp->max_recv_wr; + attr->cap.max_inline_data = qp->qp_uk.max_inline_data; + attr->cap.max_send_sge = qp->qp_uk.max_sq_frag_cnt; + attr->cap.max_recv_sge = qp->qp_uk.max_rq_frag_cnt; + attr->qp_access_flags = irdma_get_ib_acc_flags(iwqp); + attr->port_num = 1; + if (rdma_protocol_roce(ibqp->device, 1)) { + attr->path_mtu = ib_mtu_int_to_enum(iwqp->udp_info.snd_mss); + attr->qkey = iwqp->roce_info.qkey; + attr->rq_psn = iwqp->udp_info.epsn; + attr->sq_psn = iwqp->udp_info.psn_nxt; + attr->dest_qp_num = iwqp->roce_info.dest_qp; + attr->pkey_index = iwqp->roce_info.p_key; + attr->retry_cnt = iwqp->udp_info.rexmit_thresh; + attr->rnr_retry = iwqp->udp_info.rnr_nak_thresh; + attr->max_rd_atomic = iwqp->roce_info.ord_size; + attr->max_dest_rd_atomic = iwqp->roce_info.ird_size; + } + + init_attr->event_handler = iwqp->ibqp.event_handler; + init_attr->qp_context = iwqp->ibqp.qp_context; + init_attr->send_cq = iwqp->ibqp.send_cq; + init_attr->recv_cq = iwqp->ibqp.recv_cq; + init_attr->cap = attr->cap; + + return 0; +} + +/** + * irdma_query_pkey - Query partition key + * @ibdev: device pointer from stack + * @port: port number + * @index: index of pkey + * @pkey: pointer to store the pkey + */ +static int irdma_query_pkey(struct ib_device *ibdev, u32 port, u16 index, + u16 *pkey) +{ + if (index >= IRDMA_PKEY_TBL_SZ) + return -EINVAL; + + *pkey = IRDMA_DEFAULT_PKEY; + return 0; +} + +/** + * irdma_modify_qp_roce - modify qp request + * @ibqp: qp's pointer for modify + * @attr: access attributes + * @attr_mask: state mask + * @udata: user data + */ +int irdma_modify_qp_roce(struct ib_qp *ibqp, struct ib_qp_attr *attr, + int attr_mask, struct ib_udata *udata) +{ + struct irdma_pd *iwpd = to_iwpd(ibqp->pd); + struct irdma_qp *iwqp = to_iwqp(ibqp); + struct irdma_device *iwdev = iwqp->iwdev; + struct irdma_sc_dev *dev = &iwdev->rf->sc_dev; + struct irdma_qp_host_ctx_info *ctx_info; + struct irdma_roce_offload_info *roce_info; + struct irdma_udp_offload_info *udp_info; + struct irdma_modify_qp_info info = {}; + struct irdma_modify_qp_resp uresp = {}; + struct irdma_modify_qp_req ureq = {}; + unsigned long flags; + u8 issue_modify_qp = 0; + int ret = 0; + + ctx_info = &iwqp->ctx_info; + roce_info = &iwqp->roce_info; + udp_info = &iwqp->udp_info; + + if (attr_mask & ~IB_QP_ATTR_STANDARD_BITS) + return -EOPNOTSUPP; + + if (attr_mask & IB_QP_DEST_QPN) + roce_info->dest_qp = attr->dest_qp_num; + + if (attr_mask & IB_QP_PKEY_INDEX) { + ret = irdma_query_pkey(ibqp->device, 0, attr->pkey_index, + &roce_info->p_key); + if (ret) + return ret; + } + + if (attr_mask & IB_QP_QKEY) + roce_info->qkey = attr->qkey; + + if (attr_mask & IB_QP_PATH_MTU) + udp_info->snd_mss = ib_mtu_enum_to_int(attr->path_mtu); + + if (attr_mask & IB_QP_SQ_PSN) { + udp_info->psn_nxt = attr->sq_psn; + udp_info->lsn = 0xffff; + udp_info->psn_una = attr->sq_psn; + udp_info->psn_max = attr->sq_psn; + } + + if (attr_mask & IB_QP_RQ_PSN) + udp_info->epsn = attr->rq_psn; + + if (attr_mask & IB_QP_RNR_RETRY) + udp_info->rnr_nak_thresh = attr->rnr_retry; + + if (attr_mask & IB_QP_RETRY_CNT) + udp_info->rexmit_thresh = attr->retry_cnt; + + ctx_info->roce_info->pd_id = iwpd->sc_pd.pd_id; + + if (attr_mask & IB_QP_AV) { + struct irdma_av *av = &iwqp->roce_ah.av; + const struct ib_gid_attr *sgid_attr; + u16 vlan_id = VLAN_N_VID; + u32 local_ip[4]; + + memset(&iwqp->roce_ah, 0, sizeof(iwqp->roce_ah)); + if (attr->ah_attr.ah_flags & IB_AH_GRH) { + udp_info->ttl = attr->ah_attr.grh.hop_limit; + udp_info->flow_label = attr->ah_attr.grh.flow_label; + udp_info->tos = attr->ah_attr.grh.traffic_class; + irdma_qp_rem_qos(&iwqp->sc_qp); + dev->ws_remove(iwqp->sc_qp.vsi, ctx_info->user_pri); + ctx_info->user_pri = rt_tos2priority(udp_info->tos); + iwqp->sc_qp.user_pri = ctx_info->user_pri; + if (dev->ws_add(iwqp->sc_qp.vsi, ctx_info->user_pri)) + return -ENOMEM; + irdma_qp_add_qos(&iwqp->sc_qp); + } + sgid_attr = attr->ah_attr.grh.sgid_attr; + ret = rdma_read_gid_l2_fields(sgid_attr, &vlan_id, + ctx_info->roce_info->mac_addr); + if (ret) + return ret; + + if (vlan_id >= VLAN_N_VID && iwdev->dcb) + vlan_id = 0; + if (vlan_id < VLAN_N_VID) { + udp_info->insert_vlan_tag = true; + udp_info->vlan_tag = vlan_id | + ctx_info->user_pri << VLAN_PRIO_SHIFT; + } else { + udp_info->insert_vlan_tag = false; + } + + av->attrs = attr->ah_attr; + rdma_gid2ip((struct sockaddr *)&av->sgid_addr, &sgid_attr->gid); + rdma_gid2ip((struct sockaddr *)&av->dgid_addr, &attr->ah_attr.grh.dgid); + roce_info->local_qp = ibqp->qp_num; + if (av->sgid_addr.saddr.sa_family == AF_INET6) { + __be32 *daddr = + av->dgid_addr.saddr_in6.sin6_addr.in6_u.u6_addr32; + __be32 *saddr = + av->sgid_addr.saddr_in6.sin6_addr.in6_u.u6_addr32; + + irdma_copy_ip_ntohl(&udp_info->dest_ip_addr[0], daddr); + irdma_copy_ip_ntohl(&udp_info->local_ipaddr[0], saddr); + + udp_info->ipv4 = false; + irdma_copy_ip_ntohl(local_ip, daddr); + + udp_info->arp_idx = irdma_arp_table(iwdev->rf, + &local_ip[0], + false, NULL, + IRDMA_ARP_RESOLVE); + } else { + __be32 saddr = av->sgid_addr.saddr_in.sin_addr.s_addr; + __be32 daddr = av->dgid_addr.saddr_in.sin_addr.s_addr; + + local_ip[0] = ntohl(daddr); + + udp_info->ipv4 = true; + udp_info->dest_ip_addr[0] = 0; + udp_info->dest_ip_addr[1] = 0; + udp_info->dest_ip_addr[2] = 0; + udp_info->dest_ip_addr[3] = local_ip[0]; + + udp_info->local_ipaddr[0] = 0; + udp_info->local_ipaddr[1] = 0; + udp_info->local_ipaddr[2] = 0; + udp_info->local_ipaddr[3] = ntohl(saddr); + } + udp_info->arp_idx = + irdma_add_arp(iwdev->rf, local_ip, udp_info->ipv4, + attr->ah_attr.roce.dmac); + } + + if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC) { + if (attr->max_rd_atomic > dev->hw_attrs.max_hw_ord) { + ibdev_err(&iwdev->ibdev, + "rd_atomic = %d, above max_hw_ord=%d\n", + attr->max_rd_atomic, + dev->hw_attrs.max_hw_ord); + return -EINVAL; + } + if (attr->max_rd_atomic) + roce_info->ord_size = attr->max_rd_atomic; + info.ord_valid = true; + } + + if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) { + if (attr->max_dest_rd_atomic > dev->hw_attrs.max_hw_ird) { + ibdev_err(&iwdev->ibdev, + "rd_atomic = %d, above max_hw_ird=%d\n", + attr->max_rd_atomic, + dev->hw_attrs.max_hw_ird); + return -EINVAL; + } + if (attr->max_dest_rd_atomic) + roce_info->ird_size = attr->max_dest_rd_atomic; + } + + if (attr_mask & IB_QP_ACCESS_FLAGS) { + if (attr->qp_access_flags & IB_ACCESS_LOCAL_WRITE) + roce_info->wr_rdresp_en = true; + if (attr->qp_access_flags & IB_ACCESS_REMOTE_WRITE) + roce_info->wr_rdresp_en = true; + if (attr->qp_access_flags & IB_ACCESS_REMOTE_READ) + roce_info->rd_en = true; + } + + wait_event(iwqp->mod_qp_waitq, !atomic_read(&iwqp->hw_mod_qp_pend)); + + ibdev_dbg(&iwdev->ibdev, + "VERBS: caller: %pS qp_id=%d to_ibqpstate=%d ibqpstate=%d irdma_qpstate=%d attr_mask=0x%x\n", + __builtin_return_address(0), ibqp->qp_num, attr->qp_state, + iwqp->ibqp_state, iwqp->iwarp_state, attr_mask); + + spin_lock_irqsave(&iwqp->lock, flags); + if (attr_mask & IB_QP_STATE) { + if (!ib_modify_qp_is_ok(iwqp->ibqp_state, attr->qp_state, + iwqp->ibqp.qp_type, attr_mask)) { + ibdev_warn(&iwdev->ibdev, "modify_qp invalid for qp_id=%d, old_state=0x%x, new_state=0x%x\n", + iwqp->ibqp.qp_num, iwqp->ibqp_state, + attr->qp_state); + ret = -EINVAL; + goto exit; + } + info.curr_iwarp_state = iwqp->iwarp_state; + + switch (attr->qp_state) { + case IB_QPS_INIT: + if (iwqp->iwarp_state > IRDMA_QP_STATE_IDLE) { + ret = -EINVAL; + goto exit; + } + + if (iwqp->iwarp_state == IRDMA_QP_STATE_INVALID) { + info.next_iwarp_state = IRDMA_QP_STATE_IDLE; + issue_modify_qp = 1; + } + break; + case IB_QPS_RTR: + if (iwqp->iwarp_state > IRDMA_QP_STATE_IDLE) { + ret = -EINVAL; + goto exit; + } + info.arp_cache_idx_valid = true; + info.cq_num_valid = true; + info.next_iwarp_state = IRDMA_QP_STATE_RTR; + issue_modify_qp = 1; + break; + case IB_QPS_RTS: + if (iwqp->ibqp_state < IB_QPS_RTR || + iwqp->ibqp_state == IB_QPS_ERR) { + ret = -EINVAL; + goto exit; + } + + info.arp_cache_idx_valid = true; + info.cq_num_valid = true; + info.ord_valid = true; + info.next_iwarp_state = IRDMA_QP_STATE_RTS; + issue_modify_qp = 1; + if (iwdev->push_mode && udata && + iwqp->sc_qp.push_idx == IRDMA_INVALID_PUSH_PAGE_INDEX && + dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) { + spin_unlock_irqrestore(&iwqp->lock, flags); + irdma_alloc_push_page(iwqp); + spin_lock_irqsave(&iwqp->lock, flags); + } + break; + case IB_QPS_SQD: + if (iwqp->iwarp_state == IRDMA_QP_STATE_SQD) + goto exit; + + if (iwqp->iwarp_state != IRDMA_QP_STATE_RTS) { + ret = -EINVAL; + goto exit; + } + + info.next_iwarp_state = IRDMA_QP_STATE_SQD; + issue_modify_qp = 1; + break; + case IB_QPS_SQE: + case IB_QPS_ERR: + case IB_QPS_RESET: + if (iwqp->iwarp_state == IRDMA_QP_STATE_RTS) { + spin_unlock_irqrestore(&iwqp->lock, flags); + info.next_iwarp_state = IRDMA_QP_STATE_SQD; + irdma_hw_modify_qp(iwdev, iwqp, &info, true); + spin_lock_irqsave(&iwqp->lock, flags); + } + + if (iwqp->iwarp_state == IRDMA_QP_STATE_ERROR) { + spin_unlock_irqrestore(&iwqp->lock, flags); + if (udata) { + if (ib_copy_from_udata(&ureq, udata, + min(sizeof(ureq), udata->inlen))) + return -EINVAL; + + irdma_flush_wqes(iwqp, + (ureq.sq_flush ? IRDMA_FLUSH_SQ : 0) | + (ureq.rq_flush ? IRDMA_FLUSH_RQ : 0) | + IRDMA_REFLUSH); + } + return 0; + } + + info.next_iwarp_state = IRDMA_QP_STATE_ERROR; + issue_modify_qp = 1; + break; + default: + ret = -EINVAL; + goto exit; + } + + iwqp->ibqp_state = attr->qp_state; + } + + ctx_info->send_cq_num = iwqp->iwscq->sc_cq.cq_uk.cq_id; + ctx_info->rcv_cq_num = iwqp->iwrcq->sc_cq.cq_uk.cq_id; + irdma_sc_qp_setctx_roce(&iwqp->sc_qp, iwqp->host_ctx.va, ctx_info); + spin_unlock_irqrestore(&iwqp->lock, flags); + + if (attr_mask & IB_QP_STATE) { + if (issue_modify_qp) { + ctx_info->rem_endpoint_idx = udp_info->arp_idx; + if (irdma_hw_modify_qp(iwdev, iwqp, &info, true)) + return -EINVAL; + spin_lock_irqsave(&iwqp->lock, flags); + if (iwqp->iwarp_state == info.curr_iwarp_state) { + iwqp->iwarp_state = info.next_iwarp_state; + iwqp->ibqp_state = attr->qp_state; + } + if (iwqp->ibqp_state > IB_QPS_RTS && + !iwqp->flush_issued) { + iwqp->flush_issued = 1; + spin_unlock_irqrestore(&iwqp->lock, flags); + irdma_flush_wqes(iwqp, IRDMA_FLUSH_SQ | + IRDMA_FLUSH_RQ | + IRDMA_FLUSH_WAIT); + } else { + spin_unlock_irqrestore(&iwqp->lock, flags); + } + } else { + iwqp->ibqp_state = attr->qp_state; + } + if (udata && dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) { + struct irdma_ucontext *ucontext; + + ucontext = rdma_udata_to_drv_context(udata, + struct irdma_ucontext, ibucontext); + if (iwqp->sc_qp.push_idx != IRDMA_INVALID_PUSH_PAGE_INDEX && + !iwqp->push_wqe_mmap_entry && + !irdma_setup_push_mmap_entries(ucontext, iwqp, + &uresp.push_wqe_mmap_key, &uresp.push_db_mmap_key)) { + uresp.push_valid = 1; + uresp.push_offset = iwqp->sc_qp.push_offset; + } + ret = ib_copy_to_udata(udata, &uresp, min(sizeof(uresp), + udata->outlen)); + if (ret) { + irdma_remove_push_mmap_entries(iwqp); + ibdev_dbg(&iwdev->ibdev, + "VERBS: copy_to_udata failed\n"); + return ret; + } + } + } + + return 0; +exit: + spin_unlock_irqrestore(&iwqp->lock, flags); + + return ret; +} + +/** + * irdma_modify_qp - modify qp request + * @ibqp: qp's pointer for modify + * @attr: access attributes + * @attr_mask: state mask + * @udata: user data + */ +int irdma_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask, + struct ib_udata *udata) +{ + struct irdma_qp *iwqp = to_iwqp(ibqp); + struct irdma_device *iwdev = iwqp->iwdev; + struct irdma_sc_dev *dev = &iwdev->rf->sc_dev; + struct irdma_qp_host_ctx_info *ctx_info; + struct irdma_tcp_offload_info *tcp_info; + struct irdma_iwarp_offload_info *offload_info; + struct irdma_modify_qp_info info = {}; + struct irdma_modify_qp_resp uresp = {}; + struct irdma_modify_qp_req ureq = {}; + u8 issue_modify_qp = 0; + u8 dont_wait = 0; + int err; + unsigned long flags; + + if (attr_mask & ~IB_QP_ATTR_STANDARD_BITS) + return ~EOPNOTSUPP; + + ctx_info = &iwqp->ctx_info; + offload_info = &iwqp->iwarp_info; + tcp_info = &iwqp->tcp_info; + wait_event(iwqp->mod_qp_waitq, !atomic_read(&iwqp->hw_mod_qp_pend)); + ibdev_dbg(&iwdev->ibdev, + "VERBS: caller: %pS qp_id=%d to_ibqpstate=%d ibqpstate=%d irdma_qpstate=%d last_aeq=%d hw_tcp_state=%d hw_iwarp_state=%d attr_mask=0x%x\n", + __builtin_return_address(0), ibqp->qp_num, attr->qp_state, + iwqp->ibqp_state, iwqp->iwarp_state, iwqp->last_aeq, + iwqp->hw_tcp_state, iwqp->hw_iwarp_state, attr_mask); + + spin_lock_irqsave(&iwqp->lock, flags); + if (attr_mask & IB_QP_STATE) { + info.curr_iwarp_state = iwqp->iwarp_state; + switch (attr->qp_state) { + case IB_QPS_INIT: + case IB_QPS_RTR: + if (iwqp->iwarp_state > IRDMA_QP_STATE_IDLE) { + err = -EINVAL; + goto exit; + } + + if (iwqp->iwarp_state == IRDMA_QP_STATE_INVALID) { + info.next_iwarp_state = IRDMA_QP_STATE_IDLE; + issue_modify_qp = 1; + } + if (iwdev->push_mode && udata && + iwqp->sc_qp.push_idx == IRDMA_INVALID_PUSH_PAGE_INDEX && + dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) { + spin_unlock_irqrestore(&iwqp->lock, flags); + irdma_alloc_push_page(iwqp); + spin_lock_irqsave(&iwqp->lock, flags); + } + break; + case IB_QPS_RTS: + if (iwqp->iwarp_state > IRDMA_QP_STATE_RTS || + !iwqp->cm_id) { + err = -EINVAL; + goto exit; + } + + issue_modify_qp = 1; + iwqp->hw_tcp_state = IRDMA_TCP_STATE_ESTABLISHED; + iwqp->hte_added = 1; + info.next_iwarp_state = IRDMA_QP_STATE_RTS; + info.tcp_ctx_valid = true; + info.ord_valid = true; + info.arp_cache_idx_valid = true; + info.cq_num_valid = true; + break; + case IB_QPS_SQD: + if (iwqp->hw_iwarp_state > IRDMA_QP_STATE_RTS) { + err = 0; + goto exit; + } + + if (iwqp->iwarp_state == IRDMA_QP_STATE_CLOSING || + iwqp->iwarp_state < IRDMA_QP_STATE_RTS) { + err = 0; + goto exit; + } + + if (iwqp->iwarp_state > IRDMA_QP_STATE_CLOSING) { + err = -EINVAL; + goto exit; + } + + info.next_iwarp_state = IRDMA_QP_STATE_CLOSING; + issue_modify_qp = 1; + break; + case IB_QPS_SQE: + if (iwqp->iwarp_state >= IRDMA_QP_STATE_TERMINATE) { + err = -EINVAL; + goto exit; + } + + info.next_iwarp_state = IRDMA_QP_STATE_TERMINATE; + issue_modify_qp = 1; + break; + case IB_QPS_ERR: + case IB_QPS_RESET: + if (iwqp->iwarp_state == IRDMA_QP_STATE_ERROR) { + spin_unlock_irqrestore(&iwqp->lock, flags); + if (udata) { + if (ib_copy_from_udata(&ureq, udata, + min(sizeof(ureq), udata->inlen))) + return -EINVAL; + + irdma_flush_wqes(iwqp, + (ureq.sq_flush ? IRDMA_FLUSH_SQ : 0) | + (ureq.rq_flush ? IRDMA_FLUSH_RQ : 0) | + IRDMA_REFLUSH); + } + return 0; + } + + if (iwqp->sc_qp.term_flags) { + spin_unlock_irqrestore(&iwqp->lock, flags); + irdma_terminate_del_timer(&iwqp->sc_qp); + spin_lock_irqsave(&iwqp->lock, flags); + } + info.next_iwarp_state = IRDMA_QP_STATE_ERROR; + if (iwqp->hw_tcp_state > IRDMA_TCP_STATE_CLOSED && + iwdev->iw_status && + iwqp->hw_tcp_state != IRDMA_TCP_STATE_TIME_WAIT) + info.reset_tcp_conn = true; + else + dont_wait = 1; + + issue_modify_qp = 1; + info.next_iwarp_state = IRDMA_QP_STATE_ERROR; + break; + default: + err = -EINVAL; + goto exit; + } + + iwqp->ibqp_state = attr->qp_state; + } + if (attr_mask & IB_QP_ACCESS_FLAGS) { + ctx_info->iwarp_info_valid = true; + if (attr->qp_access_flags & IB_ACCESS_LOCAL_WRITE) + offload_info->wr_rdresp_en = true; + if (attr->qp_access_flags & IB_ACCESS_REMOTE_WRITE) + offload_info->wr_rdresp_en = true; + if (attr->qp_access_flags & IB_ACCESS_REMOTE_READ) + offload_info->rd_en = true; + } + + if (ctx_info->iwarp_info_valid) { + ctx_info->send_cq_num = iwqp->iwscq->sc_cq.cq_uk.cq_id; + ctx_info->rcv_cq_num = iwqp->iwrcq->sc_cq.cq_uk.cq_id; + irdma_sc_qp_setctx(&iwqp->sc_qp, iwqp->host_ctx.va, ctx_info); + } + spin_unlock_irqrestore(&iwqp->lock, flags); + + if (attr_mask & IB_QP_STATE) { + if (issue_modify_qp) { + ctx_info->rem_endpoint_idx = tcp_info->arp_idx; + if (irdma_hw_modify_qp(iwdev, iwqp, &info, true)) + return -EINVAL; + } + + spin_lock_irqsave(&iwqp->lock, flags); + if (iwqp->iwarp_state == info.curr_iwarp_state) { + iwqp->iwarp_state = info.next_iwarp_state; + iwqp->ibqp_state = attr->qp_state; + } + spin_unlock_irqrestore(&iwqp->lock, flags); + } + + if (issue_modify_qp && iwqp->ibqp_state > IB_QPS_RTS) { + if (dont_wait) { + if (iwqp->cm_id && iwqp->hw_tcp_state) { + spin_lock_irqsave(&iwqp->lock, flags); + iwqp->hw_tcp_state = IRDMA_TCP_STATE_CLOSED; + iwqp->last_aeq = IRDMA_AE_RESET_SENT; + spin_unlock_irqrestore(&iwqp->lock, flags); + irdma_cm_disconn(iwqp); + } + } else { + int close_timer_started; + + spin_lock_irqsave(&iwdev->cm_core.ht_lock, flags); + + if (iwqp->cm_node) { + refcount_inc(&iwqp->cm_node->refcnt); + spin_unlock_irqrestore(&iwdev->cm_core.ht_lock, flags); + close_timer_started = atomic_inc_return(&iwqp->close_timer_started); + if (iwqp->cm_id && close_timer_started == 1) + irdma_schedule_cm_timer(iwqp->cm_node, + (struct irdma_puda_buf *)iwqp, + IRDMA_TIMER_TYPE_CLOSE, 1, 0); + + irdma_rem_ref_cm_node(iwqp->cm_node); + } else { + spin_unlock_irqrestore(&iwdev->cm_core.ht_lock, flags); + } + } + } + if (attr_mask & IB_QP_STATE && udata && + dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) { + struct irdma_ucontext *ucontext; + + ucontext = rdma_udata_to_drv_context(udata, + struct irdma_ucontext, ibucontext); + if (iwqp->sc_qp.push_idx != IRDMA_INVALID_PUSH_PAGE_INDEX && + !iwqp->push_wqe_mmap_entry && + !irdma_setup_push_mmap_entries(ucontext, iwqp, + &uresp.push_wqe_mmap_key, &uresp.push_db_mmap_key)) { + uresp.push_valid = 1; + uresp.push_offset = iwqp->sc_qp.push_offset; + } + + err = ib_copy_to_udata(udata, &uresp, min(sizeof(uresp), + udata->outlen)); + if (err) { + irdma_remove_push_mmap_entries(iwqp); + ibdev_dbg(&iwdev->ibdev, + "VERBS: copy_to_udata failed\n"); + return err; + } + } + + return 0; +exit: + spin_unlock_irqrestore(&iwqp->lock, flags); + + return err; +} + +/** + * irdma_cq_free_rsrc - free up resources for cq + * @rf: RDMA PCI function + * @iwcq: cq ptr + */ +static void irdma_cq_free_rsrc(struct irdma_pci_f *rf, struct irdma_cq *iwcq) +{ + struct irdma_sc_cq *cq = &iwcq->sc_cq; + + if (!iwcq->user_mode) { + dma_free_coherent(rf->sc_dev.hw->device, iwcq->kmem.size, + iwcq->kmem.va, iwcq->kmem.pa); + iwcq->kmem.va = NULL; + dma_free_coherent(rf->sc_dev.hw->device, + iwcq->kmem_shadow.size, + iwcq->kmem_shadow.va, iwcq->kmem_shadow.pa); + iwcq->kmem_shadow.va = NULL; + } + + irdma_free_rsrc(rf, rf->allocated_cqs, cq->cq_uk.cq_id); +} + +/** + * irdma_free_cqbuf - worker to free a cq buffer + * @work: provides access to the cq buffer to free + */ +static void irdma_free_cqbuf(struct work_struct *work) +{ + struct irdma_cq_buf *cq_buf = container_of(work, struct irdma_cq_buf, work); + + dma_free_coherent(cq_buf->hw->device, cq_buf->kmem_buf.size, + cq_buf->kmem_buf.va, cq_buf->kmem_buf.pa); + cq_buf->kmem_buf.va = NULL; + kfree(cq_buf); +} + +/** + * irdma_process_resize_list - remove resized cq buffers from the resize_list + * @iwcq: cq which owns the resize_list + * @iwdev: irdma device + * @lcqe_buf: the buffer where the last cqe is received + */ +static int irdma_process_resize_list(struct irdma_cq *iwcq, + struct irdma_device *iwdev, + struct irdma_cq_buf *lcqe_buf) +{ + struct list_head *tmp_node, *list_node; + struct irdma_cq_buf *cq_buf; + int cnt = 0; + + list_for_each_safe(list_node, tmp_node, &iwcq->resize_list) { + cq_buf = list_entry(list_node, struct irdma_cq_buf, list); + if (cq_buf == lcqe_buf) + return cnt; + + list_del(&cq_buf->list); + queue_work(iwdev->cleanup_wq, &cq_buf->work); + cnt++; + } + + return cnt; +} + +/** + * irdma_destroy_cq - destroy cq + * @ib_cq: cq pointer + * @udata: user data + */ +static int irdma_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata) +{ + struct irdma_device *iwdev = to_iwdev(ib_cq->device); + struct irdma_cq *iwcq = to_iwcq(ib_cq); + struct irdma_sc_cq *cq = &iwcq->sc_cq; + struct irdma_sc_dev *dev = cq->dev; + struct irdma_sc_ceq *ceq = dev->ceq[cq->ceq_id]; + struct irdma_ceq *iwceq = container_of(ceq, struct irdma_ceq, sc_ceq); + unsigned long flags; + + spin_lock_irqsave(&iwcq->lock, flags); + if (!list_empty(&iwcq->resize_list)) + irdma_process_resize_list(iwcq, iwdev, NULL); + spin_unlock_irqrestore(&iwcq->lock, flags); + + irdma_cq_wq_destroy(iwdev->rf, cq); + irdma_cq_free_rsrc(iwdev->rf, iwcq); + + spin_lock_irqsave(&iwceq->ce_lock, flags); + irdma_sc_cleanup_ceqes(cq, ceq); + spin_unlock_irqrestore(&iwceq->ce_lock, flags); + + return 0; +} + +/** + * irdma_resize_cq - resize cq + * @ibcq: cq to be resized + * @entries: desired cq size + * @udata: user data + */ +static int irdma_resize_cq(struct ib_cq *ibcq, int entries, + struct ib_udata *udata) +{ + struct irdma_cq *iwcq = to_iwcq(ibcq); + struct irdma_sc_dev *dev = iwcq->sc_cq.dev; + struct irdma_cqp_request *cqp_request; + struct cqp_cmds_info *cqp_info; + struct irdma_modify_cq_info *m_info; + struct irdma_modify_cq_info info = {}; + struct irdma_dma_mem kmem_buf; + struct irdma_cq_mr *cqmr_buf; + struct irdma_pbl *iwpbl_buf; + struct irdma_device *iwdev; + struct irdma_pci_f *rf; + struct irdma_cq_buf *cq_buf = NULL; + enum irdma_status_code status = 0; + unsigned long flags; + int ret; + + iwdev = to_iwdev(ibcq->device); + rf = iwdev->rf; + + if (!(rf->sc_dev.hw_attrs.uk_attrs.feature_flags & + IRDMA_FEATURE_CQ_RESIZE)) + return -EOPNOTSUPP; + + if (entries > rf->max_cqe) + return -EINVAL; + + if (!iwcq->user_mode) { + entries++; + if (rf->sc_dev.hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) + entries *= 2; + } + + info.cq_size = max(entries, 4); + + if (info.cq_size == iwcq->sc_cq.cq_uk.cq_size - 1) + return 0; + + if (udata) { + struct irdma_resize_cq_req req = {}; + struct irdma_ucontext *ucontext = + rdma_udata_to_drv_context(udata, struct irdma_ucontext, + ibucontext); + + /* CQ resize not supported with legacy GEN_1 libi40iw */ + if (ucontext->legacy_mode) + return -EOPNOTSUPP; + + if (ib_copy_from_udata(&req, udata, + min(sizeof(req), udata->inlen))) + return -EINVAL; + + spin_lock_irqsave(&ucontext->cq_reg_mem_list_lock, flags); + iwpbl_buf = irdma_get_pbl((unsigned long)req.user_cq_buffer, + &ucontext->cq_reg_mem_list); + spin_unlock_irqrestore(&ucontext->cq_reg_mem_list_lock, flags); + + if (!iwpbl_buf) + return -ENOMEM; + + cqmr_buf = &iwpbl_buf->cq_mr; + if (iwpbl_buf->pbl_allocated) { + info.virtual_map = true; + info.pbl_chunk_size = 1; + info.first_pm_pbl_idx = cqmr_buf->cq_pbl.idx; + } else { + info.cq_pa = cqmr_buf->cq_pbl.addr; + } + } else { + /* Kmode CQ resize */ + int rsize; + + rsize = info.cq_size * sizeof(struct irdma_cqe); + kmem_buf.size = ALIGN(round_up(rsize, 256), 256); + kmem_buf.va = dma_alloc_coherent(dev->hw->device, + kmem_buf.size, &kmem_buf.pa, + GFP_KERNEL); + if (!kmem_buf.va) + return -ENOMEM; + + info.cq_base = kmem_buf.va; + info.cq_pa = kmem_buf.pa; + cq_buf = kzalloc(sizeof(*cq_buf), GFP_KERNEL); + if (!cq_buf) { + ret = -ENOMEM; + goto error; + } + } + + cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true); + if (!cqp_request) { + ret = -ENOMEM; + goto error; + } + + info.shadow_read_threshold = iwcq->sc_cq.shadow_read_threshold; + info.cq_resize = true; + + cqp_info = &cqp_request->info; + m_info = &cqp_info->in.u.cq_modify.info; + memcpy(m_info, &info, sizeof(*m_info)); + + cqp_info->cqp_cmd = IRDMA_OP_CQ_MODIFY; + cqp_info->in.u.cq_modify.cq = &iwcq->sc_cq; + cqp_info->in.u.cq_modify.scratch = (uintptr_t)cqp_request; + cqp_info->post_sq = 1; + status = irdma_handle_cqp_op(rf, cqp_request); + irdma_put_cqp_request(&rf->cqp, cqp_request); + if (status) { + ret = -EPROTO; + goto error; + } + + spin_lock_irqsave(&iwcq->lock, flags); + if (cq_buf) { + cq_buf->kmem_buf = iwcq->kmem; + cq_buf->hw = dev->hw; + memcpy(&cq_buf->cq_uk, &iwcq->sc_cq.cq_uk, sizeof(cq_buf->cq_uk)); + INIT_WORK(&cq_buf->work, irdma_free_cqbuf); + list_add_tail(&cq_buf->list, &iwcq->resize_list); + iwcq->kmem = kmem_buf; + } + + irdma_sc_cq_resize(&iwcq->sc_cq, &info); + ibcq->cqe = info.cq_size - 1; + spin_unlock_irqrestore(&iwcq->lock, flags); + + return 0; +error: + if (!udata) { + dma_free_coherent(dev->hw->device, kmem_buf.size, kmem_buf.va, + kmem_buf.pa); + kmem_buf.va = NULL; + } + kfree(cq_buf); + + return ret; +} + +static inline int cq_validate_flags(u32 flags, u8 hw_rev) +{ + /* GEN1 does not support CQ create flags */ + if (hw_rev == IRDMA_GEN_1) + return flags ? -EOPNOTSUPP : 0; + + return flags & ~IB_UVERBS_CQ_FLAGS_TIMESTAMP_COMPLETION ? -EOPNOTSUPP : 0; +} + +/** + * irdma_create_cq - create cq + * @ibcq: CQ allocated + * @attr: attributes for cq + * @udata: user data + */ +static int irdma_create_cq(struct ib_cq *ibcq, + const struct ib_cq_init_attr *attr, + struct ib_udata *udata) +{ + struct ib_device *ibdev = ibcq->device; + struct irdma_device *iwdev = to_iwdev(ibdev); + struct irdma_pci_f *rf = iwdev->rf; + struct irdma_cq *iwcq = to_iwcq(ibcq); + u32 cq_num = 0; + struct irdma_sc_cq *cq; + struct irdma_sc_dev *dev = &rf->sc_dev; + struct irdma_cq_init_info info = {}; + enum irdma_status_code status; + struct irdma_cqp_request *cqp_request; + struct cqp_cmds_info *cqp_info; + struct irdma_cq_uk_init_info *ukinfo = &info.cq_uk_init_info; + unsigned long flags; + int err_code; + int entries = attr->cqe; + + err_code = cq_validate_flags(attr->flags, dev->hw_attrs.uk_attrs.hw_rev); + if (err_code) + return err_code; + err_code = irdma_alloc_rsrc(rf, rf->allocated_cqs, rf->max_cq, &cq_num, + &rf->next_cq); + if (err_code) + return err_code; + + cq = &iwcq->sc_cq; + cq->back_cq = iwcq; + spin_lock_init(&iwcq->lock); + INIT_LIST_HEAD(&iwcq->resize_list); + info.dev = dev; + ukinfo->cq_size = max(entries, 4); + ukinfo->cq_id = cq_num; + iwcq->ibcq.cqe = info.cq_uk_init_info.cq_size; + if (attr->comp_vector < rf->ceqs_count) + info.ceq_id = attr->comp_vector; + info.ceq_id_valid = true; + info.ceqe_mask = 1; + info.type = IRDMA_CQ_TYPE_IWARP; + info.vsi = &iwdev->vsi; + + if (udata) { + struct irdma_ucontext *ucontext; + struct irdma_create_cq_req req = {}; + struct irdma_cq_mr *cqmr; + struct irdma_pbl *iwpbl; + struct irdma_pbl *iwpbl_shadow; + struct irdma_cq_mr *cqmr_shadow; + + iwcq->user_mode = true; + ucontext = + rdma_udata_to_drv_context(udata, struct irdma_ucontext, + ibucontext); + if (ib_copy_from_udata(&req, udata, + min(sizeof(req), udata->inlen))) { + err_code = -EFAULT; + goto cq_free_rsrc; + } + + spin_lock_irqsave(&ucontext->cq_reg_mem_list_lock, flags); + iwpbl = irdma_get_pbl((unsigned long)req.user_cq_buf, + &ucontext->cq_reg_mem_list); + spin_unlock_irqrestore(&ucontext->cq_reg_mem_list_lock, flags); + if (!iwpbl) { + err_code = -EPROTO; + goto cq_free_rsrc; + } + + iwcq->iwpbl = iwpbl; + iwcq->cq_mem_size = 0; + cqmr = &iwpbl->cq_mr; + + if (rf->sc_dev.hw_attrs.uk_attrs.feature_flags & + IRDMA_FEATURE_CQ_RESIZE && !ucontext->legacy_mode) { + spin_lock_irqsave(&ucontext->cq_reg_mem_list_lock, flags); + iwpbl_shadow = irdma_get_pbl( + (unsigned long)req.user_shadow_area, + &ucontext->cq_reg_mem_list); + spin_unlock_irqrestore(&ucontext->cq_reg_mem_list_lock, flags); + + if (!iwpbl_shadow) { + err_code = -EPROTO; + goto cq_free_rsrc; + } + iwcq->iwpbl_shadow = iwpbl_shadow; + cqmr_shadow = &iwpbl_shadow->cq_mr; + info.shadow_area_pa = cqmr_shadow->cq_pbl.addr; + cqmr->split = true; + } else { + info.shadow_area_pa = cqmr->shadow; + } + if (iwpbl->pbl_allocated) { + info.virtual_map = true; + info.pbl_chunk_size = 1; + info.first_pm_pbl_idx = cqmr->cq_pbl.idx; + } else { + info.cq_base_pa = cqmr->cq_pbl.addr; + } + } else { + /* Kmode allocations */ + int rsize; + + if (entries > rf->max_cqe) { + err_code = -EINVAL; + goto cq_free_rsrc; + } + + entries++; + if (dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) + entries *= 2; + ukinfo->cq_size = entries; + + rsize = info.cq_uk_init_info.cq_size * sizeof(struct irdma_cqe); + iwcq->kmem.size = ALIGN(round_up(rsize, 256), 256); + iwcq->kmem.va = dma_alloc_coherent(dev->hw->device, + iwcq->kmem.size, + &iwcq->kmem.pa, GFP_KERNEL); + if (!iwcq->kmem.va) { + err_code = -ENOMEM; + goto cq_free_rsrc; + } + + iwcq->kmem_shadow.size = ALIGN(IRDMA_SHADOW_AREA_SIZE << 3, + 64); + iwcq->kmem_shadow.va = dma_alloc_coherent(dev->hw->device, + iwcq->kmem_shadow.size, + &iwcq->kmem_shadow.pa, + GFP_KERNEL); + if (!iwcq->kmem_shadow.va) { + err_code = -ENOMEM; + goto cq_free_rsrc; + } + info.shadow_area_pa = iwcq->kmem_shadow.pa; + ukinfo->shadow_area = iwcq->kmem_shadow.va; + ukinfo->cq_base = iwcq->kmem.va; + info.cq_base_pa = iwcq->kmem.pa; + } + + if (dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) + info.shadow_read_threshold = min(info.cq_uk_init_info.cq_size / 2, + (u32)IRDMA_MAX_CQ_READ_THRESH); + + if (irdma_sc_cq_init(cq, &info)) { + ibdev_dbg(&iwdev->ibdev, "VERBS: init cq fail\n"); + err_code = -EPROTO; + goto cq_free_rsrc; + } + + cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true); + if (!cqp_request) { + err_code = -ENOMEM; + goto cq_free_rsrc; + } + + cqp_info = &cqp_request->info; + cqp_info->cqp_cmd = IRDMA_OP_CQ_CREATE; + cqp_info->post_sq = 1; + cqp_info->in.u.cq_create.cq = cq; + cqp_info->in.u.cq_create.check_overflow = true; + cqp_info->in.u.cq_create.scratch = (uintptr_t)cqp_request; + status = irdma_handle_cqp_op(rf, cqp_request); + irdma_put_cqp_request(&rf->cqp, cqp_request); + if (status) { + err_code = -ENOMEM; + goto cq_free_rsrc; + } + + if (udata) { + struct irdma_create_cq_resp resp = {}; + + resp.cq_id = info.cq_uk_init_info.cq_id; + resp.cq_size = info.cq_uk_init_info.cq_size; + if (ib_copy_to_udata(udata, &resp, + min(sizeof(resp), udata->outlen))) { + ibdev_dbg(&iwdev->ibdev, + "VERBS: copy to user data\n"); + err_code = -EPROTO; + goto cq_destroy; + } + } + return 0; +cq_destroy: + irdma_cq_wq_destroy(rf, cq); +cq_free_rsrc: + irdma_cq_free_rsrc(rf, iwcq); + + return err_code; +} + +/** + * irdma_get_mr_access - get hw MR access permissions from IB access flags + * @access: IB access flags + */ +static inline u16 irdma_get_mr_access(int access) +{ + u16 hw_access = 0; + + hw_access |= (access & IB_ACCESS_LOCAL_WRITE) ? + IRDMA_ACCESS_FLAGS_LOCALWRITE : 0; + hw_access |= (access & IB_ACCESS_REMOTE_WRITE) ? + IRDMA_ACCESS_FLAGS_REMOTEWRITE : 0; + hw_access |= (access & IB_ACCESS_REMOTE_READ) ? + IRDMA_ACCESS_FLAGS_REMOTEREAD : 0; + hw_access |= (access & IB_ACCESS_MW_BIND) ? + IRDMA_ACCESS_FLAGS_BIND_WINDOW : 0; + hw_access |= (access & IB_ZERO_BASED) ? + IRDMA_ACCESS_FLAGS_ZERO_BASED : 0; + hw_access |= IRDMA_ACCESS_FLAGS_LOCALREAD; + + return hw_access; +} + +/** + * irdma_free_stag - free stag resource + * @iwdev: irdma device + * @stag: stag to free + */ +static void irdma_free_stag(struct irdma_device *iwdev, u32 stag) +{ + u32 stag_idx; + + stag_idx = (stag & iwdev->rf->mr_stagmask) >> IRDMA_CQPSQ_STAG_IDX_S; + irdma_free_rsrc(iwdev->rf, iwdev->rf->allocated_mrs, stag_idx); +} + +/** + * irdma_create_stag - create random stag + * @iwdev: irdma device + */ +static u32 irdma_create_stag(struct irdma_device *iwdev) +{ + u32 stag = 0; + u32 stag_index = 0; + u32 next_stag_index; + u32 driver_key; + u32 random; + u8 consumer_key; + int ret; + + get_random_bytes(&random, sizeof(random)); + consumer_key = (u8)random; + + driver_key = random & ~iwdev->rf->mr_stagmask; + next_stag_index = (random & iwdev->rf->mr_stagmask) >> 8; + next_stag_index %= iwdev->rf->max_mr; + + ret = irdma_alloc_rsrc(iwdev->rf, iwdev->rf->allocated_mrs, + iwdev->rf->max_mr, &stag_index, + &next_stag_index); + if (ret) + return stag; + stag = stag_index << IRDMA_CQPSQ_STAG_IDX_S; + stag |= driver_key; + stag += (u32)consumer_key; + + return stag; +} + +/** + * irdma_next_pbl_addr - Get next pbl address + * @pbl: pointer to a pble + * @pinfo: info pointer + * @idx: index + */ +static inline u64 *irdma_next_pbl_addr(u64 *pbl, struct irdma_pble_info **pinfo, + u32 *idx) +{ + *idx += 1; + if (!(*pinfo) || *idx != (*pinfo)->cnt) + return ++pbl; + *idx = 0; + (*pinfo)++; + + return (u64 *)(uintptr_t)(*pinfo)->addr; +} + +/** + * irdma_copy_user_pgaddrs - copy user page address to pble's os locally + * @iwmr: iwmr for IB's user page addresses + * @pbl: ple pointer to save 1 level or 0 level pble + * @level: indicated level 0, 1 or 2 + */ +static void irdma_copy_user_pgaddrs(struct irdma_mr *iwmr, u64 *pbl, + enum irdma_pble_level level) +{ + struct ib_umem *region = iwmr->region; + struct irdma_pbl *iwpbl = &iwmr->iwpbl; + struct irdma_pble_alloc *palloc = &iwpbl->pble_alloc; + struct irdma_pble_info *pinfo; + struct ib_block_iter biter; + u32 idx = 0; + u32 pbl_cnt = 0; + + pinfo = (level == PBLE_LEVEL_1) ? NULL : palloc->level2.leaf; + + if (iwmr->type == IRDMA_MEMREG_TYPE_QP) + iwpbl->qp_mr.sq_page = sg_page(region->sg_head.sgl); + + rdma_umem_for_each_dma_block(region, &biter, iwmr->page_size) { + *pbl = rdma_block_iter_dma_address(&biter); + if (++pbl_cnt == palloc->total_cnt) + break; + pbl = irdma_next_pbl_addr(pbl, &pinfo, &idx); + } +} + +/** + * irdma_check_mem_contiguous - check if pbls stored in arr are contiguous + * @arr: lvl1 pbl array + * @npages: page count + * @pg_size: page size + * + */ +static bool irdma_check_mem_contiguous(u64 *arr, u32 npages, u32 pg_size) +{ + u32 pg_idx; + + for (pg_idx = 0; pg_idx < npages; pg_idx++) { + if ((*arr + (pg_size * pg_idx)) != arr[pg_idx]) + return false; + } + + return true; +} + +/** + * irdma_check_mr_contiguous - check if MR is physically contiguous + * @palloc: pbl allocation struct + * @pg_size: page size + */ +static bool irdma_check_mr_contiguous(struct irdma_pble_alloc *palloc, + u32 pg_size) +{ + struct irdma_pble_level2 *lvl2 = &palloc->level2; + struct irdma_pble_info *leaf = lvl2->leaf; + u64 *arr = NULL; + u64 *start_addr = NULL; + int i; + bool ret; + + if (palloc->level == PBLE_LEVEL_1) { + arr = (u64 *)(uintptr_t)palloc->level1.addr; + ret = irdma_check_mem_contiguous(arr, palloc->total_cnt, + pg_size); + return ret; + } + + start_addr = (u64 *)(uintptr_t)leaf->addr; + + for (i = 0; i < lvl2->leaf_cnt; i++, leaf++) { + arr = (u64 *)(uintptr_t)leaf->addr; + if ((*start_addr + (i * pg_size * PBLE_PER_PAGE)) != *arr) + return false; + ret = irdma_check_mem_contiguous(arr, leaf->cnt, pg_size); + if (!ret) + return false; + } + + return true; +} + +/** + * irdma_setup_pbles - copy user pg address to pble's + * @rf: RDMA PCI function + * @iwmr: mr pointer for this memory registration + * @use_pbles: flag if to use pble's + */ +static int irdma_setup_pbles(struct irdma_pci_f *rf, struct irdma_mr *iwmr, + bool use_pbles) +{ + struct irdma_pbl *iwpbl = &iwmr->iwpbl; + struct irdma_pble_alloc *palloc = &iwpbl->pble_alloc; + struct irdma_pble_info *pinfo; + u64 *pbl; + enum irdma_status_code status; + enum irdma_pble_level level = PBLE_LEVEL_1; + + if (use_pbles) { + status = irdma_get_pble(rf->pble_rsrc, palloc, iwmr->page_cnt, + false); + if (status) + return -ENOMEM; + + iwpbl->pbl_allocated = true; + level = palloc->level; + pinfo = (level == PBLE_LEVEL_1) ? &palloc->level1 : + palloc->level2.leaf; + pbl = (u64 *)(uintptr_t)pinfo->addr; + } else { + pbl = iwmr->pgaddrmem; + } + + irdma_copy_user_pgaddrs(iwmr, pbl, level); + + if (use_pbles) + iwmr->pgaddrmem[0] = *pbl; + + return 0; +} + +/** + * irdma_handle_q_mem - handle memory for qp and cq + * @iwdev: irdma device + * @req: information for q memory management + * @iwpbl: pble struct + * @use_pbles: flag to use pble + */ +static int irdma_handle_q_mem(struct irdma_device *iwdev, + struct irdma_mem_reg_req *req, + struct irdma_pbl *iwpbl, bool use_pbles) +{ + struct irdma_pble_alloc *palloc = &iwpbl->pble_alloc; + struct irdma_mr *iwmr = iwpbl->iwmr; + struct irdma_qp_mr *qpmr = &iwpbl->qp_mr; + struct irdma_cq_mr *cqmr = &iwpbl->cq_mr; + struct irdma_hmc_pble *hmc_p; + u64 *arr = iwmr->pgaddrmem; + u32 pg_size; + int err = 0; + int total; + bool ret = true; + + total = req->sq_pages + req->rq_pages + req->cq_pages; + pg_size = iwmr->page_size; + err = irdma_setup_pbles(iwdev->rf, iwmr, use_pbles); + if (err) + return err; + + if (use_pbles && palloc->level != PBLE_LEVEL_1) { + irdma_free_pble(iwdev->rf->pble_rsrc, palloc); + iwpbl->pbl_allocated = false; + return -ENOMEM; + } + + if (use_pbles) + arr = (u64 *)(uintptr_t)palloc->level1.addr; + + switch (iwmr->type) { + case IRDMA_MEMREG_TYPE_QP: + hmc_p = &qpmr->sq_pbl; + qpmr->shadow = (dma_addr_t)arr[total]; + + if (use_pbles) { + ret = irdma_check_mem_contiguous(arr, req->sq_pages, + pg_size); + if (ret) + ret = irdma_check_mem_contiguous(&arr[req->sq_pages], + req->rq_pages, + pg_size); + } + + if (!ret) { + hmc_p->idx = palloc->level1.idx; + hmc_p = &qpmr->rq_pbl; + hmc_p->idx = palloc->level1.idx + req->sq_pages; + } else { + hmc_p->addr = arr[0]; + hmc_p = &qpmr->rq_pbl; + hmc_p->addr = arr[req->sq_pages]; + } + break; + case IRDMA_MEMREG_TYPE_CQ: + hmc_p = &cqmr->cq_pbl; + + if (!cqmr->split) + cqmr->shadow = (dma_addr_t)arr[total]; + + if (use_pbles) + ret = irdma_check_mem_contiguous(arr, req->cq_pages, + pg_size); + + if (!ret) + hmc_p->idx = palloc->level1.idx; + else + hmc_p->addr = arr[0]; + break; + default: + ibdev_dbg(&iwdev->ibdev, "VERBS: MR type error\n"); + err = -EINVAL; + } + + if (use_pbles && ret) { + irdma_free_pble(iwdev->rf->pble_rsrc, palloc); + iwpbl->pbl_allocated = false; + } + + return err; +} + +/** + * irdma_hw_alloc_mw - create the hw memory window + * @iwdev: irdma device + * @iwmr: pointer to memory window info + */ +static int irdma_hw_alloc_mw(struct irdma_device *iwdev, struct irdma_mr *iwmr) +{ + struct irdma_mw_alloc_info *info; + struct irdma_pd *iwpd = to_iwpd(iwmr->ibmr.pd); + struct irdma_cqp_request *cqp_request; + struct cqp_cmds_info *cqp_info; + enum irdma_status_code status; + + cqp_request = irdma_alloc_and_get_cqp_request(&iwdev->rf->cqp, true); + if (!cqp_request) + return -ENOMEM; + + cqp_info = &cqp_request->info; + info = &cqp_info->in.u.mw_alloc.info; + memset(info, 0, sizeof(*info)); + if (iwmr->ibmw.type == IB_MW_TYPE_1) + info->mw_wide = true; + + info->page_size = PAGE_SIZE; + info->mw_stag_index = iwmr->stag >> IRDMA_CQPSQ_STAG_IDX_S; + info->pd_id = iwpd->sc_pd.pd_id; + info->remote_access = true; + cqp_info->cqp_cmd = IRDMA_OP_MW_ALLOC; + cqp_info->post_sq = 1; + cqp_info->in.u.mw_alloc.dev = &iwdev->rf->sc_dev; + cqp_info->in.u.mw_alloc.scratch = (uintptr_t)cqp_request; + status = irdma_handle_cqp_op(iwdev->rf, cqp_request); + irdma_put_cqp_request(&iwdev->rf->cqp, cqp_request); + + return status ? -ENOMEM : 0; +} + +/** + * irdma_alloc_mw - Allocate memory window + * @ibmw: Memory Window + * @udata: user data pointer + */ +static int irdma_alloc_mw(struct ib_mw *ibmw, struct ib_udata *udata) +{ + struct irdma_device *iwdev = to_iwdev(ibmw->device); + struct irdma_mr *iwmr = to_iwmw(ibmw); + int err_code; + u32 stag; + + stag = irdma_create_stag(iwdev); + if (!stag) + return -ENOMEM; + + iwmr->stag = stag; + ibmw->rkey = stag; + + err_code = irdma_hw_alloc_mw(iwdev, iwmr); + if (err_code) { + irdma_free_stag(iwdev, stag); + return err_code; + } + + return 0; +} + +/** + * irdma_dealloc_mw - Dealloc memory window + * @ibmw: memory window structure. + */ +static int irdma_dealloc_mw(struct ib_mw *ibmw) +{ + struct ib_pd *ibpd = ibmw->pd; + struct irdma_pd *iwpd = to_iwpd(ibpd); + struct irdma_mr *iwmr = to_iwmr((struct ib_mr *)ibmw); + struct irdma_device *iwdev = to_iwdev(ibmw->device); + struct irdma_cqp_request *cqp_request; + struct cqp_cmds_info *cqp_info; + struct irdma_dealloc_stag_info *info; + + cqp_request = irdma_alloc_and_get_cqp_request(&iwdev->rf->cqp, true); + if (!cqp_request) + return -ENOMEM; + + cqp_info = &cqp_request->info; + info = &cqp_info->in.u.dealloc_stag.info; + memset(info, 0, sizeof(*info)); + info->pd_id = iwpd->sc_pd.pd_id & 0x00007fff; + info->stag_idx = ibmw->rkey >> IRDMA_CQPSQ_STAG_IDX_S; + info->mr = false; + cqp_info->cqp_cmd = IRDMA_OP_DEALLOC_STAG; + cqp_info->post_sq = 1; + cqp_info->in.u.dealloc_stag.dev = &iwdev->rf->sc_dev; + cqp_info->in.u.dealloc_stag.scratch = (uintptr_t)cqp_request; + irdma_handle_cqp_op(iwdev->rf, cqp_request); + irdma_put_cqp_request(&iwdev->rf->cqp, cqp_request); + irdma_free_stag(iwdev, iwmr->stag); + + return 0; +} + +/** + * irdma_hw_alloc_stag - cqp command to allocate stag + * @iwdev: irdma device + * @iwmr: irdma mr pointer + */ +static int irdma_hw_alloc_stag(struct irdma_device *iwdev, + struct irdma_mr *iwmr) +{ + struct irdma_allocate_stag_info *info; + struct irdma_pd *iwpd = to_iwpd(iwmr->ibmr.pd); + enum irdma_status_code status; + int err = 0; + struct irdma_cqp_request *cqp_request; + struct cqp_cmds_info *cqp_info; + + cqp_request = irdma_alloc_and_get_cqp_request(&iwdev->rf->cqp, true); + if (!cqp_request) + return -ENOMEM; + + cqp_info = &cqp_request->info; + info = &cqp_info->in.u.alloc_stag.info; + memset(info, 0, sizeof(*info)); + info->page_size = PAGE_SIZE; + info->stag_idx = iwmr->stag >> IRDMA_CQPSQ_STAG_IDX_S; + info->pd_id = iwpd->sc_pd.pd_id; + info->total_len = iwmr->len; + info->remote_access = true; + cqp_info->cqp_cmd = IRDMA_OP_ALLOC_STAG; + cqp_info->post_sq = 1; + cqp_info->in.u.alloc_stag.dev = &iwdev->rf->sc_dev; + cqp_info->in.u.alloc_stag.scratch = (uintptr_t)cqp_request; + status = irdma_handle_cqp_op(iwdev->rf, cqp_request); + irdma_put_cqp_request(&iwdev->rf->cqp, cqp_request); + if (status) + err = -ENOMEM; + + return err; +} + +/** + * irdma_alloc_mr - register stag for fast memory registration + * @pd: ibpd pointer + * @mr_type: memory for stag registrion + * @max_num_sg: man number of pages + */ +static struct ib_mr *irdma_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type, + u32 max_num_sg) +{ + struct irdma_device *iwdev = to_iwdev(pd->device); + struct irdma_pble_alloc *palloc; + struct irdma_pbl *iwpbl; + struct irdma_mr *iwmr; + enum irdma_status_code status; + u32 stag; + int err_code = -ENOMEM; + + iwmr = kzalloc(sizeof(*iwmr), GFP_KERNEL); + if (!iwmr) + return ERR_PTR(-ENOMEM); + + stag = irdma_create_stag(iwdev); + if (!stag) { + err_code = -ENOMEM; + goto err; + } + + iwmr->stag = stag; + iwmr->ibmr.rkey = stag; + iwmr->ibmr.lkey = stag; + iwmr->ibmr.pd = pd; + iwmr->ibmr.device = pd->device; + iwpbl = &iwmr->iwpbl; + iwpbl->iwmr = iwmr; + iwmr->type = IRDMA_MEMREG_TYPE_MEM; + palloc = &iwpbl->pble_alloc; + iwmr->page_cnt = max_num_sg; + status = irdma_get_pble(iwdev->rf->pble_rsrc, palloc, iwmr->page_cnt, + true); + if (status) + goto err_get_pble; + + err_code = irdma_hw_alloc_stag(iwdev, iwmr); + if (err_code) + goto err_alloc_stag; + + iwpbl->pbl_allocated = true; + + return &iwmr->ibmr; +err_alloc_stag: + irdma_free_pble(iwdev->rf->pble_rsrc, palloc); +err_get_pble: + irdma_free_stag(iwdev, stag); +err: + kfree(iwmr); + + return ERR_PTR(err_code); +} + +/** + * irdma_set_page - populate pbl list for fmr + * @ibmr: ib mem to access iwarp mr pointer + * @addr: page dma address fro pbl list + */ +static int irdma_set_page(struct ib_mr *ibmr, u64 addr) +{ + struct irdma_mr *iwmr = to_iwmr(ibmr); + struct irdma_pbl *iwpbl = &iwmr->iwpbl; + struct irdma_pble_alloc *palloc = &iwpbl->pble_alloc; + u64 *pbl; + + if (unlikely(iwmr->npages == iwmr->page_cnt)) + return -ENOMEM; + + pbl = (u64 *)(uintptr_t)palloc->level1.addr; + pbl[iwmr->npages++] = addr; + + return 0; +} + +/** + * irdma_map_mr_sg - map of sg list for fmr + * @ibmr: ib mem to access iwarp mr pointer + * @sg: scatter gather list + * @sg_nents: number of sg pages + * @sg_offset: scatter gather list for fmr + */ +static int irdma_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, + int sg_nents, unsigned int *sg_offset) +{ + struct irdma_mr *iwmr = to_iwmr(ibmr); + + iwmr->npages = 0; + + return ib_sg_to_pages(ibmr, sg, sg_nents, sg_offset, irdma_set_page); +} + +/** + * irdma_hwreg_mr - send cqp command for memory registration + * @iwdev: irdma device + * @iwmr: irdma mr pointer + * @access: access for MR + */ +static int irdma_hwreg_mr(struct irdma_device *iwdev, struct irdma_mr *iwmr, + u16 access) +{ + struct irdma_pbl *iwpbl = &iwmr->iwpbl; + struct irdma_reg_ns_stag_info *stag_info; + struct irdma_pd *iwpd = to_iwpd(iwmr->ibmr.pd); + struct irdma_pble_alloc *palloc = &iwpbl->pble_alloc; + enum irdma_status_code status; + int err = 0; + struct irdma_cqp_request *cqp_request; + struct cqp_cmds_info *cqp_info; + + cqp_request = irdma_alloc_and_get_cqp_request(&iwdev->rf->cqp, true); + if (!cqp_request) + return -ENOMEM; + + cqp_info = &cqp_request->info; + stag_info = &cqp_info->in.u.mr_reg_non_shared.info; + memset(stag_info, 0, sizeof(*stag_info)); + stag_info->va = iwpbl->user_base; + stag_info->stag_idx = iwmr->stag >> IRDMA_CQPSQ_STAG_IDX_S; + stag_info->stag_key = (u8)iwmr->stag; + stag_info->total_len = iwmr->len; + stag_info->access_rights = irdma_get_mr_access(access); + stag_info->pd_id = iwpd->sc_pd.pd_id; + if (stag_info->access_rights & IRDMA_ACCESS_FLAGS_ZERO_BASED) + stag_info->addr_type = IRDMA_ADDR_TYPE_ZERO_BASED; + else + stag_info->addr_type = IRDMA_ADDR_TYPE_VA_BASED; + stag_info->page_size = iwmr->page_size; + + if (iwpbl->pbl_allocated) { + if (palloc->level == PBLE_LEVEL_1) { + stag_info->first_pm_pbl_index = palloc->level1.idx; + stag_info->chunk_size = 1; + } else { + stag_info->first_pm_pbl_index = palloc->level2.root.idx; + stag_info->chunk_size = 3; + } + } else { + stag_info->reg_addr_pa = iwmr->pgaddrmem[0]; + } + + cqp_info->cqp_cmd = IRDMA_OP_MR_REG_NON_SHARED; + cqp_info->post_sq = 1; + cqp_info->in.u.mr_reg_non_shared.dev = &iwdev->rf->sc_dev; + cqp_info->in.u.mr_reg_non_shared.scratch = (uintptr_t)cqp_request; + status = irdma_handle_cqp_op(iwdev->rf, cqp_request); + irdma_put_cqp_request(&iwdev->rf->cqp, cqp_request); + if (status) + err = -ENOMEM; + + return err; +} + +/** + * irdma_reg_user_mr - Register a user memory region + * @pd: ptr of pd + * @start: virtual start address + * @len: length of mr + * @virt: virtual address + * @access: access of mr + * @udata: user data + */ +static struct ib_mr *irdma_reg_user_mr(struct ib_pd *pd, u64 start, u64 len, + u64 virt, int access, + struct ib_udata *udata) +{ + struct irdma_device *iwdev = to_iwdev(pd->device); + struct irdma_ucontext *ucontext; + struct irdma_pble_alloc *palloc; + struct irdma_pbl *iwpbl; + struct irdma_mr *iwmr; + struct ib_umem *region; + struct irdma_mem_reg_req req; + u32 stag = 0; + bool use_pbles = false; + unsigned long flags; + int err = -EINVAL; + int ret; + + if (len > iwdev->rf->sc_dev.hw_attrs.max_mr_size) + return ERR_PTR(-EINVAL); + + region = ib_umem_get(pd->device, start, len, access); + + if (IS_ERR(region)) { + ibdev_dbg(&iwdev->ibdev, + "VERBS: Failed to create ib_umem region\n"); + return (struct ib_mr *)region; + } + + if (ib_copy_from_udata(&req, udata, min(sizeof(req), udata->inlen))) { + ib_umem_release(region); + return ERR_PTR(-EFAULT); + } + + iwmr = kzalloc(sizeof(*iwmr), GFP_KERNEL); + if (!iwmr) { + ib_umem_release(region); + return ERR_PTR(-ENOMEM); + } + + iwpbl = &iwmr->iwpbl; + iwpbl->iwmr = iwmr; + iwmr->region = region; + iwmr->ibmr.pd = pd; + iwmr->ibmr.device = pd->device; + iwmr->ibmr.iova = virt; + iwmr->page_size = PAGE_SIZE; + + if (req.reg_type == IRDMA_MEMREG_TYPE_MEM) + iwmr->page_size = ib_umem_find_best_pgsz(region, + SZ_4K | SZ_2M | SZ_1G, + virt); + iwmr->len = region->length; + iwpbl->user_base = virt; + palloc = &iwpbl->pble_alloc; + iwmr->type = req.reg_type; + iwmr->page_cnt = ib_umem_num_dma_blocks(region, iwmr->page_size); + + switch (req.reg_type) { + case IRDMA_MEMREG_TYPE_QP: + use_pbles = ((req.sq_pages + req.rq_pages) > 2); + err = irdma_handle_q_mem(iwdev, &req, iwpbl, use_pbles); + if (err) + goto error; + + ucontext = rdma_udata_to_drv_context(udata, struct irdma_ucontext, + ibucontext); + spin_lock_irqsave(&ucontext->qp_reg_mem_list_lock, flags); + list_add_tail(&iwpbl->list, &ucontext->qp_reg_mem_list); + iwpbl->on_list = true; + spin_unlock_irqrestore(&ucontext->qp_reg_mem_list_lock, flags); + break; + case IRDMA_MEMREG_TYPE_CQ: + use_pbles = (req.cq_pages > 1); + err = irdma_handle_q_mem(iwdev, &req, iwpbl, use_pbles); + if (err) + goto error; + + ucontext = rdma_udata_to_drv_context(udata, struct irdma_ucontext, + ibucontext); + spin_lock_irqsave(&ucontext->cq_reg_mem_list_lock, flags); + list_add_tail(&iwpbl->list, &ucontext->cq_reg_mem_list); + iwpbl->on_list = true; + spin_unlock_irqrestore(&ucontext->cq_reg_mem_list_lock, flags); + break; + case IRDMA_MEMREG_TYPE_MEM: + use_pbles = (iwmr->page_cnt != 1); + + err = irdma_setup_pbles(iwdev->rf, iwmr, use_pbles); + if (err) + goto error; + + if (use_pbles) { + ret = irdma_check_mr_contiguous(palloc, + iwmr->page_size); + if (ret) { + irdma_free_pble(iwdev->rf->pble_rsrc, palloc); + iwpbl->pbl_allocated = false; + } + } + + stag = irdma_create_stag(iwdev); + if (!stag) { + err = -ENOMEM; + goto error; + } + + iwmr->stag = stag; + iwmr->ibmr.rkey = stag; + iwmr->ibmr.lkey = stag; + err = irdma_hwreg_mr(iwdev, iwmr, access); + if (err) { + irdma_free_stag(iwdev, stag); + goto error; + } + + break; + default: + goto error; + } + + iwmr->type = req.reg_type; + + return &iwmr->ibmr; + +error: + if (palloc->level != PBLE_LEVEL_0 && iwpbl->pbl_allocated) + irdma_free_pble(iwdev->rf->pble_rsrc, palloc); + ib_umem_release(region); + kfree(iwmr); + + return ERR_PTR(err); +} + +/** + * irdma_reg_phys_mr - register kernel physical memory + * @pd: ibpd pointer + * @addr: physical address of memory to register + * @size: size of memory to register + * @access: Access rights + * @iova_start: start of virtual address for physical buffers + */ +struct ib_mr *irdma_reg_phys_mr(struct ib_pd *pd, u64 addr, u64 size, int access, + u64 *iova_start) +{ + struct irdma_device *iwdev = to_iwdev(pd->device); + struct irdma_pbl *iwpbl; + struct irdma_mr *iwmr; + enum irdma_status_code status; + u32 stag; + int ret; + + iwmr = kzalloc(sizeof(*iwmr), GFP_KERNEL); + if (!iwmr) + return ERR_PTR(-ENOMEM); + + iwmr->ibmr.pd = pd; + iwmr->ibmr.device = pd->device; + iwpbl = &iwmr->iwpbl; + iwpbl->iwmr = iwmr; + iwmr->type = IRDMA_MEMREG_TYPE_MEM; + iwpbl->user_base = *iova_start; + stag = irdma_create_stag(iwdev); + if (!stag) { + ret = -ENOMEM; + goto err; + } + + iwmr->stag = stag; + iwmr->ibmr.iova = *iova_start; + iwmr->ibmr.rkey = stag; + iwmr->ibmr.lkey = stag; + iwmr->page_cnt = 1; + iwmr->pgaddrmem[0] = addr; + iwmr->len = size; + iwmr->page_size = SZ_4K; + status = irdma_hwreg_mr(iwdev, iwmr, access); + if (status) { + irdma_free_stag(iwdev, stag); + ret = -ENOMEM; + goto err; + } + + return &iwmr->ibmr; + +err: + kfree(iwmr); + + return ERR_PTR(ret); +} + +/** + * irdma_get_dma_mr - register physical mem + * @pd: ptr of pd + * @acc: access for memory + */ +static struct ib_mr *irdma_get_dma_mr(struct ib_pd *pd, int acc) +{ + u64 kva = 0; + + return irdma_reg_phys_mr(pd, 0, 0, acc, &kva); +} + +/** + * irdma_del_memlist - Deleting pbl list entries for CQ/QP + * @iwmr: iwmr for IB's user page addresses + * @ucontext: ptr to user context + */ +static void irdma_del_memlist(struct irdma_mr *iwmr, + struct irdma_ucontext *ucontext) +{ + struct irdma_pbl *iwpbl = &iwmr->iwpbl; + unsigned long flags; + + switch (iwmr->type) { + case IRDMA_MEMREG_TYPE_CQ: + spin_lock_irqsave(&ucontext->cq_reg_mem_list_lock, flags); + if (iwpbl->on_list) { + iwpbl->on_list = false; + list_del(&iwpbl->list); + } + spin_unlock_irqrestore(&ucontext->cq_reg_mem_list_lock, flags); + break; + case IRDMA_MEMREG_TYPE_QP: + spin_lock_irqsave(&ucontext->qp_reg_mem_list_lock, flags); + if (iwpbl->on_list) { + iwpbl->on_list = false; + list_del(&iwpbl->list); + } + spin_unlock_irqrestore(&ucontext->qp_reg_mem_list_lock, flags); + break; + default: + break; + } +} + +/** + * irdma_dereg_mr - deregister mr + * @ib_mr: mr ptr for dereg + * @udata: user data + */ +static int irdma_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata) +{ + struct ib_pd *ibpd = ib_mr->pd; + struct irdma_pd *iwpd = to_iwpd(ibpd); + struct irdma_mr *iwmr = to_iwmr(ib_mr); + struct irdma_device *iwdev = to_iwdev(ib_mr->device); + struct irdma_dealloc_stag_info *info; + struct irdma_pbl *iwpbl = &iwmr->iwpbl; + struct irdma_pble_alloc *palloc = &iwpbl->pble_alloc; + struct irdma_cqp_request *cqp_request; + struct cqp_cmds_info *cqp_info; + + if (iwmr->type != IRDMA_MEMREG_TYPE_MEM) { + if (iwmr->region) { + struct irdma_ucontext *ucontext; + + ucontext = rdma_udata_to_drv_context(udata, + struct irdma_ucontext, + ibucontext); + irdma_del_memlist(iwmr, ucontext); + } + goto done; + } + + cqp_request = irdma_alloc_and_get_cqp_request(&iwdev->rf->cqp, true); + if (!cqp_request) + return -ENOMEM; + + cqp_info = &cqp_request->info; + info = &cqp_info->in.u.dealloc_stag.info; + memset(info, 0, sizeof(*info)); + info->pd_id = iwpd->sc_pd.pd_id & 0x00007fff; + info->stag_idx = ib_mr->rkey >> IRDMA_CQPSQ_STAG_IDX_S; + info->mr = true; + if (iwpbl->pbl_allocated) + info->dealloc_pbl = true; + + cqp_info->cqp_cmd = IRDMA_OP_DEALLOC_STAG; + cqp_info->post_sq = 1; + cqp_info->in.u.dealloc_stag.dev = &iwdev->rf->sc_dev; + cqp_info->in.u.dealloc_stag.scratch = (uintptr_t)cqp_request; + irdma_handle_cqp_op(iwdev->rf, cqp_request); + irdma_put_cqp_request(&iwdev->rf->cqp, cqp_request); + irdma_free_stag(iwdev, iwmr->stag); +done: + if (iwpbl->pbl_allocated) + irdma_free_pble(iwdev->rf->pble_rsrc, palloc); + ib_umem_release(iwmr->region); + kfree(iwmr); + + return 0; +} + +/** + * irdma_copy_sg_list - copy sg list for qp + * @sg_list: copied into sg_list + * @sgl: copy from sgl + * @num_sges: count of sg entries + */ +static void irdma_copy_sg_list(struct irdma_sge *sg_list, struct ib_sge *sgl, + int num_sges) +{ + unsigned int i; + + for (i = 0; (i < num_sges) && (i < IRDMA_MAX_WQ_FRAGMENT_COUNT); i++) { + sg_list[i].tag_off = sgl[i].addr; + sg_list[i].len = sgl[i].length; + sg_list[i].stag = sgl[i].lkey; + } +} + +/** + * irdma_post_send - kernel application wr + * @ibqp: qp ptr for wr + * @ib_wr: work request ptr + * @bad_wr: return of bad wr if err + */ +static int irdma_post_send(struct ib_qp *ibqp, + const struct ib_send_wr *ib_wr, + const struct ib_send_wr **bad_wr) +{ + struct irdma_qp *iwqp; + struct irdma_qp_uk *ukqp; + struct irdma_sc_dev *dev; + struct irdma_post_sq_info info; + enum irdma_status_code ret; + int err = 0; + unsigned long flags; + bool inv_stag; + struct irdma_ah *ah; + bool reflush = false; + + iwqp = to_iwqp(ibqp); + ukqp = &iwqp->sc_qp.qp_uk; + dev = &iwqp->iwdev->rf->sc_dev; + + spin_lock_irqsave(&iwqp->lock, flags); + if (iwqp->flush_issued && ukqp->sq_flush_complete) + reflush = true; + while (ib_wr) { + memset(&info, 0, sizeof(info)); + inv_stag = false; + info.wr_id = (ib_wr->wr_id); + if ((ib_wr->send_flags & IB_SEND_SIGNALED) || iwqp->sig_all) + info.signaled = true; + if (ib_wr->send_flags & IB_SEND_FENCE) + info.read_fence = true; + switch (ib_wr->opcode) { + case IB_WR_SEND_WITH_IMM: + if (ukqp->qp_caps & IRDMA_SEND_WITH_IMM) { + info.imm_data_valid = true; + info.imm_data = ntohl(ib_wr->ex.imm_data); + } else { + err = -EINVAL; + break; + } + fallthrough; + case IB_WR_SEND: + case IB_WR_SEND_WITH_INV: + if (ib_wr->opcode == IB_WR_SEND || + ib_wr->opcode == IB_WR_SEND_WITH_IMM) { + if (ib_wr->send_flags & IB_SEND_SOLICITED) + info.op_type = IRDMA_OP_TYPE_SEND_SOL; + else + info.op_type = IRDMA_OP_TYPE_SEND; + } else { + if (ib_wr->send_flags & IB_SEND_SOLICITED) + info.op_type = IRDMA_OP_TYPE_SEND_SOL_INV; + else + info.op_type = IRDMA_OP_TYPE_SEND_INV; + info.stag_to_inv = ib_wr->ex.invalidate_rkey; + } + + if (ib_wr->send_flags & IB_SEND_INLINE) { + info.op.inline_send.data = (void *)(unsigned long) + ib_wr->sg_list[0].addr; + info.op.inline_send.len = ib_wr->sg_list[0].length; + if (iwqp->ibqp.qp_type == IB_QPT_UD || + iwqp->ibqp.qp_type == IB_QPT_GSI) { + ah = to_iwah(ud_wr(ib_wr)->ah); + info.op.inline_send.ah_id = ah->sc_ah.ah_info.ah_idx; + info.op.inline_send.qkey = ud_wr(ib_wr)->remote_qkey; + info.op.inline_send.dest_qp = ud_wr(ib_wr)->remote_qpn; + } + ret = irdma_uk_inline_send(ukqp, &info, false); + } else { + info.op.send.num_sges = ib_wr->num_sge; + info.op.send.sg_list = (struct irdma_sge *) + ib_wr->sg_list; + if (iwqp->ibqp.qp_type == IB_QPT_UD || + iwqp->ibqp.qp_type == IB_QPT_GSI) { + ah = to_iwah(ud_wr(ib_wr)->ah); + info.op.send.ah_id = ah->sc_ah.ah_info.ah_idx; + info.op.send.qkey = ud_wr(ib_wr)->remote_qkey; + info.op.send.dest_qp = ud_wr(ib_wr)->remote_qpn; + } + ret = irdma_uk_send(ukqp, &info, false); + } + + if (ret) { + if (ret == IRDMA_ERR_QP_TOOMANY_WRS_POSTED) + err = -ENOMEM; + else + err = -EINVAL; + } + break; + case IB_WR_RDMA_WRITE_WITH_IMM: + if (ukqp->qp_caps & IRDMA_WRITE_WITH_IMM) { + info.imm_data_valid = true; + info.imm_data = ntohl(ib_wr->ex.imm_data); + } else { + err = -EINVAL; + break; + } + fallthrough; + case IB_WR_RDMA_WRITE: + if (ib_wr->send_flags & IB_SEND_SOLICITED) + info.op_type = IRDMA_OP_TYPE_RDMA_WRITE_SOL; + else + info.op_type = IRDMA_OP_TYPE_RDMA_WRITE; + + if (ib_wr->send_flags & IB_SEND_INLINE) { + info.op.inline_rdma_write.data = (void *)(uintptr_t)ib_wr->sg_list[0].addr; + info.op.inline_rdma_write.len = ib_wr->sg_list[0].length; + info.op.inline_rdma_write.rem_addr.tag_off = rdma_wr(ib_wr)->remote_addr; + info.op.inline_rdma_write.rem_addr.stag = rdma_wr(ib_wr)->rkey; + ret = irdma_uk_inline_rdma_write(ukqp, &info, false); + } else { + info.op.rdma_write.lo_sg_list = (void *)ib_wr->sg_list; + info.op.rdma_write.num_lo_sges = ib_wr->num_sge; + info.op.rdma_write.rem_addr.tag_off = rdma_wr(ib_wr)->remote_addr; + info.op.rdma_write.rem_addr.stag = rdma_wr(ib_wr)->rkey; + ret = irdma_uk_rdma_write(ukqp, &info, false); + } + + if (ret) { + if (ret == IRDMA_ERR_QP_TOOMANY_WRS_POSTED) + err = -ENOMEM; + else + err = -EINVAL; + } + break; + case IB_WR_RDMA_READ_WITH_INV: + inv_stag = true; + fallthrough; + case IB_WR_RDMA_READ: + if (ib_wr->num_sge > + dev->hw_attrs.uk_attrs.max_hw_read_sges) { + err = -EINVAL; + break; + } + info.op_type = IRDMA_OP_TYPE_RDMA_READ; + info.op.rdma_read.rem_addr.tag_off = rdma_wr(ib_wr)->remote_addr; + info.op.rdma_read.rem_addr.stag = rdma_wr(ib_wr)->rkey; + info.op.rdma_read.lo_sg_list = (void *)ib_wr->sg_list; + info.op.rdma_read.num_lo_sges = ib_wr->num_sge; + + ret = irdma_uk_rdma_read(ukqp, &info, inv_stag, false); + if (ret) { + if (ret == IRDMA_ERR_QP_TOOMANY_WRS_POSTED) + err = -ENOMEM; + else + err = -EINVAL; + } + break; + case IB_WR_LOCAL_INV: + info.op_type = IRDMA_OP_TYPE_INV_STAG; + info.op.inv_local_stag.target_stag = ib_wr->ex.invalidate_rkey; + ret = irdma_uk_stag_local_invalidate(ukqp, &info, true); + if (ret) + err = -ENOMEM; + break; + case IB_WR_REG_MR: { + struct irdma_mr *iwmr = to_iwmr(reg_wr(ib_wr)->mr); + struct irdma_pble_alloc *palloc = &iwmr->iwpbl.pble_alloc; + struct irdma_fast_reg_stag_info stag_info = {}; + + stag_info.signaled = info.signaled; + stag_info.read_fence = info.read_fence; + stag_info.access_rights = irdma_get_mr_access(reg_wr(ib_wr)->access); + stag_info.stag_key = reg_wr(ib_wr)->key & 0xff; + stag_info.stag_idx = reg_wr(ib_wr)->key >> 8; + stag_info.page_size = reg_wr(ib_wr)->mr->page_size; + stag_info.wr_id = ib_wr->wr_id; + stag_info.addr_type = IRDMA_ADDR_TYPE_VA_BASED; + stag_info.va = (void *)(uintptr_t)iwmr->ibmr.iova; + stag_info.total_len = iwmr->ibmr.length; + stag_info.reg_addr_pa = *((u64 *)(uintptr_t)palloc->level1.addr); + stag_info.first_pm_pbl_index = palloc->level1.idx; + stag_info.local_fence = ib_wr->send_flags & IB_SEND_FENCE; + if (iwmr->npages > IRDMA_MIN_PAGES_PER_FMR) + stag_info.chunk_size = 1; + ret = irdma_sc_mr_fast_register(&iwqp->sc_qp, &stag_info, + true); + if (ret) + err = -ENOMEM; + break; + } + default: + err = -EINVAL; + ibdev_dbg(&iwqp->iwdev->ibdev, + "VERBS: upost_send bad opcode = 0x%x\n", + ib_wr->opcode); + break; + } + + if (err) + break; + ib_wr = ib_wr->next; + } + + if (!iwqp->flush_issued && iwqp->hw_iwarp_state <= IRDMA_QP_STATE_RTS) { + irdma_uk_qp_post_wr(ukqp); + spin_unlock_irqrestore(&iwqp->lock, flags); + } else if (reflush) { + ukqp->sq_flush_complete = false; + spin_unlock_irqrestore(&iwqp->lock, flags); + irdma_flush_wqes(iwqp, IRDMA_FLUSH_SQ | IRDMA_REFLUSH); + } else { + spin_unlock_irqrestore(&iwqp->lock, flags); + } + if (err) + *bad_wr = ib_wr; + + return err; +} + +/** + * irdma_post_recv - post receive wr for kernel application + * @ibqp: ib qp pointer + * @ib_wr: work request for receive + * @bad_wr: bad wr caused an error + */ +static int irdma_post_recv(struct ib_qp *ibqp, + const struct ib_recv_wr *ib_wr, + const struct ib_recv_wr **bad_wr) +{ + struct irdma_qp *iwqp; + struct irdma_qp_uk *ukqp; + struct irdma_post_rq_info post_recv = {}; + struct irdma_sge sg_list[IRDMA_MAX_WQ_FRAGMENT_COUNT]; + enum irdma_status_code ret = 0; + unsigned long flags; + int err = 0; + bool reflush = false; + + iwqp = to_iwqp(ibqp); + ukqp = &iwqp->sc_qp.qp_uk; + + spin_lock_irqsave(&iwqp->lock, flags); + if (iwqp->flush_issued && ukqp->rq_flush_complete) + reflush = true; + while (ib_wr) { + post_recv.num_sges = ib_wr->num_sge; + post_recv.wr_id = ib_wr->wr_id; + irdma_copy_sg_list(sg_list, ib_wr->sg_list, ib_wr->num_sge); + post_recv.sg_list = sg_list; + ret = irdma_uk_post_receive(ukqp, &post_recv); + if (ret) { + ibdev_dbg(&iwqp->iwdev->ibdev, + "VERBS: post_recv err %d\n", ret); + if (ret == IRDMA_ERR_QP_TOOMANY_WRS_POSTED) + err = -ENOMEM; + else + err = -EINVAL; + goto out; + } + + ib_wr = ib_wr->next; + } + +out: + if (reflush) { + ukqp->rq_flush_complete = false; + spin_unlock_irqrestore(&iwqp->lock, flags); + irdma_flush_wqes(iwqp, IRDMA_FLUSH_RQ | IRDMA_REFLUSH); + } else { + spin_unlock_irqrestore(&iwqp->lock, flags); + } + + if (err) + *bad_wr = ib_wr; + + return err; +} + +/** + * irdma_flush_err_to_ib_wc_status - return change flush error code to IB status + * @opcode: iwarp flush code + */ +static enum ib_wc_status irdma_flush_err_to_ib_wc_status(enum irdma_flush_opcode opcode) +{ + switch (opcode) { + case FLUSH_PROT_ERR: + return IB_WC_LOC_PROT_ERR; + case FLUSH_REM_ACCESS_ERR: + return IB_WC_REM_ACCESS_ERR; + case FLUSH_LOC_QP_OP_ERR: + return IB_WC_LOC_QP_OP_ERR; + case FLUSH_REM_OP_ERR: + return IB_WC_REM_OP_ERR; + case FLUSH_LOC_LEN_ERR: + return IB_WC_LOC_LEN_ERR; + case FLUSH_GENERAL_ERR: + return IB_WC_WR_FLUSH_ERR; + case FLUSH_FATAL_ERR: + default: + return IB_WC_FATAL_ERR; + } +} + +/** + * irdma_process_cqe - process cqe info + * @entry: processed cqe + * @cq_poll_info: cqe info + */ +static void irdma_process_cqe(struct ib_wc *entry, + struct irdma_cq_poll_info *cq_poll_info) +{ + struct irdma_qp *iwqp; + struct irdma_sc_qp *qp; + + entry->wc_flags = 0; + entry->pkey_index = 0; + entry->wr_id = cq_poll_info->wr_id; + + qp = cq_poll_info->qp_handle; + iwqp = qp->qp_uk.back_qp; + entry->qp = qp->qp_uk.back_qp; + + if (cq_poll_info->error) { + entry->status = (cq_poll_info->comp_status == IRDMA_COMPL_STATUS_FLUSHED) ? + irdma_flush_err_to_ib_wc_status(cq_poll_info->minor_err) : IB_WC_GENERAL_ERR; + + entry->vendor_err = cq_poll_info->major_err << 16 | + cq_poll_info->minor_err; + } else { + entry->status = IB_WC_SUCCESS; + if (cq_poll_info->imm_valid) { + entry->ex.imm_data = htonl(cq_poll_info->imm_data); + entry->wc_flags |= IB_WC_WITH_IMM; + } + if (cq_poll_info->ud_smac_valid) { + ether_addr_copy(entry->smac, cq_poll_info->ud_smac); + entry->wc_flags |= IB_WC_WITH_SMAC; + } + + if (cq_poll_info->ud_vlan_valid) { + entry->vlan_id = cq_poll_info->ud_vlan & VLAN_VID_MASK; + entry->wc_flags |= IB_WC_WITH_VLAN; + entry->sl = cq_poll_info->ud_vlan >> VLAN_PRIO_SHIFT; + } else { + entry->sl = 0; + } + } + + switch (cq_poll_info->op_type) { + case IRDMA_OP_TYPE_RDMA_WRITE: + case IRDMA_OP_TYPE_RDMA_WRITE_SOL: + entry->opcode = IB_WC_RDMA_WRITE; + break; + case IRDMA_OP_TYPE_RDMA_READ_INV_STAG: + case IRDMA_OP_TYPE_RDMA_READ: + entry->opcode = IB_WC_RDMA_READ; + break; + case IRDMA_OP_TYPE_SEND_INV: + case IRDMA_OP_TYPE_SEND_SOL: + case IRDMA_OP_TYPE_SEND_SOL_INV: + case IRDMA_OP_TYPE_SEND: + entry->opcode = IB_WC_SEND; + break; + case IRDMA_OP_TYPE_FAST_REG_NSMR: + entry->opcode = IB_WC_REG_MR; + break; + case IRDMA_OP_TYPE_INV_STAG: + entry->opcode = IB_WC_LOCAL_INV; + break; + case IRDMA_OP_TYPE_REC_IMM: + case IRDMA_OP_TYPE_REC: + entry->opcode = cq_poll_info->op_type == IRDMA_OP_TYPE_REC_IMM ? + IB_WC_RECV_RDMA_WITH_IMM : IB_WC_RECV; + if (qp->qp_uk.qp_type != IRDMA_QP_TYPE_ROCE_UD && + cq_poll_info->stag_invalid_set) { + entry->ex.invalidate_rkey = cq_poll_info->inv_stag; + entry->wc_flags |= IB_WC_WITH_INVALIDATE; + } + break; + default: + ibdev_err(&iwqp->iwdev->ibdev, + "Invalid opcode = %d in CQE\n", cq_poll_info->op_type); + entry->status = IB_WC_GENERAL_ERR; + return; + } + + if (qp->qp_uk.qp_type == IRDMA_QP_TYPE_ROCE_UD) { + entry->src_qp = cq_poll_info->ud_src_qpn; + entry->slid = 0; + entry->wc_flags |= + (IB_WC_GRH | IB_WC_WITH_NETWORK_HDR_TYPE); + entry->network_hdr_type = cq_poll_info->ipv4 ? + RDMA_NETWORK_IPV4 : + RDMA_NETWORK_IPV6; + } else { + entry->src_qp = cq_poll_info->qp_id; + } + + entry->byte_len = cq_poll_info->bytes_xfered; +} + +/** + * irdma_poll_one - poll one entry of the CQ + * @ukcq: ukcq to poll + * @cur_cqe: current CQE info to be filled in + * @entry: ibv_wc object to be filled for non-extended CQ or NULL for extended CQ + * + * Returns the internal irdma device error code or 0 on success + */ +static inline int irdma_poll_one(struct irdma_cq_uk *ukcq, + struct irdma_cq_poll_info *cur_cqe, + struct ib_wc *entry) +{ + int ret = irdma_uk_cq_poll_cmpl(ukcq, cur_cqe); + + if (ret) + return ret; + + irdma_process_cqe(entry, cur_cqe); + + return 0; +} + +/** + * __irdma_poll_cq - poll cq for completion (kernel apps) + * @iwcq: cq to poll + * @num_entries: number of entries to poll + * @entry: wr of a completed entry + */ +static int __irdma_poll_cq(struct irdma_cq *iwcq, int num_entries, struct ib_wc *entry) +{ + struct list_head *tmp_node, *list_node; + struct irdma_cq_buf *last_buf = NULL; + struct irdma_cq_poll_info *cur_cqe = &iwcq->cur_cqe; + struct irdma_cq_buf *cq_buf; + enum irdma_status_code ret; + struct irdma_device *iwdev; + struct irdma_cq_uk *ukcq; + bool cq_new_cqe = false; + int resized_bufs = 0; + int npolled = 0; + + iwdev = to_iwdev(iwcq->ibcq.device); + ukcq = &iwcq->sc_cq.cq_uk; + + /* go through the list of previously resized CQ buffers */ + list_for_each_safe(list_node, tmp_node, &iwcq->resize_list) { + cq_buf = container_of(list_node, struct irdma_cq_buf, list); + while (npolled < num_entries) { + ret = irdma_poll_one(&cq_buf->cq_uk, cur_cqe, entry + npolled); + if (!ret) { + ++npolled; + cq_new_cqe = true; + continue; + } + if (ret == IRDMA_ERR_Q_EMPTY) + break; + /* QP using the CQ is destroyed. Skip reporting this CQE */ + if (ret == IRDMA_ERR_Q_DESTROYED) { + cq_new_cqe = true; + continue; + } + goto error; + } + + /* save the resized CQ buffer which received the last cqe */ + if (cq_new_cqe) + last_buf = cq_buf; + cq_new_cqe = false; + } + + /* check the current CQ for new cqes */ + while (npolled < num_entries) { + ret = irdma_poll_one(ukcq, cur_cqe, entry + npolled); + if (!ret) { + ++npolled; + cq_new_cqe = true; + continue; + } + + if (ret == IRDMA_ERR_Q_EMPTY) + break; + /* QP using the CQ is destroyed. Skip reporting this CQE */ + if (ret == IRDMA_ERR_Q_DESTROYED) { + cq_new_cqe = true; + continue; + } + goto error; + } + + if (cq_new_cqe) + /* all previous CQ resizes are complete */ + resized_bufs = irdma_process_resize_list(iwcq, iwdev, NULL); + else if (last_buf) + /* only CQ resizes up to the last_buf are complete */ + resized_bufs = irdma_process_resize_list(iwcq, iwdev, last_buf); + if (resized_bufs) + /* report to the HW the number of complete CQ resizes */ + irdma_uk_cq_set_resized_cnt(ukcq, resized_bufs); + + return npolled; +error: + ibdev_dbg(&iwdev->ibdev, "%s: Error polling CQ, irdma_err: %d\n", + __func__, ret); + + return -EINVAL; +} + +/** + * irdma_poll_cq - poll cq for completion (kernel apps) + * @ibcq: cq to poll + * @num_entries: number of entries to poll + * @entry: wr of a completed entry + */ +static int irdma_poll_cq(struct ib_cq *ibcq, int num_entries, + struct ib_wc *entry) +{ + struct irdma_cq *iwcq; + unsigned long flags; + int ret; + + iwcq = to_iwcq(ibcq); + + spin_lock_irqsave(&iwcq->lock, flags); + ret = __irdma_poll_cq(iwcq, num_entries, entry); + spin_unlock_irqrestore(&iwcq->lock, flags); + + return ret; +} + +/** + * irdma_req_notify_cq - arm cq kernel application + * @ibcq: cq to arm + * @notify_flags: notofication flags + */ +static int irdma_req_notify_cq(struct ib_cq *ibcq, + enum ib_cq_notify_flags notify_flags) +{ + struct irdma_cq *iwcq; + struct irdma_cq_uk *ukcq; + unsigned long flags; + enum irdma_cmpl_notify cq_notify = IRDMA_CQ_COMPL_EVENT; + + iwcq = to_iwcq(ibcq); + ukcq = &iwcq->sc_cq.cq_uk; + if (notify_flags == IB_CQ_SOLICITED) + cq_notify = IRDMA_CQ_COMPL_SOLICITED; + + spin_lock_irqsave(&iwcq->lock, flags); + irdma_uk_cq_request_notification(ukcq, cq_notify); + spin_unlock_irqrestore(&iwcq->lock, flags); + + return 0; +} + +static int irdma_roce_port_immutable(struct ib_device *ibdev, u32 port_num, + struct ib_port_immutable *immutable) +{ + struct ib_port_attr attr; + int err; + + immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP; + err = ib_query_port(ibdev, port_num, &attr); + if (err) + return err; + + immutable->max_mad_size = IB_MGMT_MAD_SIZE; + immutable->pkey_tbl_len = attr.pkey_tbl_len; + immutable->gid_tbl_len = attr.gid_tbl_len; + + return 0; +} + +static int irdma_iw_port_immutable(struct ib_device *ibdev, u32 port_num, + struct ib_port_immutable *immutable) +{ + struct ib_port_attr attr; + int err; + + immutable->core_cap_flags = RDMA_CORE_PORT_IWARP; + err = ib_query_port(ibdev, port_num, &attr); + if (err) + return err; + immutable->gid_tbl_len = 1; + + return 0; +} + +static const char *const irdma_hw_stat_names[] = { + /* 32bit names */ + [IRDMA_HW_STAT_INDEX_RXVLANERR] = "rxVlanErrors", + [IRDMA_HW_STAT_INDEX_IP4RXDISCARD] = "ip4InDiscards", + [IRDMA_HW_STAT_INDEX_IP4RXTRUNC] = "ip4InTruncatedPkts", + [IRDMA_HW_STAT_INDEX_IP4TXNOROUTE] = "ip4OutNoRoutes", + [IRDMA_HW_STAT_INDEX_IP6RXDISCARD] = "ip6InDiscards", + [IRDMA_HW_STAT_INDEX_IP6RXTRUNC] = "ip6InTruncatedPkts", + [IRDMA_HW_STAT_INDEX_IP6TXNOROUTE] = "ip6OutNoRoutes", + [IRDMA_HW_STAT_INDEX_TCPRTXSEG] = "tcpRetransSegs", + [IRDMA_HW_STAT_INDEX_TCPRXOPTERR] = "tcpInOptErrors", + [IRDMA_HW_STAT_INDEX_TCPRXPROTOERR] = "tcpInProtoErrors", + [IRDMA_HW_STAT_INDEX_RXRPCNPHANDLED] = "cnpHandled", + [IRDMA_HW_STAT_INDEX_RXRPCNPIGNORED] = "cnpIgnored", + [IRDMA_HW_STAT_INDEX_TXNPCNPSENT] = "cnpSent", + + /* 64bit names */ + [IRDMA_HW_STAT_INDEX_IP4RXOCTS + IRDMA_HW_STAT_INDEX_MAX_32] = + "ip4InOctets", + [IRDMA_HW_STAT_INDEX_IP4RXPKTS + IRDMA_HW_STAT_INDEX_MAX_32] = + "ip4InPkts", + [IRDMA_HW_STAT_INDEX_IP4RXFRAGS + IRDMA_HW_STAT_INDEX_MAX_32] = + "ip4InReasmRqd", + [IRDMA_HW_STAT_INDEX_IP4RXMCOCTS + IRDMA_HW_STAT_INDEX_MAX_32] = + "ip4InMcastOctets", + [IRDMA_HW_STAT_INDEX_IP4RXMCPKTS + IRDMA_HW_STAT_INDEX_MAX_32] = + "ip4InMcastPkts", + [IRDMA_HW_STAT_INDEX_IP4TXOCTS + IRDMA_HW_STAT_INDEX_MAX_32] = + "ip4OutOctets", + [IRDMA_HW_STAT_INDEX_IP4TXPKTS + IRDMA_HW_STAT_INDEX_MAX_32] = + "ip4OutPkts", + [IRDMA_HW_STAT_INDEX_IP4TXFRAGS + IRDMA_HW_STAT_INDEX_MAX_32] = + "ip4OutSegRqd", + [IRDMA_HW_STAT_INDEX_IP4TXMCOCTS + IRDMA_HW_STAT_INDEX_MAX_32] = + "ip4OutMcastOctets", + [IRDMA_HW_STAT_INDEX_IP4TXMCPKTS + IRDMA_HW_STAT_INDEX_MAX_32] = + "ip4OutMcastPkts", + [IRDMA_HW_STAT_INDEX_IP6RXOCTS + IRDMA_HW_STAT_INDEX_MAX_32] = + "ip6InOctets", + [IRDMA_HW_STAT_INDEX_IP6RXPKTS + IRDMA_HW_STAT_INDEX_MAX_32] = + "ip6InPkts", + [IRDMA_HW_STAT_INDEX_IP6RXFRAGS + IRDMA_HW_STAT_INDEX_MAX_32] = + "ip6InReasmRqd", + [IRDMA_HW_STAT_INDEX_IP6RXMCOCTS + IRDMA_HW_STAT_INDEX_MAX_32] = + "ip6InMcastOctets", + [IRDMA_HW_STAT_INDEX_IP6RXMCPKTS + IRDMA_HW_STAT_INDEX_MAX_32] = + "ip6InMcastPkts", + [IRDMA_HW_STAT_INDEX_IP6TXOCTS + IRDMA_HW_STAT_INDEX_MAX_32] = + "ip6OutOctets", + [IRDMA_HW_STAT_INDEX_IP6TXPKTS + IRDMA_HW_STAT_INDEX_MAX_32] = + "ip6OutPkts", + [IRDMA_HW_STAT_INDEX_IP6TXFRAGS + IRDMA_HW_STAT_INDEX_MAX_32] = + "ip6OutSegRqd", + [IRDMA_HW_STAT_INDEX_IP6TXMCOCTS + IRDMA_HW_STAT_INDEX_MAX_32] = + "ip6OutMcastOctets", + [IRDMA_HW_STAT_INDEX_IP6TXMCPKTS + IRDMA_HW_STAT_INDEX_MAX_32] = + "ip6OutMcastPkts", + [IRDMA_HW_STAT_INDEX_TCPRXSEGS + IRDMA_HW_STAT_INDEX_MAX_32] = + "tcpInSegs", + [IRDMA_HW_STAT_INDEX_TCPTXSEG + IRDMA_HW_STAT_INDEX_MAX_32] = + "tcpOutSegs", + [IRDMA_HW_STAT_INDEX_RDMARXRDS + IRDMA_HW_STAT_INDEX_MAX_32] = + "iwInRdmaReads", + [IRDMA_HW_STAT_INDEX_RDMARXSNDS + IRDMA_HW_STAT_INDEX_MAX_32] = + "iwInRdmaSends", + [IRDMA_HW_STAT_INDEX_RDMARXWRS + IRDMA_HW_STAT_INDEX_MAX_32] = + "iwInRdmaWrites", + [IRDMA_HW_STAT_INDEX_RDMATXRDS + IRDMA_HW_STAT_INDEX_MAX_32] = + "iwOutRdmaReads", + [IRDMA_HW_STAT_INDEX_RDMATXSNDS + IRDMA_HW_STAT_INDEX_MAX_32] = + "iwOutRdmaSends", + [IRDMA_HW_STAT_INDEX_RDMATXWRS + IRDMA_HW_STAT_INDEX_MAX_32] = + "iwOutRdmaWrites", + [IRDMA_HW_STAT_INDEX_RDMAVBND + IRDMA_HW_STAT_INDEX_MAX_32] = + "iwRdmaBnd", + [IRDMA_HW_STAT_INDEX_RDMAVINV + IRDMA_HW_STAT_INDEX_MAX_32] = + "iwRdmaInv", + [IRDMA_HW_STAT_INDEX_UDPRXPKTS + IRDMA_HW_STAT_INDEX_MAX_32] = + "RxUDP", + [IRDMA_HW_STAT_INDEX_UDPTXPKTS + IRDMA_HW_STAT_INDEX_MAX_32] = + "TxUDP", + [IRDMA_HW_STAT_INDEX_RXNPECNMARKEDPKTS + IRDMA_HW_STAT_INDEX_MAX_32] = + "RxECNMrkd", +}; + +static void irdma_get_dev_fw_str(struct ib_device *dev, char *str) +{ + struct irdma_device *iwdev = to_iwdev(dev); + + snprintf(str, IB_FW_VERSION_NAME_MAX, "%u.%u", + irdma_fw_major_ver(&iwdev->rf->sc_dev), + irdma_fw_minor_ver(&iwdev->rf->sc_dev)); +} + +/** + * irdma_alloc_hw_stats - Allocate a hw stats structure + * @ibdev: device pointer from stack + * @port_num: port number + */ +static struct rdma_hw_stats *irdma_alloc_hw_stats(struct ib_device *ibdev, + u32 port_num) +{ + int num_counters = IRDMA_HW_STAT_INDEX_MAX_32 + + IRDMA_HW_STAT_INDEX_MAX_64; + unsigned long lifespan = RDMA_HW_STATS_DEFAULT_LIFESPAN; + + if (!port_num) + return NULL; + + BUILD_BUG_ON(ARRAY_SIZE(irdma_hw_stat_names) != + (IRDMA_HW_STAT_INDEX_MAX_32 + IRDMA_HW_STAT_INDEX_MAX_64)); + + return rdma_alloc_hw_stats_struct(irdma_hw_stat_names, num_counters, + lifespan); +} + +/** + * irdma_get_hw_stats - Populates the rdma_hw_stats structure + * @ibdev: device pointer from stack + * @stats: stats pointer from stack + * @port_num: port number + * @index: which hw counter the stack is requesting we update + */ +static int irdma_get_hw_stats(struct ib_device *ibdev, + struct rdma_hw_stats *stats, u32 port_num, + int index) +{ + struct irdma_device *iwdev = to_iwdev(ibdev); + struct irdma_dev_hw_stats *hw_stats = &iwdev->vsi.pestat->hw_stats; + + if (iwdev->rf->rdma_ver >= IRDMA_GEN_2) + irdma_cqp_gather_stats_cmd(&iwdev->rf->sc_dev, iwdev->vsi.pestat, true); + else + irdma_cqp_gather_stats_gen1(&iwdev->rf->sc_dev, iwdev->vsi.pestat); + + memcpy(&stats->value[0], hw_stats, sizeof(*hw_stats)); + + return stats->num_counters; +} + +/** + * irdma_query_gid - Query port GID + * @ibdev: device pointer from stack + * @port: port number + * @index: Entry index + * @gid: Global ID + */ +static int irdma_query_gid(struct ib_device *ibdev, u32 port, int index, + union ib_gid *gid) +{ + struct irdma_device *iwdev = to_iwdev(ibdev); + + memset(gid->raw, 0, sizeof(gid->raw)); + ether_addr_copy(gid->raw, iwdev->netdev->dev_addr); + + return 0; +} + +/** + * mcast_list_add - Add a new mcast item to list + * @rf: RDMA PCI function + * @new_elem: pointer to element to add + */ +static void mcast_list_add(struct irdma_pci_f *rf, + struct mc_table_list *new_elem) +{ + list_add(&new_elem->list, &rf->mc_qht_list.list); +} + +/** + * mcast_list_del - Remove an mcast item from list + * @mc_qht_elem: pointer to mcast table list element + */ +static void mcast_list_del(struct mc_table_list *mc_qht_elem) +{ + if (mc_qht_elem) + list_del(&mc_qht_elem->list); +} + +/** + * mcast_list_lookup_ip - Search mcast list for address + * @rf: RDMA PCI function + * @ip_mcast: pointer to mcast IP address + */ +static struct mc_table_list *mcast_list_lookup_ip(struct irdma_pci_f *rf, + u32 *ip_mcast) +{ + struct mc_table_list *mc_qht_el; + struct list_head *pos, *q; + + list_for_each_safe (pos, q, &rf->mc_qht_list.list) { + mc_qht_el = list_entry(pos, struct mc_table_list, list); + if (!memcmp(mc_qht_el->mc_info.dest_ip, ip_mcast, + sizeof(mc_qht_el->mc_info.dest_ip))) + return mc_qht_el; + } + + return NULL; +} + +/** + * irdma_mcast_cqp_op - perform a mcast cqp operation + * @iwdev: irdma device + * @mc_grp_ctx: mcast group info + * @op: operation + * + * returns error status + */ +static int irdma_mcast_cqp_op(struct irdma_device *iwdev, + struct irdma_mcast_grp_info *mc_grp_ctx, u8 op) +{ + struct cqp_cmds_info *cqp_info; + struct irdma_cqp_request *cqp_request; + enum irdma_status_code status; + + cqp_request = irdma_alloc_and_get_cqp_request(&iwdev->rf->cqp, true); + if (!cqp_request) + return -ENOMEM; + + cqp_request->info.in.u.mc_create.info = *mc_grp_ctx; + cqp_info = &cqp_request->info; + cqp_info->cqp_cmd = op; + cqp_info->post_sq = 1; + cqp_info->in.u.mc_create.scratch = (uintptr_t)cqp_request; + cqp_info->in.u.mc_create.cqp = &iwdev->rf->cqp.sc_cqp; + status = irdma_handle_cqp_op(iwdev->rf, cqp_request); + irdma_put_cqp_request(&iwdev->rf->cqp, cqp_request); + if (status) + return -ENOMEM; + + return 0; +} + +/** + * irdma_mcast_mac - Get the multicast MAC for an IP address + * @ip_addr: IPv4 or IPv6 address + * @mac: pointer to result MAC address + * @ipv4: flag indicating IPv4 or IPv6 + * + */ +void irdma_mcast_mac(u32 *ip_addr, u8 *mac, bool ipv4) +{ + u8 *ip = (u8 *)ip_addr; + + if (ipv4) { + unsigned char mac4[ETH_ALEN] = {0x01, 0x00, 0x5E, 0x00, + 0x00, 0x00}; + + mac4[3] = ip[2] & 0x7F; + mac4[4] = ip[1]; + mac4[5] = ip[0]; + ether_addr_copy(mac, mac4); + } else { + unsigned char mac6[ETH_ALEN] = {0x33, 0x33, 0x00, 0x00, + 0x00, 0x00}; + + mac6[2] = ip[3]; + mac6[3] = ip[2]; + mac6[4] = ip[1]; + mac6[5] = ip[0]; + ether_addr_copy(mac, mac6); + } +} + +/** + * irdma_attach_mcast - attach a qp to a multicast group + * @ibqp: ptr to qp + * @ibgid: pointer to global ID + * @lid: local ID + * + * returns error status + */ +static int irdma_attach_mcast(struct ib_qp *ibqp, union ib_gid *ibgid, u16 lid) +{ + struct irdma_qp *iwqp = to_iwqp(ibqp); + struct irdma_device *iwdev = iwqp->iwdev; + struct irdma_pci_f *rf = iwdev->rf; + struct mc_table_list *mc_qht_elem; + struct irdma_mcast_grp_ctx_entry_info mcg_info = {}; + unsigned long flags; + u32 ip_addr[4] = {}; + u32 mgn; + u32 no_mgs; + int ret = 0; + bool ipv4; + u16 vlan_id; + union { + struct sockaddr saddr; + struct sockaddr_in saddr_in; + struct sockaddr_in6 saddr_in6; + } sgid_addr; + unsigned char dmac[ETH_ALEN]; + + rdma_gid2ip((struct sockaddr *)&sgid_addr, ibgid); + + if (!ipv6_addr_v4mapped((struct in6_addr *)ibgid)) { + irdma_copy_ip_ntohl(ip_addr, + sgid_addr.saddr_in6.sin6_addr.in6_u.u6_addr32); + irdma_netdev_vlan_ipv6(ip_addr, &vlan_id, NULL); + ipv4 = false; + ibdev_dbg(&iwdev->ibdev, + "VERBS: qp_id=%d, IP6address=%pI6\n", ibqp->qp_num, + ip_addr); + irdma_mcast_mac(ip_addr, dmac, false); + } else { + ip_addr[0] = ntohl(sgid_addr.saddr_in.sin_addr.s_addr); + ipv4 = true; + vlan_id = irdma_get_vlan_ipv4(ip_addr); + irdma_mcast_mac(ip_addr, dmac, true); + ibdev_dbg(&iwdev->ibdev, + "VERBS: qp_id=%d, IP4address=%pI4, MAC=%pM\n", + ibqp->qp_num, ip_addr, dmac); + } + + spin_lock_irqsave(&rf->qh_list_lock, flags); + mc_qht_elem = mcast_list_lookup_ip(rf, ip_addr); + if (!mc_qht_elem) { + struct irdma_dma_mem *dma_mem_mc; + + spin_unlock_irqrestore(&rf->qh_list_lock, flags); + mc_qht_elem = kzalloc(sizeof(*mc_qht_elem), GFP_KERNEL); + if (!mc_qht_elem) + return -ENOMEM; + + mc_qht_elem->mc_info.ipv4_valid = ipv4; + memcpy(mc_qht_elem->mc_info.dest_ip, ip_addr, + sizeof(mc_qht_elem->mc_info.dest_ip)); + ret = irdma_alloc_rsrc(rf, rf->allocated_mcgs, rf->max_mcg, + &mgn, &rf->next_mcg); + if (ret) { + kfree(mc_qht_elem); + return -ENOMEM; + } + + mc_qht_elem->mc_info.mgn = mgn; + dma_mem_mc = &mc_qht_elem->mc_grp_ctx.dma_mem_mc; + dma_mem_mc->size = ALIGN(sizeof(u64) * IRDMA_MAX_MGS_PER_CTX, + IRDMA_HW_PAGE_SIZE); + dma_mem_mc->va = dma_alloc_coherent(rf->hw.device, + dma_mem_mc->size, + &dma_mem_mc->pa, + GFP_KERNEL); + if (!dma_mem_mc->va) { + irdma_free_rsrc(rf, rf->allocated_mcgs, mgn); + kfree(mc_qht_elem); + return -ENOMEM; + } + + mc_qht_elem->mc_grp_ctx.mg_id = (u16)mgn; + memcpy(mc_qht_elem->mc_grp_ctx.dest_ip_addr, ip_addr, + sizeof(mc_qht_elem->mc_grp_ctx.dest_ip_addr)); + mc_qht_elem->mc_grp_ctx.ipv4_valid = ipv4; + mc_qht_elem->mc_grp_ctx.vlan_id = vlan_id; + if (vlan_id < VLAN_N_VID) + mc_qht_elem->mc_grp_ctx.vlan_valid = true; + mc_qht_elem->mc_grp_ctx.hmc_fcn_id = iwdev->vsi.fcn_id; + mc_qht_elem->mc_grp_ctx.qs_handle = + iwqp->sc_qp.vsi->qos[iwqp->sc_qp.user_pri].qs_handle; + ether_addr_copy(mc_qht_elem->mc_grp_ctx.dest_mac_addr, dmac); + + spin_lock_irqsave(&rf->qh_list_lock, flags); + mcast_list_add(rf, mc_qht_elem); + } else { + if (mc_qht_elem->mc_grp_ctx.no_of_mgs == + IRDMA_MAX_MGS_PER_CTX) { + spin_unlock_irqrestore(&rf->qh_list_lock, flags); + return -ENOMEM; + } + } + + mcg_info.qp_id = iwqp->ibqp.qp_num; + no_mgs = mc_qht_elem->mc_grp_ctx.no_of_mgs; + irdma_sc_add_mcast_grp(&mc_qht_elem->mc_grp_ctx, &mcg_info); + spin_unlock_irqrestore(&rf->qh_list_lock, flags); + + /* Only if there is a change do we need to modify or create */ + if (!no_mgs) { + ret = irdma_mcast_cqp_op(iwdev, &mc_qht_elem->mc_grp_ctx, + IRDMA_OP_MC_CREATE); + } else if (no_mgs != mc_qht_elem->mc_grp_ctx.no_of_mgs) { + ret = irdma_mcast_cqp_op(iwdev, &mc_qht_elem->mc_grp_ctx, + IRDMA_OP_MC_MODIFY); + } else { + return 0; + } + + if (ret) + goto error; + + return 0; + +error: + irdma_sc_del_mcast_grp(&mc_qht_elem->mc_grp_ctx, &mcg_info); + if (!mc_qht_elem->mc_grp_ctx.no_of_mgs) { + mcast_list_del(mc_qht_elem); + dma_free_coherent(rf->hw.device, + mc_qht_elem->mc_grp_ctx.dma_mem_mc.size, + mc_qht_elem->mc_grp_ctx.dma_mem_mc.va, + mc_qht_elem->mc_grp_ctx.dma_mem_mc.pa); + mc_qht_elem->mc_grp_ctx.dma_mem_mc.va = NULL; + irdma_free_rsrc(rf, rf->allocated_mcgs, + mc_qht_elem->mc_grp_ctx.mg_id); + kfree(mc_qht_elem); + } + + return ret; +} + +/** + * irdma_detach_mcast - detach a qp from a multicast group + * @ibqp: ptr to qp + * @ibgid: pointer to global ID + * @lid: local ID + * + * returns error status + */ +static int irdma_detach_mcast(struct ib_qp *ibqp, union ib_gid *ibgid, u16 lid) +{ + struct irdma_qp *iwqp = to_iwqp(ibqp); + struct irdma_device *iwdev = iwqp->iwdev; + struct irdma_pci_f *rf = iwdev->rf; + u32 ip_addr[4] = {}; + struct mc_table_list *mc_qht_elem; + struct irdma_mcast_grp_ctx_entry_info mcg_info = {}; + int ret; + unsigned long flags; + union { + struct sockaddr saddr; + struct sockaddr_in saddr_in; + struct sockaddr_in6 saddr_in6; + } sgid_addr; + + rdma_gid2ip((struct sockaddr *)&sgid_addr, ibgid); + if (!ipv6_addr_v4mapped((struct in6_addr *)ibgid)) + irdma_copy_ip_ntohl(ip_addr, + sgid_addr.saddr_in6.sin6_addr.in6_u.u6_addr32); + else + ip_addr[0] = ntohl(sgid_addr.saddr_in.sin_addr.s_addr); + + spin_lock_irqsave(&rf->qh_list_lock, flags); + mc_qht_elem = mcast_list_lookup_ip(rf, ip_addr); + if (!mc_qht_elem) { + spin_unlock_irqrestore(&rf->qh_list_lock, flags); + ibdev_dbg(&iwdev->ibdev, + "VERBS: address not found MCG\n"); + return 0; + } + + mcg_info.qp_id = iwqp->ibqp.qp_num; + irdma_sc_del_mcast_grp(&mc_qht_elem->mc_grp_ctx, &mcg_info); + if (!mc_qht_elem->mc_grp_ctx.no_of_mgs) { + mcast_list_del(mc_qht_elem); + spin_unlock_irqrestore(&rf->qh_list_lock, flags); + ret = irdma_mcast_cqp_op(iwdev, &mc_qht_elem->mc_grp_ctx, + IRDMA_OP_MC_DESTROY); + if (ret) { + ibdev_dbg(&iwdev->ibdev, + "VERBS: failed MC_DESTROY MCG\n"); + spin_lock_irqsave(&rf->qh_list_lock, flags); + mcast_list_add(rf, mc_qht_elem); + spin_unlock_irqrestore(&rf->qh_list_lock, flags); + return -EAGAIN; + } + + dma_free_coherent(rf->hw.device, + mc_qht_elem->mc_grp_ctx.dma_mem_mc.size, + mc_qht_elem->mc_grp_ctx.dma_mem_mc.va, + mc_qht_elem->mc_grp_ctx.dma_mem_mc.pa); + mc_qht_elem->mc_grp_ctx.dma_mem_mc.va = NULL; + irdma_free_rsrc(rf, rf->allocated_mcgs, + mc_qht_elem->mc_grp_ctx.mg_id); + kfree(mc_qht_elem); + } else { + spin_unlock_irqrestore(&rf->qh_list_lock, flags); + ret = irdma_mcast_cqp_op(iwdev, &mc_qht_elem->mc_grp_ctx, + IRDMA_OP_MC_MODIFY); + if (ret) { + ibdev_dbg(&iwdev->ibdev, + "VERBS: failed Modify MCG\n"); + return ret; + } + } + + return 0; +} + +/** + * irdma_create_ah - create address handle + * @ibah: address handle + * @attr: address handle attributes + * @udata: User data + * + * returns 0 on success, error otherwise + */ +static int irdma_create_ah(struct ib_ah *ibah, + struct rdma_ah_init_attr *attr, + struct ib_udata *udata) +{ + struct irdma_pd *pd = to_iwpd(ibah->pd); + struct irdma_ah *ah = container_of(ibah, struct irdma_ah, ibah); + struct rdma_ah_attr *ah_attr = attr->ah_attr; + const struct ib_gid_attr *sgid_attr; + struct irdma_device *iwdev = to_iwdev(ibah->pd->device); + struct irdma_pci_f *rf = iwdev->rf; + struct irdma_sc_ah *sc_ah; + u32 ah_id = 0; + struct irdma_ah_info *ah_info; + struct irdma_create_ah_resp uresp; + union { + struct sockaddr saddr; + struct sockaddr_in saddr_in; + struct sockaddr_in6 saddr_in6; + } sgid_addr, dgid_addr; + int err; + u8 dmac[ETH_ALEN]; + + err = irdma_alloc_rsrc(rf, rf->allocated_ahs, rf->max_ah, &ah_id, + &rf->next_ah); + if (err) + return err; + + ah->pd = pd; + sc_ah = &ah->sc_ah; + sc_ah->ah_info.ah_idx = ah_id; + sc_ah->ah_info.vsi = &iwdev->vsi; + irdma_sc_init_ah(&rf->sc_dev, sc_ah); + ah->sgid_index = ah_attr->grh.sgid_index; + sgid_attr = ah_attr->grh.sgid_attr; + memcpy(&ah->dgid, &ah_attr->grh.dgid, sizeof(ah->dgid)); + rdma_gid2ip((struct sockaddr *)&sgid_addr, &sgid_attr->gid); + rdma_gid2ip((struct sockaddr *)&dgid_addr, &ah_attr->grh.dgid); + ah->av.attrs = *ah_attr; + ah->av.net_type = rdma_gid_attr_network_type(sgid_attr); + ah->av.sgid_addr.saddr = sgid_addr.saddr; + ah->av.dgid_addr.saddr = dgid_addr.saddr; + ah_info = &sc_ah->ah_info; + ah_info->ah_idx = ah_id; + ah_info->pd_idx = pd->sc_pd.pd_id; + if (ah_attr->ah_flags & IB_AH_GRH) { + ah_info->flow_label = ah_attr->grh.flow_label; + ah_info->hop_ttl = ah_attr->grh.hop_limit; + ah_info->tc_tos = ah_attr->grh.traffic_class; + } + + ether_addr_copy(dmac, ah_attr->roce.dmac); + if (rdma_gid_attr_network_type(sgid_attr) == RDMA_NETWORK_IPV4) { + ah_info->ipv4_valid = true; + ah_info->dest_ip_addr[0] = + ntohl(dgid_addr.saddr_in.sin_addr.s_addr); + ah_info->src_ip_addr[0] = + ntohl(sgid_addr.saddr_in.sin_addr.s_addr); + ah_info->do_lpbk = irdma_ipv4_is_lpb(ah_info->src_ip_addr[0], + ah_info->dest_ip_addr[0]); + if (ipv4_is_multicast(dgid_addr.saddr_in.sin_addr.s_addr)) { + ah_info->do_lpbk = true; + irdma_mcast_mac(ah_info->dest_ip_addr, dmac, true); + } + } else { + irdma_copy_ip_ntohl(ah_info->dest_ip_addr, + dgid_addr.saddr_in6.sin6_addr.in6_u.u6_addr32); + irdma_copy_ip_ntohl(ah_info->src_ip_addr, + sgid_addr.saddr_in6.sin6_addr.in6_u.u6_addr32); + ah_info->do_lpbk = irdma_ipv6_is_lpb(ah_info->src_ip_addr, + ah_info->dest_ip_addr); + if (rdma_is_multicast_addr(&dgid_addr.saddr_in6.sin6_addr)) { + ah_info->do_lpbk = true; + irdma_mcast_mac(ah_info->dest_ip_addr, dmac, false); + } + } + + err = rdma_read_gid_l2_fields(sgid_attr, &ah_info->vlan_tag, + ah_info->mac_addr); + if (err) + goto error; + + ah_info->dst_arpindex = irdma_add_arp(iwdev->rf, ah_info->dest_ip_addr, + ah_info->ipv4_valid, dmac); + + if (ah_info->dst_arpindex == -1) { + err = -EINVAL; + goto error; + } + + if (ah_info->vlan_tag >= VLAN_N_VID && iwdev->dcb) + ah_info->vlan_tag = 0; + + if (ah_info->vlan_tag < VLAN_N_VID) { + ah_info->insert_vlan_tag = true; + ah_info->vlan_tag |= + rt_tos2priority(ah_info->tc_tos) << VLAN_PRIO_SHIFT; + } + + err = irdma_ah_cqp_op(iwdev->rf, sc_ah, IRDMA_OP_AH_CREATE, + attr->flags & RDMA_CREATE_AH_SLEEPABLE, + irdma_gsi_ud_qp_ah_cb, sc_ah); + + if (err) { + ibdev_dbg(&iwdev->ibdev, + "VERBS: CQP-OP Create AH fail"); + goto error; + } + + if (!(attr->flags & RDMA_CREATE_AH_SLEEPABLE)) { + int cnt = CQP_COMPL_WAIT_TIME_MS * CQP_TIMEOUT_THRESHOLD; + + do { + irdma_cqp_ce_handler(rf, &rf->ccq.sc_cq); + mdelay(1); + } while (!sc_ah->ah_info.ah_valid && --cnt); + + if (!cnt) { + ibdev_dbg(&iwdev->ibdev, + "VERBS: CQP create AH timed out"); + err = -ETIMEDOUT; + goto error; + } + } + + if (udata) { + uresp.ah_id = ah->sc_ah.ah_info.ah_idx; + err = ib_copy_to_udata(udata, &uresp, + min(sizeof(uresp), udata->outlen)); + } + return 0; + +error: + irdma_free_rsrc(iwdev->rf, iwdev->rf->allocated_ahs, ah_id); + + return err; +} + +/** + * irdma_destroy_ah - Destroy address handle + * @ibah: pointer to address handle + * @ah_flags: flags for sleepable + */ +static int irdma_destroy_ah(struct ib_ah *ibah, u32 ah_flags) +{ + struct irdma_device *iwdev = to_iwdev(ibah->device); + struct irdma_ah *ah = to_iwah(ibah); + + irdma_ah_cqp_op(iwdev->rf, &ah->sc_ah, IRDMA_OP_AH_DESTROY, + false, NULL, ah); + + irdma_free_rsrc(iwdev->rf, iwdev->rf->allocated_ahs, + ah->sc_ah.ah_info.ah_idx); + + return 0; +} + +/** + * irdma_query_ah - Query address handle + * @ibah: pointer to address handle + * @ah_attr: address handle attributes + */ +static int irdma_query_ah(struct ib_ah *ibah, struct rdma_ah_attr *ah_attr) +{ + struct irdma_ah *ah = to_iwah(ibah); + + memset(ah_attr, 0, sizeof(*ah_attr)); + if (ah->av.attrs.ah_flags & IB_AH_GRH) { + ah_attr->ah_flags = IB_AH_GRH; + ah_attr->grh.flow_label = ah->sc_ah.ah_info.flow_label; + ah_attr->grh.traffic_class = ah->sc_ah.ah_info.tc_tos; + ah_attr->grh.hop_limit = ah->sc_ah.ah_info.hop_ttl; + ah_attr->grh.sgid_index = ah->sgid_index; + ah_attr->grh.sgid_index = ah->sgid_index; + memcpy(&ah_attr->grh.dgid, &ah->dgid, + sizeof(ah_attr->grh.dgid)); + } + + return 0; +} + +static enum rdma_link_layer irdma_get_link_layer(struct ib_device *ibdev, + u32 port_num) +{ + return IB_LINK_LAYER_ETHERNET; +} + +static __be64 irdma_mac_to_guid(struct net_device *ndev) +{ + unsigned char *mac = ndev->dev_addr; + __be64 guid; + unsigned char *dst = (unsigned char *)&guid; + + dst[0] = mac[0] ^ 2; + dst[1] = mac[1]; + dst[2] = mac[2]; + dst[3] = 0xff; + dst[4] = 0xfe; + dst[5] = mac[3]; + dst[6] = mac[4]; + dst[7] = mac[5]; + + return guid; +} + +static const struct ib_device_ops irdma_roce_dev_ops = { + .attach_mcast = irdma_attach_mcast, + .create_ah = irdma_create_ah, + .create_user_ah = irdma_create_ah, + .destroy_ah = irdma_destroy_ah, + .detach_mcast = irdma_detach_mcast, + .get_link_layer = irdma_get_link_layer, + .get_port_immutable = irdma_roce_port_immutable, + .modify_qp = irdma_modify_qp_roce, + .query_ah = irdma_query_ah, + .query_pkey = irdma_query_pkey, +}; + +static const struct ib_device_ops irdma_iw_dev_ops = { + .modify_qp = irdma_modify_qp, + .get_port_immutable = irdma_iw_port_immutable, + .query_gid = irdma_query_gid, +}; + +static const struct ib_device_ops irdma_dev_ops = { + .owner = THIS_MODULE, + .driver_id = RDMA_DRIVER_IRDMA, + .uverbs_abi_ver = IRDMA_ABI_VER, + + .alloc_hw_stats = irdma_alloc_hw_stats, + .alloc_mr = irdma_alloc_mr, + .alloc_mw = irdma_alloc_mw, + .alloc_pd = irdma_alloc_pd, + .alloc_ucontext = irdma_alloc_ucontext, + .create_cq = irdma_create_cq, + .create_qp = irdma_create_qp, + .dealloc_driver = irdma_ib_dealloc_device, + .dealloc_mw = irdma_dealloc_mw, + .dealloc_pd = irdma_dealloc_pd, + .dealloc_ucontext = irdma_dealloc_ucontext, + .dereg_mr = irdma_dereg_mr, + .destroy_cq = irdma_destroy_cq, + .destroy_qp = irdma_destroy_qp, + .disassociate_ucontext = irdma_disassociate_ucontext, + .get_dev_fw_str = irdma_get_dev_fw_str, + .get_dma_mr = irdma_get_dma_mr, + .get_hw_stats = irdma_get_hw_stats, + .map_mr_sg = irdma_map_mr_sg, + .mmap = irdma_mmap, + .mmap_free = irdma_mmap_free, + .poll_cq = irdma_poll_cq, + .post_recv = irdma_post_recv, + .post_send = irdma_post_send, + .query_device = irdma_query_device, + .query_port = irdma_query_port, + .query_qp = irdma_query_qp, + .reg_user_mr = irdma_reg_user_mr, + .req_notify_cq = irdma_req_notify_cq, + .resize_cq = irdma_resize_cq, + INIT_RDMA_OBJ_SIZE(ib_pd, irdma_pd, ibpd), + INIT_RDMA_OBJ_SIZE(ib_ucontext, irdma_ucontext, ibucontext), + INIT_RDMA_OBJ_SIZE(ib_ah, irdma_ah, ibah), + INIT_RDMA_OBJ_SIZE(ib_cq, irdma_cq, ibcq), + INIT_RDMA_OBJ_SIZE(ib_mw, irdma_mr, ibmw), +}; + +/** + * irdma_init_roce_device - initialization of roce rdma device + * @iwdev: irdma device + */ +static void irdma_init_roce_device(struct irdma_device *iwdev) +{ + iwdev->ibdev.node_type = RDMA_NODE_IB_CA; + iwdev->ibdev.node_guid = irdma_mac_to_guid(iwdev->netdev); + ib_set_device_ops(&iwdev->ibdev, &irdma_roce_dev_ops); +} + +/** + * irdma_init_iw_device - initialization of iwarp rdma device + * @iwdev: irdma device + */ +static int irdma_init_iw_device(struct irdma_device *iwdev) +{ + struct net_device *netdev = iwdev->netdev; + + iwdev->ibdev.node_type = RDMA_NODE_RNIC; + ether_addr_copy((u8 *)&iwdev->ibdev.node_guid, netdev->dev_addr); + iwdev->ibdev.ops.iw_add_ref = irdma_qp_add_ref; + iwdev->ibdev.ops.iw_rem_ref = irdma_qp_rem_ref; + iwdev->ibdev.ops.iw_get_qp = irdma_get_qp; + iwdev->ibdev.ops.iw_connect = irdma_connect; + iwdev->ibdev.ops.iw_accept = irdma_accept; + iwdev->ibdev.ops.iw_reject = irdma_reject; + iwdev->ibdev.ops.iw_create_listen = irdma_create_listen; + iwdev->ibdev.ops.iw_destroy_listen = irdma_destroy_listen; + memcpy(iwdev->ibdev.iw_ifname, netdev->name, + sizeof(iwdev->ibdev.iw_ifname)); + ib_set_device_ops(&iwdev->ibdev, &irdma_iw_dev_ops); + + return 0; +} + +/** + * irdma_init_rdma_device - initialization of rdma device + * @iwdev: irdma device + */ +static int irdma_init_rdma_device(struct irdma_device *iwdev) +{ + struct pci_dev *pcidev = iwdev->rf->pcidev; + int ret; + + if (iwdev->roce_mode) { + irdma_init_roce_device(iwdev); + } else { + ret = irdma_init_iw_device(iwdev); + if (ret) + return ret; + } + iwdev->ibdev.phys_port_cnt = 1; + iwdev->ibdev.num_comp_vectors = iwdev->rf->ceqs_count; + iwdev->ibdev.dev.parent = &pcidev->dev; + ib_set_device_ops(&iwdev->ibdev, &irdma_dev_ops); + + return 0; +} + +/** + * irdma_port_ibevent - indicate port event + * @iwdev: irdma device + */ +void irdma_port_ibevent(struct irdma_device *iwdev) +{ + struct ib_event event; + + event.device = &iwdev->ibdev; + event.element.port_num = 1; + event.event = + iwdev->iw_status ? IB_EVENT_PORT_ACTIVE : IB_EVENT_PORT_ERR; + ib_dispatch_event(&event); +} + +/** + * irdma_ib_unregister_device - unregister rdma device from IB + * core + * @iwdev: irdma device + */ +void irdma_ib_unregister_device(struct irdma_device *iwdev) +{ + iwdev->iw_status = 0; + irdma_port_ibevent(iwdev); + ib_unregister_device(&iwdev->ibdev); +} + +/** + * irdma_ib_register_device - register irdma device to IB core + * @iwdev: irdma device + */ +int irdma_ib_register_device(struct irdma_device *iwdev) +{ + int ret; + + ret = irdma_init_rdma_device(iwdev); + if (ret) + return ret; + + ret = ib_device_set_netdev(&iwdev->ibdev, iwdev->netdev, 1); + if (ret) + goto error; + dma_set_max_seg_size(iwdev->rf->hw.device, UINT_MAX); + ret = ib_register_device(&iwdev->ibdev, "irdma%d", iwdev->rf->hw.device); + if (ret) + goto error; + + iwdev->iw_status = 1; + irdma_port_ibevent(iwdev); + + return 0; + +error: + if (ret) + ibdev_dbg(&iwdev->ibdev, "VERBS: Register RDMA device fail\n"); + + return ret; +} + +/** + * irdma_ib_dealloc_device + * @ibdev: ib device + * + * callback from ibdev dealloc_driver to deallocate resources + * unber irdma device + */ +void irdma_ib_dealloc_device(struct ib_device *ibdev) +{ + struct irdma_device *iwdev = to_iwdev(ibdev); + + irdma_rt_deinit_hw(iwdev); + irdma_ctrl_deinit_hw(iwdev->rf); + kfree(iwdev->rf); +} diff --git a/drivers/infiniband/hw/irdma/verbs.h b/drivers/infiniband/hw/irdma/verbs.h new file mode 100644 index 000000000000..5c244cd321a3 --- /dev/null +++ b/drivers/infiniband/hw/irdma/verbs.h @@ -0,0 +1,225 @@ +/* SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB */ +/* Copyright (c) 2015 - 2021 Intel Corporation */ +#ifndef IRDMA_VERBS_H +#define IRDMA_VERBS_H + +#define IRDMA_MAX_SAVED_PHY_PGADDR 4 + +#define IRDMA_PKEY_TBL_SZ 1 +#define IRDMA_DEFAULT_PKEY 0xFFFF + +struct irdma_ucontext { + struct ib_ucontext ibucontext; + struct irdma_device *iwdev; + struct rdma_user_mmap_entry *db_mmap_entry; + struct list_head cq_reg_mem_list; + spinlock_t cq_reg_mem_list_lock; /* protect CQ memory list */ + struct list_head qp_reg_mem_list; + spinlock_t qp_reg_mem_list_lock; /* protect QP memory list */ + int abi_ver; + bool legacy_mode; +}; + +struct irdma_pd { + struct ib_pd ibpd; + struct irdma_sc_pd sc_pd; +}; + +struct irdma_av { + u8 macaddr[16]; + struct rdma_ah_attr attrs; + union { + struct sockaddr saddr; + struct sockaddr_in saddr_in; + struct sockaddr_in6 saddr_in6; + } sgid_addr, dgid_addr; + u8 net_type; +}; + +struct irdma_ah { + struct ib_ah ibah; + struct irdma_sc_ah sc_ah; + struct irdma_pd *pd; + struct irdma_av av; + u8 sgid_index; + union ib_gid dgid; +}; + +struct irdma_hmc_pble { + union { + u32 idx; + dma_addr_t addr; + }; +}; + +struct irdma_cq_mr { + struct irdma_hmc_pble cq_pbl; + dma_addr_t shadow; + bool split; +}; + +struct irdma_qp_mr { + struct irdma_hmc_pble sq_pbl; + struct irdma_hmc_pble rq_pbl; + dma_addr_t shadow; + struct page *sq_page; +}; + +struct irdma_cq_buf { + struct irdma_dma_mem kmem_buf; + struct irdma_cq_uk cq_uk; + struct irdma_hw *hw; + struct list_head list; + struct work_struct work; +}; + +struct irdma_pbl { + struct list_head list; + union { + struct irdma_qp_mr qp_mr; + struct irdma_cq_mr cq_mr; + }; + + bool pbl_allocated:1; + bool on_list:1; + u64 user_base; + struct irdma_pble_alloc pble_alloc; + struct irdma_mr *iwmr; +}; + +struct irdma_mr { + union { + struct ib_mr ibmr; + struct ib_mw ibmw; + }; + struct ib_umem *region; + u16 type; + u32 page_cnt; + u64 page_size; + u32 npages; + u32 stag; + u64 len; + u64 pgaddrmem[IRDMA_MAX_SAVED_PHY_PGADDR]; + struct irdma_pbl iwpbl; +}; + +struct irdma_cq { + struct ib_cq ibcq; + struct irdma_sc_cq sc_cq; + u16 cq_head; + u16 cq_size; + u16 cq_num; + bool user_mode; + u32 polled_cmpls; + u32 cq_mem_size; + struct irdma_dma_mem kmem; + struct irdma_dma_mem kmem_shadow; + spinlock_t lock; /* for poll cq */ + struct irdma_pbl *iwpbl; + struct irdma_pbl *iwpbl_shadow; + struct list_head resize_list; + struct irdma_cq_poll_info cur_cqe; +}; + +struct disconn_work { + struct work_struct work; + struct irdma_qp *iwqp; +}; + +struct iw_cm_id; + +struct irdma_qp_kmode { + struct irdma_dma_mem dma_mem; + struct irdma_sq_uk_wr_trk_info *sq_wrid_mem; + u64 *rq_wrid_mem; +}; + +struct irdma_qp { + struct ib_qp ibqp; + struct irdma_sc_qp sc_qp; + struct irdma_device *iwdev; + struct irdma_cq *iwscq; + struct irdma_cq *iwrcq; + struct irdma_pd *iwpd; + struct rdma_user_mmap_entry *push_wqe_mmap_entry; + struct rdma_user_mmap_entry *push_db_mmap_entry; + struct irdma_qp_host_ctx_info ctx_info; + union { + struct irdma_iwarp_offload_info iwarp_info; + struct irdma_roce_offload_info roce_info; + }; + + union { + struct irdma_tcp_offload_info tcp_info; + struct irdma_udp_offload_info udp_info; + }; + + struct irdma_ah roce_ah; + struct list_head teardown_entry; + refcount_t refcnt; + struct iw_cm_id *cm_id; + struct irdma_cm_node *cm_node; + struct ib_mr *lsmm_mr; + atomic_t hw_mod_qp_pend; + enum ib_qp_state ibqp_state; + u32 qp_mem_size; + u32 last_aeq; + int max_send_wr; + int max_recv_wr; + atomic_t close_timer_started; + spinlock_t lock; /* serialize posting WRs to SQ/RQ */ + struct irdma_qp_context *iwqp_context; + void *pbl_vbase; + dma_addr_t pbl_pbase; + struct page *page; + u8 active_conn : 1; + u8 user_mode : 1; + u8 hte_added : 1; + u8 flush_issued : 1; + u8 sig_all : 1; + u8 pau_mode : 1; + u8 rsvd : 1; + u8 iwarp_state; + u16 term_sq_flush_code; + u16 term_rq_flush_code; + u8 hw_iwarp_state; + u8 hw_tcp_state; + struct irdma_qp_kmode kqp; + struct irdma_dma_mem host_ctx; + struct timer_list terminate_timer; + struct irdma_pbl *iwpbl; + struct irdma_dma_mem q2_ctx_mem; + struct irdma_dma_mem ietf_mem; + struct completion free_qp; + wait_queue_head_t waitq; + wait_queue_head_t mod_qp_waitq; + u8 rts_ae_rcvd; +}; + +enum irdma_mmap_flag { + IRDMA_MMAP_IO_NC, + IRDMA_MMAP_IO_WC, +}; + +struct irdma_user_mmap_entry { + struct rdma_user_mmap_entry rdma_entry; + u64 bar_offset; + u8 mmap_flag; +}; + +static inline u16 irdma_fw_major_ver(struct irdma_sc_dev *dev) +{ + return (u16)FIELD_GET(IRDMA_FW_VER_MAJOR, dev->feature_info[IRDMA_FEATURE_FW_INFO]); +} + +static inline u16 irdma_fw_minor_ver(struct irdma_sc_dev *dev) +{ + return (u16)FIELD_GET(IRDMA_FW_VER_MINOR, dev->feature_info[IRDMA_FEATURE_FW_INFO]); +} + +void irdma_mcast_mac(u32 *ip_addr, u8 *mac, bool ipv4); +int irdma_ib_register_device(struct irdma_device *iwdev); +void irdma_ib_unregister_device(struct irdma_device *iwdev); +void irdma_ib_dealloc_device(struct ib_device *ibdev); +void irdma_ib_qp_event(struct irdma_qp *iwqp, enum irdma_qp_event_type event); +#endif /* IRDMA_VERBS_H */ diff --git a/include/uapi/rdma/ib_user_ioctl_verbs.h b/include/uapi/rdma/ib_user_ioctl_verbs.h index 22483799cd07..3072e5d6b692 100644 --- a/include/uapi/rdma/ib_user_ioctl_verbs.h +++ b/include/uapi/rdma/ib_user_ioctl_verbs.h @@ -240,6 +240,7 @@ enum rdma_driver_id { RDMA_DRIVER_OCRDMA, RDMA_DRIVER_NES, RDMA_DRIVER_I40IW, + RDMA_DRIVER_IRDMA = RDMA_DRIVER_I40IW, RDMA_DRIVER_VMW_PVRDMA, RDMA_DRIVER_QEDR, RDMA_DRIVER_HNS, -- cgit v1.2.3 From 48d6b3336a9fc570b48126c4409abf94dd5c5e8a Mon Sep 17 00:00:00 2001 From: Mustafa Ismail Date: Wed, 2 Jun 2021 15:51:36 -0500 Subject: RDMA/irdma: Add ABI definitions Add ABI definitions for irdma. Link: https://lore.kernel.org/r/20210602205138.889-15-shiraz.saleem@intel.com Signed-off-by: Mustafa Ismail Signed-off-by: Shiraz Saleem Signed-off-by: Jason Gunthorpe --- include/uapi/rdma/irdma-abi.h | 111 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 111 insertions(+) create mode 100644 include/uapi/rdma/irdma-abi.h (limited to 'include/uapi') diff --git a/include/uapi/rdma/irdma-abi.h b/include/uapi/rdma/irdma-abi.h new file mode 100644 index 000000000000..26b638a7ad97 --- /dev/null +++ b/include/uapi/rdma/irdma-abi.h @@ -0,0 +1,111 @@ +/* SPDX-License-Identifier: (GPL-2.0 WITH Linux-syscall-note) OR Linux-OpenIB) */ +/* + * Copyright (c) 2006 - 2021 Intel Corporation. All rights reserved. + * Copyright (c) 2005 Topspin Communications. All rights reserved. + * Copyright (c) 2005 Cisco Systems. All rights reserved. + * Copyright (c) 2005 Open Grid Computing, Inc. All rights reserved. + */ + +#ifndef IRDMA_ABI_H +#define IRDMA_ABI_H + +#include + +/* irdma must support legacy GEN_1 i40iw kernel + * and user-space whose last ABI ver is 5 + */ +#define IRDMA_ABI_VER 5 + +enum irdma_memreg_type { + IRDMA_MEMREG_TYPE_MEM = 0, + IRDMA_MEMREG_TYPE_QP = 1, + IRDMA_MEMREG_TYPE_CQ = 2, +}; + +struct irdma_alloc_ucontext_req { + __u32 rsvd32; + __u8 userspace_ver; + __u8 rsvd8[3]; +}; + +struct irdma_alloc_ucontext_resp { + __u32 max_pds; + __u32 max_qps; + __u32 wq_size; /* size of the WQs (SQ+RQ) in the mmaped area */ + __u8 kernel_ver; + __u8 rsvd[3]; + __aligned_u64 feature_flags; + __aligned_u64 db_mmap_key; + __u32 max_hw_wq_frags; + __u32 max_hw_read_sges; + __u32 max_hw_inline; + __u32 max_hw_rq_quanta; + __u32 max_hw_wq_quanta; + __u32 min_hw_cq_size; + __u32 max_hw_cq_size; + __u16 max_hw_sq_chunk; + __u8 hw_rev; + __u8 rsvd2; +}; + +struct irdma_alloc_pd_resp { + __u32 pd_id; + __u8 rsvd[4]; +}; + +struct irdma_resize_cq_req { + __aligned_u64 user_cq_buffer; +}; + +struct irdma_create_cq_req { + __aligned_u64 user_cq_buf; + __aligned_u64 user_shadow_area; +}; + +struct irdma_create_qp_req { + __aligned_u64 user_wqe_bufs; + __aligned_u64 user_compl_ctx; +}; + +struct irdma_mem_reg_req { + __u16 reg_type; /* enum irdma_memreg_type */ + __u16 cq_pages; + __u16 rq_pages; + __u16 sq_pages; +}; + +struct irdma_modify_qp_req { + __u8 sq_flush; + __u8 rq_flush; + __u8 rsvd[6]; +}; + +struct irdma_create_cq_resp { + __u32 cq_id; + __u32 cq_size; +}; + +struct irdma_create_qp_resp { + __u32 qp_id; + __u32 actual_sq_size; + __u32 actual_rq_size; + __u32 irdma_drv_opt; + __u16 push_idx; + __u8 lsmm; + __u8 rsvd; + __u32 qp_caps; +}; + +struct irdma_modify_qp_resp { + __aligned_u64 push_wqe_mmap_key; + __aligned_u64 push_db_mmap_key; + __u16 push_offset; + __u8 push_valid; + __u8 rsvd[5]; +}; + +struct irdma_create_ah_resp { + __u32 ah_id; + __u8 rsvd[4]; +}; +#endif /* IRDMA_ABI_H */ -- cgit v1.2.3 From fa0cf568fd76550c1ddb806c03a65a1a4a1ea909 Mon Sep 17 00:00:00 2001 From: Shiraz Saleem Date: Wed, 2 Jun 2021 15:51:37 -0500 Subject: RDMA/irdma: Add irdma Kconfig/Makefile and remove i40iw Add Kconfig and Makefile to build irdma driver. Remove i40iw driver and add an alias in irdma. Remove legacy exported symbols i40e_register_client and i40e_unregister_client from i40e as they are no longer used. irdma is the replacement driver that supports X722. Link: https://lore.kernel.org/r/20210602205138.889-16-shiraz.saleem@intel.com Signed-off-by: Shiraz Saleem Signed-off-by: Jason Gunthorpe --- Documentation/ABI/stable/sysfs-class-infiniband | 20 - MAINTAINERS | 8 - drivers/infiniband/Kconfig | 2 +- drivers/infiniband/hw/Makefile | 2 +- drivers/infiniband/hw/i40iw/Kconfig | 9 - drivers/infiniband/hw/i40iw/Makefile | 9 - drivers/infiniband/hw/i40iw/i40iw.h | 602 --- drivers/infiniband/hw/i40iw/i40iw_cm.c | 4419 ------------------- drivers/infiniband/hw/i40iw/i40iw_cm.h | 462 -- drivers/infiniband/hw/i40iw/i40iw_ctrl.c | 5243 ----------------------- drivers/infiniband/hw/i40iw/i40iw_d.h | 1746 -------- drivers/infiniband/hw/i40iw/i40iw_hmc.c | 821 ---- drivers/infiniband/hw/i40iw/i40iw_hmc.h | 241 -- drivers/infiniband/hw/i40iw/i40iw_hw.c | 851 ---- drivers/infiniband/hw/i40iw/i40iw_main.c | 2064 --------- drivers/infiniband/hw/i40iw/i40iw_osdep.h | 195 - drivers/infiniband/hw/i40iw/i40iw_p.h | 129 - drivers/infiniband/hw/i40iw/i40iw_pble.c | 611 --- drivers/infiniband/hw/i40iw/i40iw_pble.h | 131 - drivers/infiniband/hw/i40iw/i40iw_puda.c | 1496 ------- drivers/infiniband/hw/i40iw/i40iw_puda.h | 188 - drivers/infiniband/hw/i40iw/i40iw_register.h | 1030 ----- drivers/infiniband/hw/i40iw/i40iw_status.h | 101 - drivers/infiniband/hw/i40iw/i40iw_type.h | 1358 ------ drivers/infiniband/hw/i40iw/i40iw_uk.c | 1200 ------ drivers/infiniband/hw/i40iw/i40iw_user.h | 422 -- drivers/infiniband/hw/i40iw/i40iw_utils.c | 1518 ------- drivers/infiniband/hw/i40iw/i40iw_verbs.c | 2652 ------------ drivers/infiniband/hw/i40iw/i40iw_verbs.h | 179 - drivers/infiniband/hw/i40iw/i40iw_vf.c | 85 - drivers/infiniband/hw/i40iw/i40iw_vf.h | 62 - drivers/infiniband/hw/i40iw/i40iw_virtchnl.c | 759 ---- drivers/infiniband/hw/i40iw/i40iw_virtchnl.h | 124 - drivers/infiniband/hw/irdma/Kconfig | 12 + drivers/infiniband/hw/irdma/Makefile | 27 + drivers/net/ethernet/intel/i40e/i40e_client.c | 149 - include/linux/net/intel/i40e_client.h | 3 - include/uapi/rdma/i40iw-abi.h | 107 - 38 files changed, 41 insertions(+), 28996 deletions(-) delete mode 100644 drivers/infiniband/hw/i40iw/Kconfig delete mode 100644 drivers/infiniband/hw/i40iw/Makefile delete mode 100644 drivers/infiniband/hw/i40iw/i40iw.h delete mode 100644 drivers/infiniband/hw/i40iw/i40iw_cm.c delete mode 100644 drivers/infiniband/hw/i40iw/i40iw_cm.h delete mode 100644 drivers/infiniband/hw/i40iw/i40iw_ctrl.c delete mode 100644 drivers/infiniband/hw/i40iw/i40iw_d.h delete mode 100644 drivers/infiniband/hw/i40iw/i40iw_hmc.c delete mode 100644 drivers/infiniband/hw/i40iw/i40iw_hmc.h delete mode 100644 drivers/infiniband/hw/i40iw/i40iw_hw.c delete mode 100644 drivers/infiniband/hw/i40iw/i40iw_main.c delete mode 100644 drivers/infiniband/hw/i40iw/i40iw_osdep.h delete mode 100644 drivers/infiniband/hw/i40iw/i40iw_p.h delete mode 100644 drivers/infiniband/hw/i40iw/i40iw_pble.c delete mode 100644 drivers/infiniband/hw/i40iw/i40iw_pble.h delete mode 100644 drivers/infiniband/hw/i40iw/i40iw_puda.c delete mode 100644 drivers/infiniband/hw/i40iw/i40iw_puda.h delete mode 100644 drivers/infiniband/hw/i40iw/i40iw_register.h delete mode 100644 drivers/infiniband/hw/i40iw/i40iw_status.h delete mode 100644 drivers/infiniband/hw/i40iw/i40iw_type.h delete mode 100644 drivers/infiniband/hw/i40iw/i40iw_uk.c delete mode 100644 drivers/infiniband/hw/i40iw/i40iw_user.h delete mode 100644 drivers/infiniband/hw/i40iw/i40iw_utils.c delete mode 100644 drivers/infiniband/hw/i40iw/i40iw_verbs.c delete mode 100644 drivers/infiniband/hw/i40iw/i40iw_verbs.h delete mode 100644 drivers/infiniband/hw/i40iw/i40iw_vf.c delete mode 100644 drivers/infiniband/hw/i40iw/i40iw_vf.h delete mode 100644 drivers/infiniband/hw/i40iw/i40iw_virtchnl.c delete mode 100644 drivers/infiniband/hw/i40iw/i40iw_virtchnl.h create mode 100644 drivers/infiniband/hw/irdma/Kconfig create mode 100644 drivers/infiniband/hw/irdma/Makefile delete mode 100644 include/uapi/rdma/i40iw-abi.h (limited to 'include/uapi') diff --git a/Documentation/ABI/stable/sysfs-class-infiniband b/Documentation/ABI/stable/sysfs-class-infiniband index 348c4ac803ad..9b1bdfa43354 100644 --- a/Documentation/ABI/stable/sysfs-class-infiniband +++ b/Documentation/ABI/stable/sysfs-class-infiniband @@ -731,26 +731,6 @@ Description: is the irq number of "sdma3", and M is irq number of "sdma4" in the /proc/interrupts file. - -sysfs interface for Intel(R) X722 iWARP i40iw driver ----------------------------------------------------- - -What: /sys/class/infiniband/i40iwX/hw_rev -What: /sys/class/infiniband/i40iwX/hca_type -What: /sys/class/infiniband/i40iwX/board_id -Date: Jan, 2016 -KernelVersion: v4.10 -Contact: linux-rdma@vger.kernel.org -Description: - =============== ==== ======================== - hw_rev: (RO) Hardware revision number - - hca_type: (RO) Show HCA type (I40IW) - - board_id: (RO) I40IW board ID - =============== ==== ======================== - - sysfs interface for QLogic qedr NIC Driver ------------------------------------------ diff --git a/MAINTAINERS b/MAINTAINERS index 34d2bf36b5ad..05cab3b1c955 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -9365,14 +9365,6 @@ L: linux-pm@vger.kernel.org S: Supported F: drivers/cpufreq/intel_pstate.c -INTEL RDMA RNIC DRIVER -M: Faisal Latif -M: Shiraz Saleem -L: linux-rdma@vger.kernel.org -S: Supported -F: drivers/infiniband/hw/i40iw/ -F: include/uapi/rdma/i40iw-abi.h - INTEL SCU DRIVERS M: Mika Westerberg S: Maintained diff --git a/drivers/infiniband/Kconfig b/drivers/infiniband/Kconfig index 04a78d9f8fe3..33d3ce9c888e 100644 --- a/drivers/infiniband/Kconfig +++ b/drivers/infiniband/Kconfig @@ -82,7 +82,7 @@ source "drivers/infiniband/hw/mthca/Kconfig" source "drivers/infiniband/hw/qib/Kconfig" source "drivers/infiniband/hw/cxgb4/Kconfig" source "drivers/infiniband/hw/efa/Kconfig" -source "drivers/infiniband/hw/i40iw/Kconfig" +source "drivers/infiniband/hw/irdma/Kconfig" source "drivers/infiniband/hw/mlx4/Kconfig" source "drivers/infiniband/hw/mlx5/Kconfig" source "drivers/infiniband/hw/ocrdma/Kconfig" diff --git a/drivers/infiniband/hw/Makefile b/drivers/infiniband/hw/Makefile index 0aeccd984889..fba0b3be903e 100644 --- a/drivers/infiniband/hw/Makefile +++ b/drivers/infiniband/hw/Makefile @@ -3,7 +3,7 @@ obj-$(CONFIG_INFINIBAND_MTHCA) += mthca/ obj-$(CONFIG_INFINIBAND_QIB) += qib/ obj-$(CONFIG_INFINIBAND_CXGB4) += cxgb4/ obj-$(CONFIG_INFINIBAND_EFA) += efa/ -obj-$(CONFIG_INFINIBAND_I40IW) += i40iw/ +obj-$(CONFIG_INFINIBAND_IRDMA) += irdma/ obj-$(CONFIG_MLX4_INFINIBAND) += mlx4/ obj-$(CONFIG_MLX5_INFINIBAND) += mlx5/ obj-$(CONFIG_INFINIBAND_OCRDMA) += ocrdma/ diff --git a/drivers/infiniband/hw/i40iw/Kconfig b/drivers/infiniband/hw/i40iw/Kconfig deleted file mode 100644 index 7476f3b14c39..000000000000 --- a/drivers/infiniband/hw/i40iw/Kconfig +++ /dev/null @@ -1,9 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-only -config INFINIBAND_I40IW - tristate "Intel(R) Ethernet X722 iWARP Driver" - depends on INET && I40E - depends on IPV6 || !IPV6 - depends on PCI - select GENERIC_ALLOCATOR - help - Intel(R) Ethernet X722 iWARP Driver diff --git a/drivers/infiniband/hw/i40iw/Makefile b/drivers/infiniband/hw/i40iw/Makefile deleted file mode 100644 index 34da9eba8a7c..000000000000 --- a/drivers/infiniband/hw/i40iw/Makefile +++ /dev/null @@ -1,9 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0 - -obj-$(CONFIG_INFINIBAND_I40IW) += i40iw.o - -i40iw-objs :=\ - i40iw_cm.o i40iw_ctrl.o \ - i40iw_hmc.o i40iw_hw.o i40iw_main.o \ - i40iw_pble.o i40iw_puda.o i40iw_uk.o i40iw_utils.o \ - i40iw_verbs.o i40iw_virtchnl.o i40iw_vf.o diff --git a/drivers/infiniband/hw/i40iw/i40iw.h b/drivers/infiniband/hw/i40iw/i40iw.h deleted file mode 100644 index be4094ac4fac..000000000000 --- a/drivers/infiniband/hw/i40iw/i40iw.h +++ /dev/null @@ -1,602 +0,0 @@ -/******************************************************************************* -* -* Copyright (c) 2015-2016 Intel Corporation. All rights reserved. -* -* This software is available to you under a choice of one of two -* licenses. You may choose to be licensed under the terms of the GNU -* General Public License (GPL) Version 2, available from the file -* COPYING in the main directory of this source tree, or the -* OpenFabrics.org BSD license below: -* -* Redistribution and use in source and binary forms, with or -* without modification, are permitted provided that the following -* conditions are met: -* -* - Redistributions of source code must retain the above -* copyright notice, this list of conditions and the following -* disclaimer. -* -* - Redistributions in binary form must reproduce the above -* copyright notice, this list of conditions and the following -* disclaimer in the documentation and/or other materials -* provided with the distribution. -* -* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS -* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN -* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -* SOFTWARE. -* -*******************************************************************************/ - -#ifndef I40IW_IW_H -#define I40IW_IW_H -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "i40iw_status.h" -#include "i40iw_osdep.h" -#include "i40iw_d.h" -#include "i40iw_hmc.h" - -#include "i40iw_type.h" -#include "i40iw_p.h" -#include -#include "i40iw_pble.h" -#include "i40iw_verbs.h" -#include "i40iw_cm.h" -#include "i40iw_user.h" -#include "i40iw_puda.h" - -#define I40IW_FW_VER_DEFAULT 2 -#define I40IW_HW_VERSION 2 - -#define I40IW_ARP_ADD 1 -#define I40IW_ARP_DELETE 2 -#define I40IW_ARP_RESOLVE 3 - -#define I40IW_MACIP_ADD 1 -#define I40IW_MACIP_DELETE 2 - -#define IW_CCQ_SIZE (I40IW_CQP_SW_SQSIZE_2048 + 1) -#define IW_CEQ_SIZE 2048 -#define IW_AEQ_SIZE 2048 - -#define RX_BUF_SIZE (1536 + 8) -#define IW_REG0_SIZE (4 * 1024) -#define IW_TX_TIMEOUT (6 * HZ) -#define IW_FIRST_QPN 1 -#define IW_SW_CONTEXT_ALIGN 1024 - -#define MAX_DPC_ITERATIONS 128 - -#define I40IW_EVENT_TIMEOUT 100000 -#define I40IW_VCHNL_EVENT_TIMEOUT 100000 - -#define I40IW_NO_VLAN 0xffff -#define I40IW_NO_QSET 0xffff - -/* access to mcast filter list */ -#define IW_ADD_MCAST false -#define IW_DEL_MCAST true - -#define I40IW_DRV_OPT_ENABLE_MPA_VER_0 0x00000001 -#define I40IW_DRV_OPT_DISABLE_MPA_CRC 0x00000002 -#define I40IW_DRV_OPT_DISABLE_FIRST_WRITE 0x00000004 -#define I40IW_DRV_OPT_DISABLE_INTF 0x00000008 -#define I40IW_DRV_OPT_ENABLE_MSI 0x00000010 -#define I40IW_DRV_OPT_DUAL_LOGICAL_PORT 0x00000020 -#define I40IW_DRV_OPT_NO_INLINE_DATA 0x00000080 -#define I40IW_DRV_OPT_DISABLE_INT_MOD 0x00000100 -#define I40IW_DRV_OPT_DISABLE_VIRT_WQ 0x00000200 -#define I40IW_DRV_OPT_ENABLE_PAU 0x00000400 -#define I40IW_DRV_OPT_MCAST_LOGPORT_MAP 0x00000800 - -#define IW_HMC_OBJ_TYPE_NUM ARRAY_SIZE(iw_hmc_obj_types) -#define IW_CFG_FPM_QP_COUNT 32768 -#define I40IW_MAX_PAGES_PER_FMR 512 -#define I40IW_MIN_PAGES_PER_FMR 1 -#define I40IW_CQP_COMPL_RQ_WQE_FLUSHED 2 -#define I40IW_CQP_COMPL_SQ_WQE_FLUSHED 3 -#define I40IW_CQP_COMPL_RQ_SQ_WQE_FLUSHED 4 - -struct i40iw_cqp_compl_info { - u32 op_ret_val; - u16 maj_err_code; - u16 min_err_code; - bool error; - u8 op_code; -}; - -#define i40iw_pr_err(fmt, args ...) pr_err("%s: "fmt, __func__, ## args) - -#define i40iw_pr_info(fmt, args ...) pr_info("%s: " fmt, __func__, ## args) - -#define i40iw_pr_warn(fmt, args ...) pr_warn("%s: " fmt, __func__, ## args) - -struct i40iw_cqp_request { - struct cqp_commands_info info; - wait_queue_head_t waitq; - struct list_head list; - atomic_t refcount; - void (*callback_fcn)(struct i40iw_cqp_request*, u32); - void *param; - struct i40iw_cqp_compl_info compl_info; - bool waiting; - bool request_done; - bool dynamic; -}; - -struct i40iw_cqp { - struct i40iw_sc_cqp sc_cqp; - spinlock_t req_lock; /*cqp request list */ - wait_queue_head_t waitq; - struct i40iw_dma_mem sq; - struct i40iw_dma_mem host_ctx; - u64 *scratch_array; - struct i40iw_cqp_request *cqp_requests; - struct list_head cqp_avail_reqs; - struct list_head cqp_pending_reqs; -}; - -struct i40iw_device; - -struct i40iw_ccq { - struct i40iw_sc_cq sc_cq; - spinlock_t lock; /* ccq control */ - wait_queue_head_t waitq; - struct i40iw_dma_mem mem_cq; - struct i40iw_dma_mem shadow_area; -}; - -struct i40iw_ceq { - struct i40iw_sc_ceq sc_ceq; - struct i40iw_dma_mem mem; - u32 irq; - u32 msix_idx; - struct i40iw_device *iwdev; - struct tasklet_struct dpc_tasklet; -}; - -struct i40iw_aeq { - struct i40iw_sc_aeq sc_aeq; - struct i40iw_dma_mem mem; -}; - -struct i40iw_arp_entry { - u32 ip_addr[4]; - u8 mac_addr[ETH_ALEN]; -}; - -enum init_completion_state { - INVALID_STATE = 0, - INITIAL_STATE, - CQP_CREATED, - HMC_OBJS_CREATED, - PBLE_CHUNK_MEM, - CCQ_CREATED, - AEQ_CREATED, - CEQ_CREATED, - ILQ_CREATED, - IEQ_CREATED, - IP_ADDR_REGISTERED, - RDMA_DEV_REGISTERED -}; - -struct i40iw_msix_vector { - u32 idx; - u32 irq; - u32 cpu_affinity; - u32 ceq_id; - cpumask_t mask; -}; - -struct l2params_work { - struct work_struct work; - struct i40iw_device *iwdev; - struct i40iw_l2params l2params; -}; - -#define I40IW_MSIX_TABLE_SIZE 65 - -struct virtchnl_work { - struct work_struct work; - union { - struct i40iw_cqp_request *cqp_request; - struct i40iw_virtchnl_work_info work_info; - }; -}; - -struct i40e_qvlist_info; - -struct i40iw_device { - struct i40iw_ib_device *iwibdev; - struct net_device *netdev; - wait_queue_head_t vchnl_waitq; - struct i40iw_sc_dev sc_dev; - struct i40iw_sc_vsi vsi; - struct i40iw_handler *hdl; - struct i40e_info *ldev; - struct i40e_client *client; - struct i40iw_hw hw; - struct i40iw_cm_core cm_core; - u8 *mem_resources; - unsigned long *allocated_qps; - unsigned long *allocated_cqs; - unsigned long *allocated_mrs; - unsigned long *allocated_pds; - unsigned long *allocated_arps; - struct i40iw_qp **qp_table; - bool msix_shared; - u32 msix_count; - struct i40iw_msix_vector *iw_msixtbl; - struct i40e_qvlist_info *iw_qvlist; - - struct i40iw_hmc_pble_rsrc *pble_rsrc; - struct i40iw_arp_entry *arp_table; - struct i40iw_cqp cqp; - struct i40iw_ccq ccq; - u32 ceqs_count; - struct i40iw_ceq *ceqlist; - struct i40iw_aeq aeq; - u32 arp_table_size; - u32 next_arp_index; - spinlock_t resource_lock; /* hw resource access */ - spinlock_t qptable_lock; - u32 vendor_id; - u32 vendor_part_id; - u32 of_device_registered; - - u32 device_cap_flags; - unsigned long db_start; - u8 resource_profile; - u8 max_rdma_vfs; - u8 max_enabled_vfs; - u8 max_sge; - u8 iw_status; - u8 send_term_ok; - - /* x710 specific */ - struct mutex pbl_mutex; - struct tasklet_struct dpc_tasklet; - struct workqueue_struct *virtchnl_wq; - struct virtchnl_work virtchnl_w[I40IW_MAX_PE_ENABLED_VF_COUNT]; - struct i40iw_dma_mem obj_mem; - struct i40iw_dma_mem obj_next; - u8 *hmc_info_mem; - u32 sd_type; - struct workqueue_struct *param_wq; - atomic_t params_busy; - enum init_completion_state init_state; - u16 mac_ip_table_idx; - atomic_t vchnl_msgs; - u32 max_mr; - u32 max_qp; - u32 max_cq; - u32 max_pd; - u32 next_qp; - u32 next_cq; - u32 next_pd; - u32 max_mr_size; - u32 max_qp_wr; - u32 max_cqe; - u32 mr_stagmask; - u32 mpa_version; - bool dcb; - bool closing; - bool reset; - u32 used_pds; - u32 used_cqs; - u32 used_mrs; - u32 used_qps; - wait_queue_head_t close_wq; - atomic64_t use_count; -}; - -struct i40iw_ib_device { - struct ib_device ibdev; - struct i40iw_device *iwdev; -}; - -struct i40iw_handler { - struct list_head list; - struct i40e_client *client; - struct i40iw_device device; - struct i40e_info ldev; -}; - -/** - * i40iw_fw_major_ver - get firmware major version - * @dev: iwarp device - **/ -static inline u64 i40iw_fw_major_ver(struct i40iw_sc_dev *dev) -{ - return RS_64(dev->feature_info[I40IW_FEATURE_FW_INFO], - I40IW_FW_VER_MAJOR); -} - -/** - * i40iw_fw_minor_ver - get firmware minor version - * @dev: iwarp device - **/ -static inline u64 i40iw_fw_minor_ver(struct i40iw_sc_dev *dev) -{ - return RS_64(dev->feature_info[I40IW_FEATURE_FW_INFO], - I40IW_FW_VER_MINOR); -} - -/** - * to_iwdev - get device - * @ibdev: ib device - **/ -static inline struct i40iw_device *to_iwdev(struct ib_device *ibdev) -{ - return container_of(ibdev, struct i40iw_ib_device, ibdev)->iwdev; -} - -/** - * to_ucontext - get user context - * @ibucontext: ib user context - **/ -static inline struct i40iw_ucontext *to_ucontext(struct ib_ucontext *ibucontext) -{ - return container_of(ibucontext, struct i40iw_ucontext, ibucontext); -} - -/** - * to_iwpd - get protection domain - * @ibpd: ib pd - **/ -static inline struct i40iw_pd *to_iwpd(struct ib_pd *ibpd) -{ - return container_of(ibpd, struct i40iw_pd, ibpd); -} - -/** - * to_iwmr - get device memory region - * @ibdev: ib memory region - **/ -static inline struct i40iw_mr *to_iwmr(struct ib_mr *ibmr) -{ - return container_of(ibmr, struct i40iw_mr, ibmr); -} - -/** - * to_iwmw - get device memory window - * @ibmw: ib memory window - **/ -static inline struct i40iw_mr *to_iwmw(struct ib_mw *ibmw) -{ - return container_of(ibmw, struct i40iw_mr, ibmw); -} - -/** - * to_iwcq - get completion queue - * @ibcq: ib cqdevice - **/ -static inline struct i40iw_cq *to_iwcq(struct ib_cq *ibcq) -{ - return container_of(ibcq, struct i40iw_cq, ibcq); -} - -/** - * to_iwqp - get device qp - * @ibqp: ib qp - **/ -static inline struct i40iw_qp *to_iwqp(struct ib_qp *ibqp) -{ - return container_of(ibqp, struct i40iw_qp, ibqp); -} - -/* i40iw.c */ -void i40iw_qp_add_ref(struct ib_qp *ibqp); -void i40iw_qp_rem_ref(struct ib_qp *ibqp); -struct ib_qp *i40iw_get_qp(struct ib_device *, int); - -void i40iw_flush_wqes(struct i40iw_device *iwdev, - struct i40iw_qp *qp); - -void i40iw_manage_arp_cache(struct i40iw_device *iwdev, - unsigned char *mac_addr, - u32 *ip_addr, - bool ipv4, - u32 action); - -int i40iw_manage_apbvt(struct i40iw_device *iwdev, - u16 accel_local_port, - bool add_port); - -struct i40iw_cqp_request *i40iw_get_cqp_request(struct i40iw_cqp *cqp, bool wait); -void i40iw_free_cqp_request(struct i40iw_cqp *cqp, struct i40iw_cqp_request *cqp_request); -void i40iw_put_cqp_request(struct i40iw_cqp *cqp, struct i40iw_cqp_request *cqp_request); - -/** - * i40iw_alloc_resource - allocate a resource - * @iwdev: device pointer - * @resource_array: resource bit array: - * @max_resources: maximum resource number - * @req_resources_num: Allocated resource number - * @next: next free id - **/ -static inline int i40iw_alloc_resource(struct i40iw_device *iwdev, - unsigned long *resource_array, - u32 max_resources, - u32 *req_resource_num, - u32 *next) -{ - u32 resource_num; - unsigned long flags; - - spin_lock_irqsave(&iwdev->resource_lock, flags); - resource_num = find_next_zero_bit(resource_array, max_resources, *next); - if (resource_num >= max_resources) { - resource_num = find_first_zero_bit(resource_array, max_resources); - if (resource_num >= max_resources) { - spin_unlock_irqrestore(&iwdev->resource_lock, flags); - return -EOVERFLOW; - } - } - set_bit(resource_num, resource_array); - *next = resource_num + 1; - if (*next == max_resources) - *next = 0; - *req_resource_num = resource_num; - spin_unlock_irqrestore(&iwdev->resource_lock, flags); - - return 0; -} - -/** - * i40iw_is_resource_allocated - detrmine if resource is - * allocated - * @iwdev: device pointer - * @resource_array: resource array for the resource_num - * @resource_num: resource number to check - **/ -static inline bool i40iw_is_resource_allocated(struct i40iw_device *iwdev, - unsigned long *resource_array, - u32 resource_num) -{ - bool bit_is_set; - unsigned long flags; - - spin_lock_irqsave(&iwdev->resource_lock, flags); - - bit_is_set = test_bit(resource_num, resource_array); - spin_unlock_irqrestore(&iwdev->resource_lock, flags); - - return bit_is_set; -} - -/** - * i40iw_free_resource - free a resource - * @iwdev: device pointer - * @resource_array: resource array for the resource_num - * @resource_num: resource number to free - **/ -static inline void i40iw_free_resource(struct i40iw_device *iwdev, - unsigned long *resource_array, - u32 resource_num) -{ - unsigned long flags; - - spin_lock_irqsave(&iwdev->resource_lock, flags); - clear_bit(resource_num, resource_array); - spin_unlock_irqrestore(&iwdev->resource_lock, flags); -} - -struct i40iw_handler *i40iw_find_netdev(struct net_device *netdev); - -/** - * iw_init_resources - - */ -u32 i40iw_initialize_hw_resources(struct i40iw_device *iwdev); - -int i40iw_register_rdma_device(struct i40iw_device *iwdev); -void i40iw_port_ibevent(struct i40iw_device *iwdev); -void i40iw_cm_disconn(struct i40iw_qp *iwqp); -void i40iw_cm_disconn_worker(void *); -int mini_cm_recv_pkt(struct i40iw_cm_core *, struct i40iw_device *, - struct sk_buff *); - -enum i40iw_status_code i40iw_handle_cqp_op(struct i40iw_device *iwdev, - struct i40iw_cqp_request *cqp_request); -enum i40iw_status_code i40iw_add_mac_addr(struct i40iw_device *iwdev, - u8 *mac_addr, u8 *mac_index); -int i40iw_modify_qp(struct ib_qp *, struct ib_qp_attr *, int, struct ib_udata *); -void i40iw_cq_wq_destroy(struct i40iw_device *iwdev, struct i40iw_sc_cq *cq); - -void i40iw_cleanup_pending_cqp_op(struct i40iw_device *iwdev); -void i40iw_rem_pdusecount(struct i40iw_pd *iwpd, struct i40iw_device *iwdev); -void i40iw_add_pdusecount(struct i40iw_pd *iwpd); -void i40iw_rem_devusecount(struct i40iw_device *iwdev); -void i40iw_add_devusecount(struct i40iw_device *iwdev); -void i40iw_hw_modify_qp(struct i40iw_device *iwdev, struct i40iw_qp *iwqp, - struct i40iw_modify_qp_info *info, bool wait); - -void i40iw_qp_suspend_resume(struct i40iw_sc_dev *dev, - struct i40iw_sc_qp *qp, - bool suspend); -enum i40iw_status_code i40iw_manage_qhash(struct i40iw_device *iwdev, - struct i40iw_cm_info *cminfo, - enum i40iw_quad_entry_type etype, - enum i40iw_quad_hash_manage_type mtype, - void *cmnode, - bool wait); -void i40iw_receive_ilq(struct i40iw_sc_vsi *vsi, struct i40iw_puda_buf *rbuf); -void i40iw_free_sqbuf(struct i40iw_sc_vsi *vsi, void *bufp); -void i40iw_free_qp_resources(struct i40iw_qp *iwqp); - -enum i40iw_status_code i40iw_obj_aligned_mem(struct i40iw_device *iwdev, - struct i40iw_dma_mem *memptr, - u32 size, u32 mask); - -void i40iw_request_reset(struct i40iw_device *iwdev); -void i40iw_destroy_rdma_device(struct i40iw_ib_device *iwibdev); -int i40iw_setup_cm_core(struct i40iw_device *iwdev); -void i40iw_cleanup_cm_core(struct i40iw_cm_core *cm_core); -void i40iw_process_ceq(struct i40iw_device *, struct i40iw_ceq *iwceq); -void i40iw_process_aeq(struct i40iw_device *); -void i40iw_next_iw_state(struct i40iw_qp *iwqp, - u8 state, u8 del_hash, - u8 term, u8 term_len); -int i40iw_send_syn(struct i40iw_cm_node *cm_node, u32 sendack); -int i40iw_send_reset(struct i40iw_cm_node *cm_node); -struct i40iw_cm_node *i40iw_find_node(struct i40iw_cm_core *cm_core, - u16 rem_port, - u32 *rem_addr, - u16 loc_port, - u32 *loc_addr, - bool add_refcnt, - bool accelerated_list); - -enum i40iw_status_code i40iw_hw_flush_wqes(struct i40iw_device *iwdev, - struct i40iw_sc_qp *qp, - struct i40iw_qp_flush_info *info, - bool wait); - -void i40iw_gen_ae(struct i40iw_device *iwdev, - struct i40iw_sc_qp *qp, - struct i40iw_gen_ae_info *info, - bool wait); - -void i40iw_copy_ip_ntohl(u32 *dst, __be32 *src); -struct ib_mr *i40iw_reg_phys_mr(struct ib_pd *ib_pd, - u64 addr, - u64 size, - int acc, - u64 *iova_start); - -int i40iw_inetaddr_event(struct notifier_block *notifier, - unsigned long event, - void *ptr); -int i40iw_inet6addr_event(struct notifier_block *notifier, - unsigned long event, - void *ptr); -int i40iw_net_event(struct notifier_block *notifier, - unsigned long event, - void *ptr); -int i40iw_netdevice_event(struct notifier_block *notifier, - unsigned long event, - void *ptr); - -#endif diff --git a/drivers/infiniband/hw/i40iw/i40iw_cm.c b/drivers/infiniband/hw/i40iw/i40iw_cm.c deleted file mode 100644 index 2450b7dd51f6..000000000000 --- a/drivers/infiniband/hw/i40iw/i40iw_cm.c +++ /dev/null @@ -1,4419 +0,0 @@ -/******************************************************************************* -* -* Copyright (c) 2015-2016 Intel Corporation. All rights reserved. -* -* This software is available to you under a choice of one of two -* licenses. You may choose to be licensed under the terms of the GNU -* General Public License (GPL) Version 2, available from the file -* COPYING in the main directory of this source tree, or the -* OpenFabrics.org BSD license below: -* -* Redistribution and use in source and binary forms, with or -* without modification, are permitted provided that the following -* conditions are met: -* -* - Redistributions of source code must retain the above -* copyright notice, this list of conditions and the following -* disclaimer. -* -* - Redistributions in binary form must reproduce the above -* copyright notice, this list of conditions and the following -* disclaimer in the documentation and/or other materials -* provided with the distribution. -* -* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS -* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN -* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -* SOFTWARE. -* -*******************************************************************************/ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "i40iw.h" - -static void i40iw_rem_ref_cm_node(struct i40iw_cm_node *); -static void i40iw_cm_post_event(struct i40iw_cm_event *event); -static void i40iw_disconnect_worker(struct work_struct *work); - -/** - * i40iw_free_sqbuf - put back puda buffer if refcount = 0 - * @vsi: pointer to vsi structure - * @bufp: puda buffer to free - */ -void i40iw_free_sqbuf(struct i40iw_sc_vsi *vsi, void *bufp) -{ - struct i40iw_puda_buf *buf = (struct i40iw_puda_buf *)bufp; - struct i40iw_puda_rsrc *ilq = vsi->ilq; - - if (!atomic_dec_return(&buf->refcount)) - i40iw_puda_ret_bufpool(ilq, buf); -} - -/** - * i40iw_derive_hw_ird_setting - Calculate IRD - * - * @cm_ird: IRD of connection's node - * - * The ird from the connection is rounded to a supported HW - * setting (2,8,32,64) and then encoded for ird_size field of - * qp_ctx - */ -static u8 i40iw_derive_hw_ird_setting(u16 cm_ird) -{ - u8 encoded_ird_size; - - /* ird_size field is encoded in qp_ctx */ - switch (cm_ird ? roundup_pow_of_two(cm_ird) : 0) { - case I40IW_HW_IRD_SETTING_64: - encoded_ird_size = 3; - break; - case I40IW_HW_IRD_SETTING_32: - case I40IW_HW_IRD_SETTING_16: - encoded_ird_size = 2; - break; - case I40IW_HW_IRD_SETTING_8: - case I40IW_HW_IRD_SETTING_4: - encoded_ird_size = 1; - break; - case I40IW_HW_IRD_SETTING_2: - default: - encoded_ird_size = 0; - break; - } - return encoded_ird_size; -} - -/** - * i40iw_record_ird_ord - Record IRD/ORD passed in - * @cm_node: connection's node - * @conn_ird: connection IRD - * @conn_ord: connection ORD - */ -static void i40iw_record_ird_ord(struct i40iw_cm_node *cm_node, u32 conn_ird, - u32 conn_ord) -{ - if (conn_ird > I40IW_MAX_IRD_SIZE) - conn_ird = I40IW_MAX_IRD_SIZE; - - if (conn_ord > I40IW_MAX_ORD_SIZE) - conn_ord = I40IW_MAX_ORD_SIZE; - else if (!conn_ord && cm_node->send_rdma0_op == SEND_RDMA_READ_ZERO) - conn_ord = 1; - - cm_node->ird_size = conn_ird; - cm_node->ord_size = conn_ord; -} - -/** - * i40iw_copy_ip_ntohl - change network to host ip - * @dst: host ip - * @src: big endian - */ -void i40iw_copy_ip_ntohl(u32 *dst, __be32 *src) -{ - *dst++ = ntohl(*src++); - *dst++ = ntohl(*src++); - *dst++ = ntohl(*src++); - *dst = ntohl(*src); -} - -/** - * i40iw_copy_ip_htonl - change host addr to network ip - * @dst: host ip - * @src: little endian - */ -static inline void i40iw_copy_ip_htonl(__be32 *dst, u32 *src) -{ - *dst++ = htonl(*src++); - *dst++ = htonl(*src++); - *dst++ = htonl(*src++); - *dst = htonl(*src); -} - -/** - * i40iw_fill_sockaddr4 - get addr info for passive connection - * @cm_node: connection's node - * @event: upper layer's cm event - */ -static inline void i40iw_fill_sockaddr4(struct i40iw_cm_node *cm_node, - struct iw_cm_event *event) -{ - struct sockaddr_in *laddr = (struct sockaddr_in *)&event->local_addr; - struct sockaddr_in *raddr = (struct sockaddr_in *)&event->remote_addr; - - laddr->sin_family = AF_INET; - raddr->sin_family = AF_INET; - - laddr->sin_port = htons(cm_node->loc_port); - raddr->sin_port = htons(cm_node->rem_port); - - laddr->sin_addr.s_addr = htonl(cm_node->loc_addr[0]); - raddr->sin_addr.s_addr = htonl(cm_node->rem_addr[0]); -} - -/** - * i40iw_fill_sockaddr6 - get ipv6 addr info for passive side - * @cm_node: connection's node - * @event: upper layer's cm event - */ -static inline void i40iw_fill_sockaddr6(struct i40iw_cm_node *cm_node, - struct iw_cm_event *event) -{ - struct sockaddr_in6 *laddr6 = (struct sockaddr_in6 *)&event->local_addr; - struct sockaddr_in6 *raddr6 = (struct sockaddr_in6 *)&event->remote_addr; - - laddr6->sin6_family = AF_INET6; - raddr6->sin6_family = AF_INET6; - - laddr6->sin6_port = htons(cm_node->loc_port); - raddr6->sin6_port = htons(cm_node->rem_port); - - i40iw_copy_ip_htonl(laddr6->sin6_addr.in6_u.u6_addr32, - cm_node->loc_addr); - i40iw_copy_ip_htonl(raddr6->sin6_addr.in6_u.u6_addr32, - cm_node->rem_addr); -} - -/** - * i40iw_get_addr_info - * @cm_node: contains ip/tcp info - * @cm_info: to get a copy of the cm_node ip/tcp info -*/ -static void i40iw_get_addr_info(struct i40iw_cm_node *cm_node, - struct i40iw_cm_info *cm_info) -{ - cm_info->ipv4 = cm_node->ipv4; - cm_info->vlan_id = cm_node->vlan_id; - memcpy(cm_info->loc_addr, cm_node->loc_addr, sizeof(cm_info->loc_addr)); - memcpy(cm_info->rem_addr, cm_node->rem_addr, sizeof(cm_info->rem_addr)); - cm_info->loc_port = cm_node->loc_port; - cm_info->rem_port = cm_node->rem_port; - cm_info->user_pri = cm_node->user_pri; -} - -/** - * i40iw_get_cmevent_info - for cm event upcall - * @cm_node: connection's node - * @cm_id: upper layers cm struct for the event - * @event: upper layer's cm event - */ -static inline void i40iw_get_cmevent_info(struct i40iw_cm_node *cm_node, - struct iw_cm_id *cm_id, - struct iw_cm_event *event) -{ - memcpy(&event->local_addr, &cm_id->m_local_addr, - sizeof(event->local_addr)); - memcpy(&event->remote_addr, &cm_id->m_remote_addr, - sizeof(event->remote_addr)); - if (cm_node) { - event->private_data = (void *)cm_node->pdata_buf; - event->private_data_len = (u8)cm_node->pdata.size; - event->ird = cm_node->ird_size; - event->ord = cm_node->ord_size; - } -} - -/** - * i40iw_send_cm_event - upcall cm's event handler - * @cm_node: connection's node - * @cm_id: upper layer's cm info struct - * @type: Event type to indicate - * @status: status for the event type - */ -static int i40iw_send_cm_event(struct i40iw_cm_node *cm_node, - struct iw_cm_id *cm_id, - enum iw_cm_event_type type, - int status) -{ - struct iw_cm_event event; - - memset(&event, 0, sizeof(event)); - event.event = type; - event.status = status; - switch (type) { - case IW_CM_EVENT_CONNECT_REQUEST: - if (cm_node->ipv4) - i40iw_fill_sockaddr4(cm_node, &event); - else - i40iw_fill_sockaddr6(cm_node, &event); - event.provider_data = (void *)cm_node; - event.private_data = (void *)cm_node->pdata_buf; - event.private_data_len = (u8)cm_node->pdata.size; - event.ird = cm_node->ird_size; - break; - case IW_CM_EVENT_CONNECT_REPLY: - i40iw_get_cmevent_info(cm_node, cm_id, &event); - break; - case IW_CM_EVENT_ESTABLISHED: - event.ird = cm_node->ird_size; - event.ord = cm_node->ord_size; - break; - case IW_CM_EVENT_DISCONNECT: - break; - case IW_CM_EVENT_CLOSE: - break; - default: - i40iw_pr_err("event type received type = %d\n", type); - return -1; - } - return cm_id->event_handler(cm_id, &event); -} - -/** - * i40iw_create_event - create cm event - * @cm_node: connection's node - * @type: Event type to generate - */ -static struct i40iw_cm_event *i40iw_create_event(struct i40iw_cm_node *cm_node, - enum i40iw_cm_event_type type) -{ - struct i40iw_cm_event *event; - - if (!cm_node->cm_id) - return NULL; - - event = kzalloc(sizeof(*event), GFP_ATOMIC); - - if (!event) - return NULL; - - event->type = type; - event->cm_node = cm_node; - memcpy(event->cm_info.rem_addr, cm_node->rem_addr, sizeof(event->cm_info.rem_addr)); - memcpy(event->cm_info.loc_addr, cm_node->loc_addr, sizeof(event->cm_info.loc_addr)); - event->cm_info.rem_port = cm_node->rem_port; - event->cm_info.loc_port = cm_node->loc_port; - event->cm_info.cm_id = cm_node->cm_id; - - i40iw_debug(cm_node->dev, - I40IW_DEBUG_CM, - "node=%p event=%p type=%u dst=%pI4 src=%pI4\n", - cm_node, - event, - type, - event->cm_info.loc_addr, - event->cm_info.rem_addr); - - i40iw_cm_post_event(event); - return event; -} - -/** - * i40iw_free_retrans_entry - free send entry - * @cm_node: connection's node - */ -static void i40iw_free_retrans_entry(struct i40iw_cm_node *cm_node) -{ - struct i40iw_device *iwdev = cm_node->iwdev; - struct i40iw_timer_entry *send_entry; - - send_entry = cm_node->send_entry; - if (send_entry) { - cm_node->send_entry = NULL; - i40iw_free_sqbuf(&iwdev->vsi, (void *)send_entry->sqbuf); - kfree(send_entry); - atomic_dec(&cm_node->ref_count); - } -} - -/** - * i40iw_cleanup_retrans_entry - free send entry with lock - * @cm_node: connection's node - */ -static void i40iw_cleanup_retrans_entry(struct i40iw_cm_node *cm_node) -{ - unsigned long flags; - - spin_lock_irqsave(&cm_node->retrans_list_lock, flags); - i40iw_free_retrans_entry(cm_node); - spin_unlock_irqrestore(&cm_node->retrans_list_lock, flags); -} - -/** - * i40iw_form_cm_frame - get a free packet and build frame - * @cm_node: connection's node ionfo to use in frame - * @options: pointer to options info - * @hdr: pointer mpa header - * @pdata: pointer to private data - * @flags: indicates FIN or ACK - */ -static struct i40iw_puda_buf *i40iw_form_cm_frame(struct i40iw_cm_node *cm_node, - struct i40iw_kmem_info *options, - struct i40iw_kmem_info *hdr, - struct i40iw_kmem_info *pdata, - u8 flags) -{ - struct i40iw_puda_buf *sqbuf; - struct i40iw_sc_vsi *vsi = &cm_node->iwdev->vsi; - u8 *buf; - - struct tcphdr *tcph; - struct iphdr *iph; - struct ipv6hdr *ip6h; - struct ethhdr *ethh; - u16 packetsize; - u16 eth_hlen = ETH_HLEN; - u32 opts_len = 0; - u32 pd_len = 0; - u32 hdr_len = 0; - u16 vtag; - - sqbuf = i40iw_puda_get_bufpool(vsi->ilq); - if (!sqbuf) - return NULL; - buf = sqbuf->mem.va; - - if (options) - opts_len = (u32)options->size; - - if (hdr) - hdr_len = hdr->size; - - if (pdata) - pd_len = pdata->size; - - if (cm_node->vlan_id <= VLAN_VID_MASK) - eth_hlen += 4; - - if (cm_node->ipv4) - packetsize = sizeof(*iph) + sizeof(*tcph); - else - packetsize = sizeof(*ip6h) + sizeof(*tcph); - packetsize += opts_len + hdr_len + pd_len; - - memset(buf, 0x00, eth_hlen + packetsize); - - sqbuf->totallen = packetsize + eth_hlen; - sqbuf->maclen = eth_hlen; - sqbuf->tcphlen = sizeof(*tcph) + opts_len; - sqbuf->scratch = (void *)cm_node; - - ethh = (struct ethhdr *)buf; - buf += eth_hlen; - - if (cm_node->ipv4) { - sqbuf->ipv4 = true; - - iph = (struct iphdr *)buf; - buf += sizeof(*iph); - tcph = (struct tcphdr *)buf; - buf += sizeof(*tcph); - - ether_addr_copy(ethh->h_dest, cm_node->rem_mac); - ether_addr_copy(ethh->h_source, cm_node->loc_mac); - if (cm_node->vlan_id <= VLAN_VID_MASK) { - ((struct vlan_ethhdr *)ethh)->h_vlan_proto = htons(ETH_P_8021Q); - vtag = (cm_node->user_pri << VLAN_PRIO_SHIFT) | cm_node->vlan_id; - ((struct vlan_ethhdr *)ethh)->h_vlan_TCI = htons(vtag); - - ((struct vlan_ethhdr *)ethh)->h_vlan_encapsulated_proto = htons(ETH_P_IP); - } else { - ethh->h_proto = htons(ETH_P_IP); - } - - iph->version = IPVERSION; - iph->ihl = 5; /* 5 * 4Byte words, IP headr len */ - iph->tos = cm_node->tos; - iph->tot_len = htons(packetsize); - iph->id = htons(++cm_node->tcp_cntxt.loc_id); - - iph->frag_off = htons(0x4000); - iph->ttl = 0x40; - iph->protocol = IPPROTO_TCP; - iph->saddr = htonl(cm_node->loc_addr[0]); - iph->daddr = htonl(cm_node->rem_addr[0]); - } else { - sqbuf->ipv4 = false; - ip6h = (struct ipv6hdr *)buf; - buf += sizeof(*ip6h); - tcph = (struct tcphdr *)buf; - buf += sizeof(*tcph); - - ether_addr_copy(ethh->h_dest, cm_node->rem_mac); - ether_addr_copy(ethh->h_source, cm_node->loc_mac); - if (cm_node->vlan_id <= VLAN_VID_MASK) { - ((struct vlan_ethhdr *)ethh)->h_vlan_proto = htons(ETH_P_8021Q); - vtag = (cm_node->user_pri << VLAN_PRIO_SHIFT) | cm_node->vlan_id; - ((struct vlan_ethhdr *)ethh)->h_vlan_TCI = htons(vtag); - ((struct vlan_ethhdr *)ethh)->h_vlan_encapsulated_proto = htons(ETH_P_IPV6); - } else { - ethh->h_proto = htons(ETH_P_IPV6); - } - ip6h->version = 6; - ip6h->priority = cm_node->tos >> 4; - ip6h->flow_lbl[0] = cm_node->tos << 4; - ip6h->flow_lbl[1] = 0; - ip6h->flow_lbl[2] = 0; - ip6h->payload_len = htons(packetsize - sizeof(*ip6h)); - ip6h->nexthdr = 6; - ip6h->hop_limit = 128; - i40iw_copy_ip_htonl(ip6h->saddr.in6_u.u6_addr32, - cm_node->loc_addr); - i40iw_copy_ip_htonl(ip6h->daddr.in6_u.u6_addr32, - cm_node->rem_addr); - } - - tcph->source = htons(cm_node->loc_port); - tcph->dest = htons(cm_node->rem_port); - - tcph->seq = htonl(cm_node->tcp_cntxt.loc_seq_num); - - if (flags & SET_ACK) { - cm_node->tcp_cntxt.loc_ack_num = cm_node->tcp_cntxt.rcv_nxt; - tcph->ack_seq = htonl(cm_node->tcp_cntxt.loc_ack_num); - tcph->ack = 1; - } else { - tcph->ack_seq = 0; - } - - if (flags & SET_SYN) { - cm_node->tcp_cntxt.loc_seq_num++; - tcph->syn = 1; - } else { - cm_node->tcp_cntxt.loc_seq_num += hdr_len + pd_len; - } - - if (flags & SET_FIN) { - cm_node->tcp_cntxt.loc_seq_num++; - tcph->fin = 1; - } - - if (flags & SET_RST) - tcph->rst = 1; - - tcph->doff = (u16)((sizeof(*tcph) + opts_len + 3) >> 2); - sqbuf->tcphlen = tcph->doff << 2; - tcph->window = htons(cm_node->tcp_cntxt.rcv_wnd); - tcph->urg_ptr = 0; - - if (opts_len) { - memcpy(buf, options->addr, opts_len); - buf += opts_len; - } - - if (hdr_len) { - memcpy(buf, hdr->addr, hdr_len); - buf += hdr_len; - } - - if (pdata && pdata->addr) - memcpy(buf, pdata->addr, pdata->size); - - atomic_set(&sqbuf->refcount, 1); - - return sqbuf; -} - -/** - * i40iw_send_reset - Send RST packet - * @cm_node: connection's node - */ -int i40iw_send_reset(struct i40iw_cm_node *cm_node) -{ - struct i40iw_puda_buf *sqbuf; - int flags = SET_RST | SET_ACK; - - sqbuf = i40iw_form_cm_frame(cm_node, NULL, NULL, NULL, flags); - if (!sqbuf) { - i40iw_pr_err("no sqbuf\n"); - return -1; - } - - return i40iw_schedule_cm_timer(cm_node, sqbuf, I40IW_TIMER_TYPE_SEND, 0, 1); -} - -/** - * i40iw_active_open_err - send event for active side cm error - * @cm_node: connection's node - * @reset: Flag to send reset or not - */ -static void i40iw_active_open_err(struct i40iw_cm_node *cm_node, bool reset) -{ - i40iw_cleanup_retrans_entry(cm_node); - cm_node->cm_core->stats_connect_errs++; - if (reset) { - i40iw_debug(cm_node->dev, - I40IW_DEBUG_CM, - "%s cm_node=%p state=%d\n", - __func__, - cm_node, - cm_node->state); - atomic_inc(&cm_node->ref_count); - i40iw_send_reset(cm_node); - } - - cm_node->state = I40IW_CM_STATE_CLOSED; - i40iw_create_event(cm_node, I40IW_CM_EVENT_ABORTED); -} - -/** - * i40iw_passive_open_err - handle passive side cm error - * @cm_node: connection's node - * @reset: send reset or just free cm_node - */ -static void i40iw_passive_open_err(struct i40iw_cm_node *cm_node, bool reset) -{ - i40iw_cleanup_retrans_entry(cm_node); - cm_node->cm_core->stats_passive_errs++; - cm_node->state = I40IW_CM_STATE_CLOSED; - i40iw_debug(cm_node->dev, - I40IW_DEBUG_CM, - "%s cm_node=%p state =%d\n", - __func__, - cm_node, - cm_node->state); - if (reset) - i40iw_send_reset(cm_node); - else - i40iw_rem_ref_cm_node(cm_node); -} - -/** - * i40iw_event_connect_error - to create connect error event - * @event: cm information for connect event - */ -static void i40iw_event_connect_error(struct i40iw_cm_event *event) -{ - struct i40iw_qp *iwqp; - struct iw_cm_id *cm_id; - - cm_id = event->cm_node->cm_id; - if (!cm_id) - return; - - iwqp = cm_id->provider_data; - - if (!iwqp || !iwqp->iwdev) - return; - - iwqp->cm_id = NULL; - cm_id->provider_data = NULL; - i40iw_send_cm_event(event->cm_node, cm_id, - IW_CM_EVENT_CONNECT_REPLY, - -ECONNRESET); - cm_id->rem_ref(cm_id); - i40iw_rem_ref_cm_node(event->cm_node); -} - -/** - * i40iw_process_options - * @cm_node: connection's node - * @optionsloc: point to start of options - * @optionsize: size of all options - * @syn_packet: flag if syn packet - */ -static int i40iw_process_options(struct i40iw_cm_node *cm_node, - u8 *optionsloc, - u32 optionsize, - u32 syn_packet) -{ - u32 tmp; - u32 offset = 0; - union all_known_options *all_options; - char got_mss_option = 0; - - while (offset < optionsize) { - all_options = (union all_known_options *)(optionsloc + offset); - switch (all_options->as_base.optionnum) { - case OPTION_NUMBER_END: - offset = optionsize; - break; - case OPTION_NUMBER_NONE: - offset += 1; - continue; - case OPTION_NUMBER_MSS: - i40iw_debug(cm_node->dev, - I40IW_DEBUG_CM, - "%s: MSS Length: %d Offset: %d Size: %d\n", - __func__, - all_options->as_mss.length, - offset, - optionsize); - got_mss_option = 1; - if (all_options->as_mss.length != 4) - return -1; - tmp = ntohs(all_options->as_mss.mss); - if (tmp > 0 && tmp < cm_node->tcp_cntxt.mss) - cm_node->tcp_cntxt.mss = tmp; - break; - case OPTION_NUMBER_WINDOW_SCALE: - cm_node->tcp_cntxt.snd_wscale = - all_options->as_windowscale.shiftcount; - break; - default: - i40iw_debug(cm_node->dev, - I40IW_DEBUG_CM, - "TCP Option not understood: %x\n", - all_options->as_base.optionnum); - break; - } - offset += all_options->as_base.length; - } - if (!got_mss_option && syn_packet) - cm_node->tcp_cntxt.mss = I40IW_CM_DEFAULT_MSS; - return 0; -} - -/** - * i40iw_handle_tcp_options - - * @cm_node: connection's node - * @tcph: pointer tcp header - * @optionsize: size of options rcvd - * @passive: active or passive flag - */ -static int i40iw_handle_tcp_options(struct i40iw_cm_node *cm_node, - struct tcphdr *tcph, - int optionsize, - int passive) -{ - u8 *optionsloc = (u8 *)&tcph[1]; - - if (optionsize) { - if (i40iw_process_options(cm_node, - optionsloc, - optionsize, - (u32)tcph->syn)) { - i40iw_debug(cm_node->dev, - I40IW_DEBUG_CM, - "%s: Node %p, Sending RESET\n", - __func__, - cm_node); - if (passive) - i40iw_passive_open_err(cm_node, true); - else - i40iw_active_open_err(cm_node, true); - return -1; - } - } - - cm_node->tcp_cntxt.snd_wnd = ntohs(tcph->window) << - cm_node->tcp_cntxt.snd_wscale; - - if (cm_node->tcp_cntxt.snd_wnd > cm_node->tcp_cntxt.max_snd_wnd) - cm_node->tcp_cntxt.max_snd_wnd = cm_node->tcp_cntxt.snd_wnd; - return 0; -} - -/** - * i40iw_build_mpa_v1 - build a MPA V1 frame - * @cm_node: connection's node - * @start_addr: MPA frame start address - * @mpa_key: to do read0 or write0 - */ -static void i40iw_build_mpa_v1(struct i40iw_cm_node *cm_node, - void *start_addr, - u8 mpa_key) -{ - struct ietf_mpa_v1 *mpa_frame = (struct ietf_mpa_v1 *)start_addr; - - switch (mpa_key) { - case MPA_KEY_REQUEST: - memcpy(mpa_frame->key, IEFT_MPA_KEY_REQ, IETF_MPA_KEY_SIZE); - break; - case MPA_KEY_REPLY: - memcpy(mpa_frame->key, IEFT_MPA_KEY_REP, IETF_MPA_KEY_SIZE); - break; - default: - break; - } - mpa_frame->flags = IETF_MPA_FLAGS_CRC; - mpa_frame->rev = cm_node->mpa_frame_rev; - mpa_frame->priv_data_len = htons(cm_node->pdata.size); -} - -/** - * i40iw_build_mpa_v2 - build a MPA V2 frame - * @cm_node: connection's node - * @start_addr: buffer start address - * @mpa_key: to do read0 or write0 - */ -static void i40iw_build_mpa_v2(struct i40iw_cm_node *cm_node, - void *start_addr, - u8 mpa_key) -{ - struct ietf_mpa_v2 *mpa_frame = (struct ietf_mpa_v2 *)start_addr; - struct ietf_rtr_msg *rtr_msg = &mpa_frame->rtr_msg; - u16 ctrl_ird, ctrl_ord; - - /* initialize the upper 5 bytes of the frame */ - i40iw_build_mpa_v1(cm_node, start_addr, mpa_key); - mpa_frame->flags |= IETF_MPA_V2_FLAG; - mpa_frame->priv_data_len += htons(IETF_RTR_MSG_SIZE); - - /* initialize RTR msg */ - if (cm_node->mpav2_ird_ord == IETF_NO_IRD_ORD) { - ctrl_ird = IETF_NO_IRD_ORD; - ctrl_ord = IETF_NO_IRD_ORD; - } else { - ctrl_ird = (cm_node->ird_size > IETF_NO_IRD_ORD) ? - IETF_NO_IRD_ORD : cm_node->ird_size; - ctrl_ord = (cm_node->ord_size > IETF_NO_IRD_ORD) ? - IETF_NO_IRD_ORD : cm_node->ord_size; - } - - ctrl_ird |= IETF_PEER_TO_PEER; - - switch (mpa_key) { - case MPA_KEY_REQUEST: - ctrl_ord |= IETF_RDMA0_WRITE; - ctrl_ord |= IETF_RDMA0_READ; - break; - case MPA_KEY_REPLY: - switch (cm_node->send_rdma0_op) { - case SEND_RDMA_WRITE_ZERO: - ctrl_ord |= IETF_RDMA0_WRITE; - break; - case SEND_RDMA_READ_ZERO: - ctrl_ord |= IETF_RDMA0_READ; - break; - } - break; - default: - break; - } - rtr_msg->ctrl_ird = htons(ctrl_ird); - rtr_msg->ctrl_ord = htons(ctrl_ord); -} - -/** - * i40iw_cm_build_mpa_frame - build mpa frame for mpa version 1 or version 2 - * @cm_node: connection's node - * @mpa: mpa: data buffer - * @mpa_key: to do read0 or write0 - */ -static int i40iw_cm_build_mpa_frame(struct i40iw_cm_node *cm_node, - struct i40iw_kmem_info *mpa, - u8 mpa_key) -{ - int hdr_len = 0; - - switch (cm_node->mpa_frame_rev) { - case IETF_MPA_V1: - hdr_len = sizeof(struct ietf_mpa_v1); - i40iw_build_mpa_v1(cm_node, mpa->addr, mpa_key); - break; - case IETF_MPA_V2: - hdr_len = sizeof(struct ietf_mpa_v2); - i40iw_build_mpa_v2(cm_node, mpa->addr, mpa_key); - break; - default: - break; - } - - return hdr_len; -} - -/** - * i40iw_send_mpa_request - active node send mpa request to passive node - * @cm_node: connection's node - */ -static int i40iw_send_mpa_request(struct i40iw_cm_node *cm_node) -{ - struct i40iw_puda_buf *sqbuf; - - if (!cm_node) { - i40iw_pr_err("cm_node == NULL\n"); - return -1; - } - - cm_node->mpa_hdr.addr = &cm_node->mpa_frame; - cm_node->mpa_hdr.size = i40iw_cm_build_mpa_frame(cm_node, - &cm_node->mpa_hdr, - MPA_KEY_REQUEST); - if (!cm_node->mpa_hdr.size) { - i40iw_pr_err("mpa size = %d\n", cm_node->mpa_hdr.size); - return -1; - } - - sqbuf = i40iw_form_cm_frame(cm_node, - NULL, - &cm_node->mpa_hdr, - &cm_node->pdata, - SET_ACK); - if (!sqbuf) { - i40iw_pr_err("sq_buf == NULL\n"); - return -1; - } - return i40iw_schedule_cm_timer(cm_node, sqbuf, I40IW_TIMER_TYPE_SEND, 1, 0); -} - -/** - * i40iw_send_mpa_reject - - * @cm_node: connection's node - * @pdata: reject data for connection - * @plen: length of reject data - */ -static int i40iw_send_mpa_reject(struct i40iw_cm_node *cm_node, - const void *pdata, - u8 plen) -{ - struct i40iw_puda_buf *sqbuf; - struct i40iw_kmem_info priv_info; - - cm_node->mpa_hdr.addr = &cm_node->mpa_frame; - cm_node->mpa_hdr.size = i40iw_cm_build_mpa_frame(cm_node, - &cm_node->mpa_hdr, - MPA_KEY_REPLY); - - cm_node->mpa_frame.flags |= IETF_MPA_FLAGS_REJECT; - priv_info.addr = (void *)pdata; - priv_info.size = plen; - - sqbuf = i40iw_form_cm_frame(cm_node, - NULL, - &cm_node->mpa_hdr, - &priv_info, - SET_ACK | SET_FIN); - if (!sqbuf) { - i40iw_pr_err("no sqbuf\n"); - return -ENOMEM; - } - cm_node->state = I40IW_CM_STATE_FIN_WAIT1; - return i40iw_schedule_cm_timer(cm_node, sqbuf, I40IW_TIMER_TYPE_SEND, 1, 0); -} - -/** - * i40iw_parse_mpa - process an IETF MPA frame - * @cm_node: connection's node - * @buffer: Data pointer - * @type: to return accept or reject - * @len: Len of mpa buffer - */ -static int i40iw_parse_mpa(struct i40iw_cm_node *cm_node, u8 *buffer, u32 *type, u32 len) -{ - struct ietf_mpa_v1 *mpa_frame; - struct ietf_mpa_v2 *mpa_v2_frame; - struct ietf_rtr_msg *rtr_msg; - int mpa_hdr_len; - int priv_data_len; - - *type = I40IW_MPA_REQUEST_ACCEPT; - - if (len < sizeof(struct ietf_mpa_v1)) { - i40iw_pr_err("ietf buffer small (%x)\n", len); - return -1; - } - - mpa_frame = (struct ietf_mpa_v1 *)buffer; - mpa_hdr_len = sizeof(struct ietf_mpa_v1); - priv_data_len = ntohs(mpa_frame->priv_data_len); - - if (priv_data_len > IETF_MAX_PRIV_DATA_LEN) { - i40iw_pr_err("large pri_data %d\n", priv_data_len); - return -1; - } - if (mpa_frame->rev != IETF_MPA_V1 && mpa_frame->rev != IETF_MPA_V2) { - i40iw_pr_err("unsupported mpa rev = %d\n", mpa_frame->rev); - return -1; - } - if (mpa_frame->rev > cm_node->mpa_frame_rev) { - i40iw_pr_err("rev %d\n", mpa_frame->rev); - return -1; - } - cm_node->mpa_frame_rev = mpa_frame->rev; - - if (cm_node->state != I40IW_CM_STATE_MPAREQ_SENT) { - if (memcmp(mpa_frame->key, IEFT_MPA_KEY_REQ, IETF_MPA_KEY_SIZE)) { - i40iw_pr_err("Unexpected MPA Key received\n"); - return -1; - } - } else { - if (memcmp(mpa_frame->key, IEFT_MPA_KEY_REP, IETF_MPA_KEY_SIZE)) { - i40iw_pr_err("Unexpected MPA Key received\n"); - return -1; - } - } - - if (priv_data_len + mpa_hdr_len > len) { - i40iw_pr_err("ietf buffer len(%x + %x != %x)\n", - priv_data_len, mpa_hdr_len, len); - return -1; - } - if (len > MAX_CM_BUFFER) { - i40iw_pr_err("ietf buffer large len = %d\n", len); - return -1; - } - - switch (mpa_frame->rev) { - case IETF_MPA_V2:{ - u16 ird_size; - u16 ord_size; - u16 ctrl_ord; - u16 ctrl_ird; - - mpa_v2_frame = (struct ietf_mpa_v2 *)buffer; - mpa_hdr_len += IETF_RTR_MSG_SIZE; - rtr_msg = &mpa_v2_frame->rtr_msg; - - /* parse rtr message */ - ctrl_ord = ntohs(rtr_msg->ctrl_ord); - ctrl_ird = ntohs(rtr_msg->ctrl_ird); - ird_size = ctrl_ird & IETF_NO_IRD_ORD; - ord_size = ctrl_ord & IETF_NO_IRD_ORD; - - if (!(ctrl_ird & IETF_PEER_TO_PEER)) - return -1; - - if (ird_size == IETF_NO_IRD_ORD || ord_size == IETF_NO_IRD_ORD) { - cm_node->mpav2_ird_ord = IETF_NO_IRD_ORD; - goto negotiate_done; - } - - if (cm_node->state != I40IW_CM_STATE_MPAREQ_SENT) { - /* responder */ - if (!ord_size && (ctrl_ord & IETF_RDMA0_READ)) - cm_node->ird_size = 1; - if (cm_node->ord_size > ird_size) - cm_node->ord_size = ird_size; - } else { - /* initiator */ - if (!ird_size && (ctrl_ord & IETF_RDMA0_READ)) - return -1; - if (cm_node->ord_size > ird_size) - cm_node->ord_size = ird_size; - - if (cm_node->ird_size < ord_size) - /* no resources available */ - return -1; - } - -negotiate_done: - if (ctrl_ord & IETF_RDMA0_READ) - cm_node->send_rdma0_op = SEND_RDMA_READ_ZERO; - else if (ctrl_ord & IETF_RDMA0_WRITE) - cm_node->send_rdma0_op = SEND_RDMA_WRITE_ZERO; - else /* Not supported RDMA0 operation */ - return -1; - i40iw_debug(cm_node->dev, I40IW_DEBUG_CM, - "MPAV2: Negotiated ORD: %d, IRD: %d\n", - cm_node->ord_size, cm_node->ird_size); - break; - } - break; - case IETF_MPA_V1: - default: - break; - } - - memcpy(cm_node->pdata_buf, buffer + mpa_hdr_len, priv_data_len); - cm_node->pdata.size = priv_data_len; - - if (mpa_frame->flags & IETF_MPA_FLAGS_REJECT) - *type = I40IW_MPA_REQUEST_REJECT; - - if (mpa_frame->flags & IETF_MPA_FLAGS_MARKERS) - cm_node->snd_mark_en = true; - - return 0; -} - -/** - * i40iw_schedule_cm_timer - * @cm_node: connection's node - * @sqbuf: buffer to send - * @type: if it is send or close - * @send_retrans: if rexmits to be done - * @close_when_complete: is cm_node to be removed - * - * note - cm_node needs to be protected before calling this. Encase in: - * i40iw_rem_ref_cm_node(cm_core, cm_node); - * i40iw_schedule_cm_timer(...) - * atomic_inc(&cm_node->ref_count); - */ -int i40iw_schedule_cm_timer(struct i40iw_cm_node *cm_node, - struct i40iw_puda_buf *sqbuf, - enum i40iw_timer_type type, - int send_retrans, - int close_when_complete) -{ - struct i40iw_sc_vsi *vsi = &cm_node->iwdev->vsi; - struct i40iw_cm_core *cm_core = cm_node->cm_core; - struct i40iw_timer_entry *new_send; - int ret = 0; - u32 was_timer_set; - unsigned long flags; - - new_send = kzalloc(sizeof(*new_send), GFP_ATOMIC); - if (!new_send) { - if (type != I40IW_TIMER_TYPE_CLOSE) - i40iw_free_sqbuf(vsi, (void *)sqbuf); - return -ENOMEM; - } - new_send->retrycount = I40IW_DEFAULT_RETRYS; - new_send->retranscount = I40IW_DEFAULT_RETRANS; - new_send->sqbuf = sqbuf; - new_send->timetosend = jiffies; - new_send->type = type; - new_send->send_retrans = send_retrans; - new_send->close_when_complete = close_when_complete; - - if (type == I40IW_TIMER_TYPE_CLOSE) { - new_send->timetosend += (HZ / 10); - if (cm_node->close_entry) { - kfree(new_send); - i40iw_pr_err("already close entry\n"); - return -EINVAL; - } - cm_node->close_entry = new_send; - } - - if (type == I40IW_TIMER_TYPE_SEND) { - spin_lock_irqsave(&cm_node->retrans_list_lock, flags); - cm_node->send_entry = new_send; - atomic_inc(&cm_node->ref_count); - spin_unlock_irqrestore(&cm_node->retrans_list_lock, flags); - new_send->timetosend = jiffies + I40IW_RETRY_TIMEOUT; - - atomic_inc(&sqbuf->refcount); - i40iw_puda_send_buf(vsi->ilq, sqbuf); - if (!send_retrans) { - i40iw_cleanup_retrans_entry(cm_node); - if (close_when_complete) - i40iw_rem_ref_cm_node(cm_node); - return ret; - } - } - - spin_lock_irqsave(&cm_core->ht_lock, flags); - was_timer_set = timer_pending(&cm_core->tcp_timer); - - if (!was_timer_set) { - cm_core->tcp_timer.expires = new_send->timetosend; - add_timer(&cm_core->tcp_timer); - } - spin_unlock_irqrestore(&cm_core->ht_lock, flags); - - return ret; -} - -/** - * i40iw_retrans_expired - Could not rexmit the packet - * @cm_node: connection's node - */ -static void i40iw_retrans_expired(struct i40iw_cm_node *cm_node) -{ - struct iw_cm_id *cm_id = cm_node->cm_id; - enum i40iw_cm_node_state state = cm_node->state; - - cm_node->state = I40IW_CM_STATE_CLOSED; - switch (state) { - case I40IW_CM_STATE_SYN_RCVD: - case I40IW_CM_STATE_CLOSING: - i40iw_rem_ref_cm_node(cm_node); - break; - case I40IW_CM_STATE_FIN_WAIT1: - case I40IW_CM_STATE_LAST_ACK: - if (cm_node->cm_id) - cm_id->rem_ref(cm_id); - i40iw_send_reset(cm_node); - break; - default: - atomic_inc(&cm_node->ref_count); - i40iw_send_reset(cm_node); - i40iw_create_event(cm_node, I40IW_CM_EVENT_ABORTED); - break; - } -} - -/** - * i40iw_handle_close_entry - for handling retry/timeouts - * @cm_node: connection's node - * @rem_node: flag for remove cm_node - */ -static void i40iw_handle_close_entry(struct i40iw_cm_node *cm_node, u32 rem_node) -{ - struct i40iw_timer_entry *close_entry = cm_node->close_entry; - struct iw_cm_id *cm_id = cm_node->cm_id; - struct i40iw_qp *iwqp; - unsigned long flags; - - if (!close_entry) - return; - iwqp = (struct i40iw_qp *)close_entry->sqbuf; - if (iwqp) { - spin_lock_irqsave(&iwqp->lock, flags); - if (iwqp->cm_id) { - iwqp->hw_tcp_state = I40IW_TCP_STATE_CLOSED; - iwqp->hw_iwarp_state = I40IW_QP_STATE_ERROR; - iwqp->last_aeq = I40IW_AE_RESET_SENT; - iwqp->ibqp_state = IB_QPS_ERR; - spin_unlock_irqrestore(&iwqp->lock, flags); - i40iw_cm_disconn(iwqp); - } else { - spin_unlock_irqrestore(&iwqp->lock, flags); - } - } else if (rem_node) { - /* TIME_WAIT state */ - i40iw_rem_ref_cm_node(cm_node); - } - if (cm_id) - cm_id->rem_ref(cm_id); - kfree(close_entry); - cm_node->close_entry = NULL; -} - -/** - * i40iw_build_timer_list - Add cm_nodes to timer list - * @timer_list: ptr to timer list - * @hte: ptr to accelerated or non-accelerated list - */ -static void i40iw_build_timer_list(struct list_head *timer_list, - struct list_head *hte) -{ - struct i40iw_cm_node *cm_node; - struct list_head *list_core_temp, *list_node; - - list_for_each_safe(list_node, list_core_temp, hte) { - cm_node = container_of(list_node, struct i40iw_cm_node, list); - if (cm_node->close_entry || cm_node->send_entry) { - atomic_inc(&cm_node->ref_count); - list_add(&cm_node->timer_entry, timer_list); - } - } -} - -/** - * i40iw_cm_timer_tick - system's timer expired callback - * @t: Timer instance to fetch the cm_core pointer from - */ -static void i40iw_cm_timer_tick(struct timer_list *t) -{ - unsigned long nexttimeout = jiffies + I40IW_LONG_TIME; - struct i40iw_cm_node *cm_node; - struct i40iw_timer_entry *send_entry, *close_entry; - struct list_head *list_core_temp; - struct i40iw_sc_vsi *vsi; - struct list_head *list_node; - struct i40iw_cm_core *cm_core = from_timer(cm_core, t, tcp_timer); - u32 settimer = 0; - unsigned long timetosend; - unsigned long flags; - - struct list_head timer_list; - - INIT_LIST_HEAD(&timer_list); - - spin_lock_irqsave(&cm_core->ht_lock, flags); - i40iw_build_timer_list(&timer_list, &cm_core->non_accelerated_list); - i40iw_build_timer_list(&timer_list, &cm_core->accelerated_list); - spin_unlock_irqrestore(&cm_core->ht_lock, flags); - - list_for_each_safe(list_node, list_core_temp, &timer_list) { - cm_node = container_of(list_node, - struct i40iw_cm_node, - timer_entry); - close_entry = cm_node->close_entry; - - if (close_entry) { - if (time_after(close_entry->timetosend, jiffies)) { - if (nexttimeout > close_entry->timetosend || - !settimer) { - nexttimeout = close_entry->timetosend; - settimer = 1; - } - } else { - i40iw_handle_close_entry(cm_node, 1); - } - } - - spin_lock_irqsave(&cm_node->retrans_list_lock, flags); - - send_entry = cm_node->send_entry; - if (!send_entry) - goto done; - if (time_after(send_entry->timetosend, jiffies)) { - if (cm_node->state != I40IW_CM_STATE_OFFLOADED) { - if ((nexttimeout > send_entry->timetosend) || - !settimer) { - nexttimeout = send_entry->timetosend; - settimer = 1; - } - } else { - i40iw_free_retrans_entry(cm_node); - } - goto done; - } - - if ((cm_node->state == I40IW_CM_STATE_OFFLOADED) || - (cm_node->state == I40IW_CM_STATE_CLOSED)) { - i40iw_free_retrans_entry(cm_node); - goto done; - } - - if (!send_entry->retranscount || !send_entry->retrycount) { - i40iw_free_retrans_entry(cm_node); - - spin_unlock_irqrestore(&cm_node->retrans_list_lock, flags); - i40iw_retrans_expired(cm_node); - cm_node->state = I40IW_CM_STATE_CLOSED; - spin_lock_irqsave(&cm_node->retrans_list_lock, flags); - goto done; - } - spin_unlock_irqrestore(&cm_node->retrans_list_lock, flags); - - vsi = &cm_node->iwdev->vsi; - - if (!cm_node->ack_rcvd) { - atomic_inc(&send_entry->sqbuf->refcount); - i40iw_puda_send_buf(vsi->ilq, send_entry->sqbuf); - cm_node->cm_core->stats_pkt_retrans++; - } - spin_lock_irqsave(&cm_node->retrans_list_lock, flags); - if (send_entry->send_retrans) { - send_entry->retranscount--; - timetosend = (I40IW_RETRY_TIMEOUT << - (I40IW_DEFAULT_RETRANS - - send_entry->retranscount)); - - send_entry->timetosend = jiffies + - min(timetosend, I40IW_MAX_TIMEOUT); - if (nexttimeout > send_entry->timetosend || !settimer) { - nexttimeout = send_entry->timetosend; - settimer = 1; - } - } else { - int close_when_complete; - - close_when_complete = send_entry->close_when_complete; - i40iw_debug(cm_node->dev, - I40IW_DEBUG_CM, - "cm_node=%p state=%d\n", - cm_node, - cm_node->state); - i40iw_free_retrans_entry(cm_node); - if (close_when_complete) - i40iw_rem_ref_cm_node(cm_node); - } -done: - spin_unlock_irqrestore(&cm_node->retrans_list_lock, flags); - i40iw_rem_ref_cm_node(cm_node); - } - - if (settimer) { - spin_lock_irqsave(&cm_core->ht_lock, flags); - if (!timer_pending(&cm_core->tcp_timer)) { - cm_core->tcp_timer.expires = nexttimeout; - add_timer(&cm_core->tcp_timer); - } - spin_unlock_irqrestore(&cm_core->ht_lock, flags); - } -} - -/** - * i40iw_send_syn - send SYN packet - * @cm_node: connection's node - * @sendack: flag to set ACK bit or not - */ -int i40iw_send_syn(struct i40iw_cm_node *cm_node, u32 sendack) -{ - struct i40iw_puda_buf *sqbuf; - int flags = SET_SYN; - char optionsbuffer[sizeof(struct option_mss) + - sizeof(struct option_windowscale) + - sizeof(struct option_base) + TCP_OPTIONS_PADDING]; - struct i40iw_kmem_info opts; - - int optionssize = 0; - /* Sending MSS option */ - union all_known_options *options; - - opts.addr = optionsbuffer; - if (!cm_node) { - i40iw_pr_err("no cm_node\n"); - return -EINVAL; - } - - options = (union all_known_options *)&optionsbuffer[optionssize]; - options->as_mss.optionnum = OPTION_NUMBER_MSS; - options->as_mss.length = sizeof(struct option_mss); - options->as_mss.mss = htons(cm_node->tcp_cntxt.mss); - optionssize += sizeof(struct option_mss); - - options = (union all_known_options *)&optionsbuffer[optionssize]; - options->as_windowscale.optionnum = OPTION_NUMBER_WINDOW_SCALE; - options->as_windowscale.length = sizeof(struct option_windowscale); - options->as_windowscale.shiftcount = cm_node->tcp_cntxt.rcv_wscale; - optionssize += sizeof(struct option_windowscale); - options = (union all_known_options *)&optionsbuffer[optionssize]; - options->as_end = OPTION_NUMBER_END; - optionssize += 1; - - if (sendack) - flags |= SET_ACK; - - opts.size = optionssize; - - sqbuf = i40iw_form_cm_frame(cm_node, &opts, NULL, NULL, flags); - if (!sqbuf) { - i40iw_pr_err("no sqbuf\n"); - return -1; - } - return i40iw_schedule_cm_timer(cm_node, sqbuf, I40IW_TIMER_TYPE_SEND, 1, 0); -} - -/** - * i40iw_send_ack - Send ACK packet - * @cm_node: connection's node - */ -static void i40iw_send_ack(struct i40iw_cm_node *cm_node) -{ - struct i40iw_puda_buf *sqbuf; - struct i40iw_sc_vsi *vsi = &cm_node->iwdev->vsi; - - sqbuf = i40iw_form_cm_frame(cm_node, NULL, NULL, NULL, SET_ACK); - if (sqbuf) - i40iw_puda_send_buf(vsi->ilq, sqbuf); - else - i40iw_pr_err("no sqbuf\n"); -} - -/** - * i40iw_send_fin - Send FIN pkt - * @cm_node: connection's node - */ -static int i40iw_send_fin(struct i40iw_cm_node *cm_node) -{ - struct i40iw_puda_buf *sqbuf; - - sqbuf = i40iw_form_cm_frame(cm_node, NULL, NULL, NULL, SET_ACK | SET_FIN); - if (!sqbuf) { - i40iw_pr_err("no sqbuf\n"); - return -1; - } - return i40iw_schedule_cm_timer(cm_node, sqbuf, I40IW_TIMER_TYPE_SEND, 1, 0); -} - -/** - * i40iw_find_node - find a cm node that matches the reference cm node - * @cm_core: cm's core - * @rem_port: remote tcp port num - * @rem_addr: remote ip addr - * @loc_port: local tcp port num - * @loc_addr: loc ip addr - * @add_refcnt: flag to increment refcount of cm_node - * @accelerated_list: flag for accelerated vs non-accelerated list to search - */ -struct i40iw_cm_node *i40iw_find_node(struct i40iw_cm_core *cm_core, - u16 rem_port, - u32 *rem_addr, - u16 loc_port, - u32 *loc_addr, - bool add_refcnt, - bool accelerated_list) -{ - struct list_head *hte; - struct i40iw_cm_node *cm_node; - unsigned long flags; - - hte = accelerated_list ? - &cm_core->accelerated_list : &cm_core->non_accelerated_list; - - /* walk list and find cm_node associated with this session ID */ - spin_lock_irqsave(&cm_core->ht_lock, flags); - list_for_each_entry(cm_node, hte, list) { - if (!memcmp(cm_node->loc_addr, loc_addr, sizeof(cm_node->loc_addr)) && - (cm_node->loc_port == loc_port) && - !memcmp(cm_node->rem_addr, rem_addr, sizeof(cm_node->rem_addr)) && - (cm_node->rem_port == rem_port)) { - if (add_refcnt) - atomic_inc(&cm_node->ref_count); - spin_unlock_irqrestore(&cm_core->ht_lock, flags); - return cm_node; - } - } - spin_unlock_irqrestore(&cm_core->ht_lock, flags); - - /* no owner node */ - return NULL; -} - -/** - * i40iw_find_listener - find a cm node listening on this addr-port pair - * @cm_core: cm's core - * @dst_port: listener tcp port num - * @dst_addr: listener ip addr - * @vlan_id: vlan id for the given address - * @listener_state: state to match with listen node's - */ -static struct i40iw_cm_listener *i40iw_find_listener( - struct i40iw_cm_core *cm_core, - u32 *dst_addr, - u16 dst_port, - u16 vlan_id, - enum i40iw_cm_listener_state - listener_state) -{ - struct i40iw_cm_listener *listen_node; - static const u32 ip_zero[4] = { 0, 0, 0, 0 }; - u32 listen_addr[4]; - u16 listen_port; - unsigned long flags; - - /* walk list and find cm_node associated with this session ID */ - spin_lock_irqsave(&cm_core->listen_list_lock, flags); - list_for_each_entry(listen_node, &cm_core->listen_nodes, list) { - memcpy(listen_addr, listen_node->loc_addr, sizeof(listen_addr)); - listen_port = listen_node->loc_port; - /* compare node pair, return node handle if a match */ - if ((!memcmp(listen_addr, dst_addr, sizeof(listen_addr)) || - !memcmp(listen_addr, ip_zero, sizeof(listen_addr))) && - (listen_port == dst_port) && - (listener_state & listen_node->listener_state)) { - atomic_inc(&listen_node->ref_count); - spin_unlock_irqrestore(&cm_core->listen_list_lock, flags); - return listen_node; - } - } - spin_unlock_irqrestore(&cm_core->listen_list_lock, flags); - return NULL; -} - -/** - * i40iw_add_hte_node - add a cm node to the hash table - * @cm_core: cm's core - * @cm_node: connection's node - */ -static void i40iw_add_hte_node(struct i40iw_cm_core *cm_core, - struct i40iw_cm_node *cm_node) -{ - unsigned long flags; - - if (!cm_node || !cm_core) { - i40iw_pr_err("cm_node or cm_core == NULL\n"); - return; - } - - spin_lock_irqsave(&cm_core->ht_lock, flags); - list_add_tail(&cm_node->list, &cm_core->non_accelerated_list); - spin_unlock_irqrestore(&cm_core->ht_lock, flags); -} - -/** - * i40iw_find_port - find port that matches reference port - * @hte: ptr to accelerated or non-accelerated list - * @port: port number to locate - */ -static bool i40iw_find_port(struct list_head *hte, u16 port) -{ - struct i40iw_cm_node *cm_node; - - list_for_each_entry(cm_node, hte, list) { - if (cm_node->loc_port == port) - return true; - } - return false; -} - -/** - * i40iw_port_in_use - determine if port is in use - * @cm_core: cm's core - * @port: port number - */ -bool i40iw_port_in_use(struct i40iw_cm_core *cm_core, u16 port) -{ - struct i40iw_cm_listener *listen_node; - unsigned long flags; - - spin_lock_irqsave(&cm_core->ht_lock, flags); - if (i40iw_find_port(&cm_core->accelerated_list, port) || - i40iw_find_port(&cm_core->non_accelerated_list, port)) { - spin_unlock_irqrestore(&cm_core->ht_lock, flags); - return true; - } - spin_unlock_irqrestore(&cm_core->ht_lock, flags); - - spin_lock_irqsave(&cm_core->listen_list_lock, flags); - list_for_each_entry(listen_node, &cm_core->listen_nodes, list) { - if (listen_node->loc_port == port) { - spin_unlock_irqrestore(&cm_core->listen_list_lock, flags); - return true; - } - } - spin_unlock_irqrestore(&cm_core->listen_list_lock, flags); - - return false; -} - -/** - * i40iw_del_multiple_qhash - Remove qhash and child listens - * @iwdev: iWarp device - * @cm_info: CM info for parent listen node - * @cm_parent_listen_node: The parent listen node - */ -static enum i40iw_status_code i40iw_del_multiple_qhash( - struct i40iw_device *iwdev, - struct i40iw_cm_info *cm_info, - struct i40iw_cm_listener *cm_parent_listen_node) -{ - struct i40iw_cm_listener *child_listen_node; - enum i40iw_status_code ret = I40IW_ERR_CONFIG; - struct list_head *pos, *tpos; - unsigned long flags; - - spin_lock_irqsave(&iwdev->cm_core.listen_list_lock, flags); - list_for_each_safe(pos, tpos, &cm_parent_listen_node->child_listen_list) { - child_listen_node = list_entry(pos, struct i40iw_cm_listener, child_listen_list); - if (child_listen_node->ipv4) - i40iw_debug(&iwdev->sc_dev, - I40IW_DEBUG_CM, - "removing child listen for IP=%pI4, port=%d, vlan=%d\n", - child_listen_node->loc_addr, - child_listen_node->loc_port, - child_listen_node->vlan_id); - else - i40iw_debug(&iwdev->sc_dev, I40IW_DEBUG_CM, - "removing child listen for IP=%pI6, port=%d, vlan=%d\n", - child_listen_node->loc_addr, - child_listen_node->loc_port, - child_listen_node->vlan_id); - list_del(pos); - memcpy(cm_info->loc_addr, child_listen_node->loc_addr, - sizeof(cm_info->loc_addr)); - cm_info->vlan_id = child_listen_node->vlan_id; - if (child_listen_node->qhash_set) { - ret = i40iw_manage_qhash(iwdev, cm_info, - I40IW_QHASH_TYPE_TCP_SYN, - I40IW_QHASH_MANAGE_TYPE_DELETE, - NULL, false); - child_listen_node->qhash_set = false; - } else { - ret = I40IW_SUCCESS; - } - i40iw_debug(&iwdev->sc_dev, - I40IW_DEBUG_CM, - "freed pointer = %p\n", - child_listen_node); - kfree(child_listen_node); - cm_parent_listen_node->cm_core->stats_listen_nodes_destroyed++; - } - spin_unlock_irqrestore(&iwdev->cm_core.listen_list_lock, flags); - - return ret; -} - -/** - * i40iw_netdev_vlan_ipv6 - Gets the netdev and vlan - * @addr: local IPv6 address - * @vlan_id: vlan id for the given IPv6 address - * - * Returns the net_device of the IPv6 address and also sets the - * vlan id for that address. - */ -static struct net_device *i40iw_netdev_vlan_ipv6(u32 *addr, u16 *vlan_id) -{ - struct net_device *ip_dev = NULL; - struct in6_addr laddr6; - - if (!IS_ENABLED(CONFIG_IPV6)) - return NULL; - i40iw_copy_ip_htonl(laddr6.in6_u.u6_addr32, addr); - if (vlan_id) - *vlan_id = I40IW_NO_VLAN; - rcu_read_lock(); - for_each_netdev_rcu(&init_net, ip_dev) { - if (ipv6_chk_addr(&init_net, &laddr6, ip_dev, 1)) { - if (vlan_id) - *vlan_id = rdma_vlan_dev_vlan_id(ip_dev); - break; - } - } - rcu_read_unlock(); - return ip_dev; -} - -/** - * i40iw_get_vlan_ipv4 - Returns the vlan_id for IPv4 address - * @addr: local IPv4 address - */ -static u16 i40iw_get_vlan_ipv4(u32 *addr) -{ - struct net_device *netdev; - u16 vlan_id = I40IW_NO_VLAN; - - netdev = ip_dev_find(&init_net, htonl(addr[0])); - if (netdev) { - vlan_id = rdma_vlan_dev_vlan_id(netdev); - dev_put(netdev); - } - return vlan_id; -} - -/** - * i40iw_add_mqh_6 - Adds multiple qhashes for IPv6 - * @iwdev: iWarp device - * @cm_info: CM info for parent listen node - * @cm_parent_listen_node: The parent listen node - * - * Adds a qhash and a child listen node for every IPv6 address - * on the adapter and adds the associated qhash filter - */ -static enum i40iw_status_code i40iw_add_mqh_6(struct i40iw_device *iwdev, - struct i40iw_cm_info *cm_info, - struct i40iw_cm_listener *cm_parent_listen_node) -{ - struct net_device *ip_dev; - struct inet6_dev *idev; - struct inet6_ifaddr *ifp, *tmp; - enum i40iw_status_code ret = 0; - struct i40iw_cm_listener *child_listen_node; - unsigned long flags; - - rtnl_lock(); - for_each_netdev(&init_net, ip_dev) { - if ((((rdma_vlan_dev_vlan_id(ip_dev) < I40IW_NO_VLAN) && - (rdma_vlan_dev_real_dev(ip_dev) == iwdev->netdev)) || - (ip_dev == iwdev->netdev)) && (ip_dev->flags & IFF_UP)) { - idev = __in6_dev_get(ip_dev); - if (!idev) { - i40iw_pr_err("idev == NULL\n"); - break; - } - list_for_each_entry_safe(ifp, tmp, &idev->addr_list, if_list) { - i40iw_debug(&iwdev->sc_dev, - I40IW_DEBUG_CM, - "IP=%pI6, vlan_id=%d, MAC=%pM\n", - &ifp->addr, - rdma_vlan_dev_vlan_id(ip_dev), - ip_dev->dev_addr); - child_listen_node = - kzalloc(sizeof(*child_listen_node), GFP_ATOMIC); - i40iw_debug(&iwdev->sc_dev, - I40IW_DEBUG_CM, - "Allocating child listener %p\n", - child_listen_node); - if (!child_listen_node) { - ret = I40IW_ERR_NO_MEMORY; - goto exit; - } - cm_info->vlan_id = rdma_vlan_dev_vlan_id(ip_dev); - cm_parent_listen_node->vlan_id = cm_info->vlan_id; - - memcpy(child_listen_node, cm_parent_listen_node, - sizeof(*child_listen_node)); - - i40iw_copy_ip_ntohl(child_listen_node->loc_addr, - ifp->addr.in6_u.u6_addr32); - memcpy(cm_info->loc_addr, child_listen_node->loc_addr, - sizeof(cm_info->loc_addr)); - - ret = i40iw_manage_qhash(iwdev, cm_info, - I40IW_QHASH_TYPE_TCP_SYN, - I40IW_QHASH_MANAGE_TYPE_ADD, - NULL, true); - if (!ret) { - child_listen_node->qhash_set = true; - spin_lock_irqsave(&iwdev->cm_core.listen_list_lock, flags); - list_add(&child_listen_node->child_listen_list, - &cm_parent_listen_node->child_listen_list); - spin_unlock_irqrestore(&iwdev->cm_core.listen_list_lock, flags); - cm_parent_listen_node->cm_core->stats_listen_nodes_created++; - } else { - kfree(child_listen_node); - } - } - } - } -exit: - rtnl_unlock(); - return ret; -} - -/** - * i40iw_add_mqh_4 - Adds multiple qhashes for IPv4 - * @iwdev: iWarp device - * @cm_info: CM info for parent listen node - * @cm_parent_listen_node: The parent listen node - * - * Adds a qhash and a child listen node for every IPv4 address - * on the adapter and adds the associated qhash filter - */ -static enum i40iw_status_code i40iw_add_mqh_4( - struct i40iw_device *iwdev, - struct i40iw_cm_info *cm_info, - struct i40iw_cm_listener *cm_parent_listen_node) -{ - struct net_device *dev; - struct in_device *idev; - struct i40iw_cm_listener *child_listen_node; - enum i40iw_status_code ret = 0; - unsigned long flags; - - rtnl_lock(); - for_each_netdev(&init_net, dev) { - if ((((rdma_vlan_dev_vlan_id(dev) < I40IW_NO_VLAN) && - (rdma_vlan_dev_real_dev(dev) == iwdev->netdev)) || - (dev == iwdev->netdev)) && (dev->flags & IFF_UP)) { - const struct in_ifaddr *ifa; - - idev = in_dev_get(dev); - - in_dev_for_each_ifa_rtnl(ifa, idev) { - i40iw_debug(&iwdev->sc_dev, - I40IW_DEBUG_CM, - "Allocating child CM Listener forIP=%pI4, vlan_id=%d, MAC=%pM\n", - &ifa->ifa_address, - rdma_vlan_dev_vlan_id(dev), - dev->dev_addr); - child_listen_node = kzalloc(sizeof(*child_listen_node), GFP_KERNEL); - cm_parent_listen_node->cm_core->stats_listen_nodes_created++; - i40iw_debug(&iwdev->sc_dev, - I40IW_DEBUG_CM, - "Allocating child listener %p\n", - child_listen_node); - if (!child_listen_node) { - in_dev_put(idev); - ret = I40IW_ERR_NO_MEMORY; - goto exit; - } - cm_info->vlan_id = rdma_vlan_dev_vlan_id(dev); - cm_parent_listen_node->vlan_id = cm_info->vlan_id; - memcpy(child_listen_node, - cm_parent_listen_node, - sizeof(*child_listen_node)); - - child_listen_node->loc_addr[0] = ntohl(ifa->ifa_address); - memcpy(cm_info->loc_addr, child_listen_node->loc_addr, - sizeof(cm_info->loc_addr)); - - ret = i40iw_manage_qhash(iwdev, - cm_info, - I40IW_QHASH_TYPE_TCP_SYN, - I40IW_QHASH_MANAGE_TYPE_ADD, - NULL, - true); - if (!ret) { - child_listen_node->qhash_set = true; - spin_lock_irqsave(&iwdev->cm_core.listen_list_lock, flags); - list_add(&child_listen_node->child_listen_list, - &cm_parent_listen_node->child_listen_list); - spin_unlock_irqrestore(&iwdev->cm_core.listen_list_lock, flags); - } else { - kfree(child_listen_node); - cm_parent_listen_node->cm_core->stats_listen_nodes_created--; - } - } - - in_dev_put(idev); - } - } -exit: - rtnl_unlock(); - return ret; -} - -/** - * i40iw_dec_refcnt_listen - delete listener and associated cm nodes - * @cm_core: cm's core - * @listener: passive connection's listener - * @free_hanging_nodes: to free associated cm_nodes - * @apbvt_del: flag to delete the apbvt - */ -static int i40iw_dec_refcnt_listen(struct i40iw_cm_core *cm_core, - struct i40iw_cm_listener *listener, - int free_hanging_nodes, bool apbvt_del) -{ - int ret = -EINVAL; - int err = 0; - struct list_head *list_pos; - struct list_head *list_temp; - struct i40iw_cm_node *cm_node; - struct list_head reset_list; - struct i40iw_cm_info nfo; - struct i40iw_cm_node *loopback; - enum i40iw_cm_node_state old_state; - unsigned long flags; - - /* free non-accelerated child nodes for this listener */ - INIT_LIST_HEAD(&reset_list); - if (free_hanging_nodes) { - spin_lock_irqsave(&cm_core->ht_lock, flags); - list_for_each_safe(list_pos, - list_temp, &cm_core->non_accelerated_list) { - cm_node = container_of(list_pos, struct i40iw_cm_node, list); - if ((cm_node->listener == listener) && - !cm_node->accelerated) { - atomic_inc(&cm_node->ref_count); - list_add(&cm_node->reset_entry, &reset_list); - } - } - spin_unlock_irqrestore(&cm_core->ht_lock, flags); - } - - list_for_each_safe(list_pos, list_temp, &reset_list) { - cm_node = container_of(list_pos, struct i40iw_cm_node, reset_entry); - loopback = cm_node->loopbackpartner; - if (cm_node->state >= I40IW_CM_STATE_FIN_WAIT1) { - i40iw_rem_ref_cm_node(cm_node); - } else { - if (!loopback) { - i40iw_cleanup_retrans_entry(cm_node); - err = i40iw_send_reset(cm_node); - if (err) { - cm_node->state = I40IW_CM_STATE_CLOSED; - i40iw_pr_err("send reset\n"); - } else { - old_state = cm_node->state; - cm_node->state = I40IW_CM_STATE_LISTENER_DESTROYED; - if (old_state != I40IW_CM_STATE_MPAREQ_RCVD) - i40iw_rem_ref_cm_node(cm_node); - } - } else { - struct i40iw_cm_event event; - - event.cm_node = loopback; - memcpy(event.cm_info.rem_addr, - loopback->rem_addr, sizeof(event.cm_info.rem_addr)); - memcpy(event.cm_info.loc_addr, - loopback->loc_addr, sizeof(event.cm_info.loc_addr)); - event.cm_info.rem_port = loopback->rem_port; - event.cm_info.loc_port = loopback->loc_port; - event.cm_info.cm_id = loopback->cm_id; - event.cm_info.ipv4 = loopback->ipv4; - atomic_inc(&loopback->ref_count); - loopback->state = I40IW_CM_STATE_CLOSED; - i40iw_event_connect_error(&event); - cm_node->state = I40IW_CM_STATE_LISTENER_DESTROYED; - i40iw_rem_ref_cm_node(cm_node); - } - } - } - - if (!atomic_dec_return(&listener->ref_count)) { - spin_lock_irqsave(&cm_core->listen_list_lock, flags); - list_del(&listener->list); - spin_unlock_irqrestore(&cm_core->listen_list_lock, flags); - - if (listener->iwdev) { - if (apbvt_del) - i40iw_manage_apbvt(listener->iwdev, - listener->loc_port, - I40IW_MANAGE_APBVT_DEL); - - memcpy(nfo.loc_addr, listener->loc_addr, sizeof(nfo.loc_addr)); - nfo.loc_port = listener->loc_port; - nfo.ipv4 = listener->ipv4; - nfo.vlan_id = listener->vlan_id; - nfo.user_pri = listener->user_pri; - - if (!list_empty(&listener->child_listen_list)) { - i40iw_del_multiple_qhash(listener->iwdev, &nfo, listener); - } else { - if (listener->qhash_set) - i40iw_manage_qhash(listener->iwdev, - &nfo, - I40IW_QHASH_TYPE_TCP_SYN, - I40IW_QHASH_MANAGE_TYPE_DELETE, - NULL, - false); - } - } - - cm_core->stats_listen_destroyed++; - kfree(listener); - cm_core->stats_listen_nodes_destroyed++; - listener = NULL; - ret = 0; - } - - if (listener) { - if (atomic_read(&listener->pend_accepts_cnt) > 0) - i40iw_debug(cm_core->dev, - I40IW_DEBUG_CM, - "%s: listener (%p) pending accepts=%u\n", - __func__, - listener, - atomic_read(&listener->pend_accepts_cnt)); - } - - return ret; -} - -/** - * i40iw_cm_del_listen - delete a linstener - * @cm_core: cm's core - * @listener: passive connection's listener - * @apbvt_del: flag to delete apbvt - */ -static int i40iw_cm_del_listen(struct i40iw_cm_core *cm_core, - struct i40iw_cm_listener *listener, - bool apbvt_del) -{ - listener->listener_state = I40IW_CM_LISTENER_PASSIVE_STATE; - listener->cm_id = NULL; /* going to be destroyed pretty soon */ - return i40iw_dec_refcnt_listen(cm_core, listener, 1, apbvt_del); -} - -/** - * i40iw_addr_resolve_neigh - resolve neighbor address - * @iwdev: iwarp device structure - * @src_ip: local ip address - * @dst_ip: remote ip address - * @arpindex: if there is an arp entry - */ -static int i40iw_addr_resolve_neigh(struct i40iw_device *iwdev, - u32 src_ip, - u32 dst_ip, - int arpindex) -{ - struct rtable *rt; - struct neighbour *neigh; - int rc = arpindex; - __be32 dst_ipaddr = htonl(dst_ip); - __be32 src_ipaddr = htonl(src_ip); - - rt = ip_route_output(&init_net, dst_ipaddr, src_ipaddr, 0, 0); - if (IS_ERR(rt)) { - i40iw_pr_err("ip_route_output\n"); - return rc; - } - - neigh = dst_neigh_lookup(&rt->dst, &dst_ipaddr); - - rcu_read_lock(); - if (neigh) { - if (neigh->nud_state & NUD_VALID) { - if (arpindex >= 0) { - if (ether_addr_equal(iwdev->arp_table[arpindex].mac_addr, - neigh->ha)) - /* Mac address same as arp table */ - goto resolve_neigh_exit; - i40iw_manage_arp_cache(iwdev, - iwdev->arp_table[arpindex].mac_addr, - &dst_ip, - true, - I40IW_ARP_DELETE); - } - - i40iw_manage_arp_cache(iwdev, neigh->ha, &dst_ip, true, I40IW_ARP_ADD); - rc = i40iw_arp_table(iwdev, &dst_ip, true, NULL, I40IW_ARP_RESOLVE); - } else { - neigh_event_send(neigh, NULL); - } - } - resolve_neigh_exit: - - rcu_read_unlock(); - if (neigh) - neigh_release(neigh); - - ip_rt_put(rt); - return rc; -} - -/* - * i40iw_get_dst_ipv6 - */ -static struct dst_entry *i40iw_get_dst_ipv6(struct sockaddr_in6 *src_addr, - struct sockaddr_in6 *dst_addr) -{ - struct dst_entry *dst; - struct flowi6 fl6; - - memset(&fl6, 0, sizeof(fl6)); - fl6.daddr = dst_addr->sin6_addr; - fl6.saddr = src_addr->sin6_addr; - if (ipv6_addr_type(&fl6.daddr) & IPV6_ADDR_LINKLOCAL) - fl6.flowi6_oif = dst_addr->sin6_scope_id; - - dst = ip6_route_output(&init_net, NULL, &fl6); - return dst; -} - -/** - * i40iw_addr_resolve_neigh_ipv6 - resolve neighbor ipv6 address - * @iwdev: iwarp device structure - * @src: source ip address - * @dest: remote ip address - * @arpindex: if there is an arp entry - */ -static int i40iw_addr_resolve_neigh_ipv6(struct i40iw_device *iwdev, - u32 *src, - u32 *dest, - int arpindex) -{ - struct neighbour *neigh; - int rc = arpindex; - struct dst_entry *dst; - struct sockaddr_in6 dst_addr; - struct sockaddr_in6 src_addr; - - memset(&dst_addr, 0, sizeof(dst_addr)); - dst_addr.sin6_family = AF_INET6; - i40iw_copy_ip_htonl(dst_addr.sin6_addr.in6_u.u6_addr32, dest); - memset(&src_addr, 0, sizeof(src_addr)); - src_addr.sin6_family = AF_INET6; - i40iw_copy_ip_htonl(src_addr.sin6_addr.in6_u.u6_addr32, src); - dst = i40iw_get_dst_ipv6(&src_addr, &dst_addr); - if (!dst || dst->error) { - if (dst) { - i40iw_pr_err("ip6_route_output returned dst->error = %d\n", - dst->error); - dst_release(dst); - } - return rc; - } - - neigh = dst_neigh_lookup(dst, dst_addr.sin6_addr.in6_u.u6_addr32); - - rcu_read_lock(); - if (neigh) { - i40iw_debug(&iwdev->sc_dev, I40IW_DEBUG_CM, "dst_neigh_lookup MAC=%pM\n", neigh->ha); - if (neigh->nud_state & NUD_VALID) { - if (arpindex >= 0) { - if (ether_addr_equal - (iwdev->arp_table[arpindex].mac_addr, - neigh->ha)) { - /* Mac address same as in arp table */ - goto resolve_neigh_exit6; - } - i40iw_manage_arp_cache(iwdev, - iwdev->arp_table[arpindex].mac_addr, - dest, - false, - I40IW_ARP_DELETE); - } - i40iw_manage_arp_cache(iwdev, - neigh->ha, - dest, - false, - I40IW_ARP_ADD); - rc = i40iw_arp_table(iwdev, - dest, - false, - NULL, - I40IW_ARP_RESOLVE); - } else { - neigh_event_send(neigh, NULL); - } - } - - resolve_neigh_exit6: - rcu_read_unlock(); - if (neigh) - neigh_release(neigh); - dst_release(dst); - return rc; -} - -/** - * i40iw_ipv4_is_loopback - check if loopback - * @loc_addr: local addr to compare - * @rem_addr: remote address - */ -static bool i40iw_ipv4_is_loopback(u32 loc_addr, u32 rem_addr) -{ - return ipv4_is_loopback(htonl(rem_addr)) || (loc_addr == rem_addr); -} - -/** - * i40iw_ipv6_is_loopback - check if loopback - * @loc_addr: local addr to compare - * @rem_addr: remote address - */ -static bool i40iw_ipv6_is_loopback(u32 *loc_addr, u32 *rem_addr) -{ - struct in6_addr raddr6; - - i40iw_copy_ip_htonl(raddr6.in6_u.u6_addr32, rem_addr); - return !memcmp(loc_addr, rem_addr, 16) || ipv6_addr_loopback(&raddr6); -} - -/** - * i40iw_make_cm_node - create a new instance of a cm node - * @cm_core: cm's core - * @iwdev: iwarp device structure - * @cm_info: quad info for connection - * @listener: passive connection's listener - */ -static struct i40iw_cm_node *i40iw_make_cm_node( - struct i40iw_cm_core *cm_core, - struct i40iw_device *iwdev, - struct i40iw_cm_info *cm_info, - struct i40iw_cm_listener *listener) -{ - struct i40iw_cm_node *cm_node; - int oldarpindex; - int arpindex; - struct net_device *netdev = iwdev->netdev; - - /* create an hte and cm_node for this instance */ - cm_node = kzalloc(sizeof(*cm_node), GFP_ATOMIC); - if (!cm_node) - return NULL; - - /* set our node specific transport info */ - cm_node->ipv4 = cm_info->ipv4; - cm_node->vlan_id = cm_info->vlan_id; - if ((cm_node->vlan_id == I40IW_NO_VLAN) && iwdev->dcb) - cm_node->vlan_id = 0; - cm_node->tos = cm_info->tos; - cm_node->user_pri = cm_info->user_pri; - if (listener) { - if (listener->tos != cm_info->tos) - i40iw_debug(&iwdev->sc_dev, I40IW_DEBUG_DCB, - "application TOS[%d] and remote client TOS[%d] mismatch\n", - listener->tos, cm_info->tos); - cm_node->tos = max(listener->tos, cm_info->tos); - cm_node->user_pri = rt_tos2priority(cm_node->tos); - i40iw_debug(&iwdev->sc_dev, I40IW_DEBUG_DCB, "listener: TOS:[%d] UP:[%d]\n", - cm_node->tos, cm_node->user_pri); - } - memcpy(cm_node->loc_addr, cm_info->loc_addr, sizeof(cm_node->loc_addr)); - memcpy(cm_node->rem_addr, cm_info->rem_addr, sizeof(cm_node->rem_addr)); - cm_node->loc_port = cm_info->loc_port; - cm_node->rem_port = cm_info->rem_port; - - cm_node->mpa_frame_rev = iwdev->mpa_version; - cm_node->send_rdma0_op = SEND_RDMA_READ_ZERO; - cm_node->ird_size = I40IW_MAX_IRD_SIZE; - cm_node->ord_size = I40IW_MAX_ORD_SIZE; - - cm_node->listener = listener; - cm_node->cm_id = cm_info->cm_id; - ether_addr_copy(cm_node->loc_mac, netdev->dev_addr); - spin_lock_init(&cm_node->retrans_list_lock); - cm_node->ack_rcvd = false; - - atomic_set(&cm_node->ref_count, 1); - /* associate our parent CM core */ - cm_node->cm_core = cm_core; - cm_node->tcp_cntxt.loc_id = I40IW_CM_DEF_LOCAL_ID; - cm_node->tcp_cntxt.rcv_wscale = I40IW_CM_DEFAULT_RCV_WND_SCALE; - cm_node->tcp_cntxt.rcv_wnd = - I40IW_CM_DEFAULT_RCV_WND_SCALED >> I40IW_CM_DEFAULT_RCV_WND_SCALE; - if (cm_node->ipv4) { - cm_node->tcp_cntxt.loc_seq_num = secure_tcp_seq(htonl(cm_node->loc_addr[0]), - htonl(cm_node->rem_addr[0]), - htons(cm_node->loc_port), - htons(cm_node->rem_port)); - cm_node->tcp_cntxt.mss = iwdev->vsi.mtu - I40IW_MTU_TO_MSS_IPV4; - } else if (IS_ENABLED(CONFIG_IPV6)) { - __be32 loc[4] = { - htonl(cm_node->loc_addr[0]), htonl(cm_node->loc_addr[1]), - htonl(cm_node->loc_addr[2]), htonl(cm_node->loc_addr[3]) - }; - __be32 rem[4] = { - htonl(cm_node->rem_addr[0]), htonl(cm_node->rem_addr[1]), - htonl(cm_node->rem_addr[2]), htonl(cm_node->rem_addr[3]) - }; - cm_node->tcp_cntxt.loc_seq_num = secure_tcpv6_seq(loc, rem, - htons(cm_node->loc_port), - htons(cm_node->rem_port)); - cm_node->tcp_cntxt.mss = iwdev->vsi.mtu - I40IW_MTU_TO_MSS_IPV6; - } - - cm_node->iwdev = iwdev; - cm_node->dev = &iwdev->sc_dev; - - if ((cm_node->ipv4 && - i40iw_ipv4_is_loopback(cm_node->loc_addr[0], cm_node->rem_addr[0])) || - (!cm_node->ipv4 && i40iw_ipv6_is_loopback(cm_node->loc_addr, - cm_node->rem_addr))) { - arpindex = i40iw_arp_table(iwdev, - cm_node->rem_addr, - false, - NULL, - I40IW_ARP_RESOLVE); - } else { - oldarpindex = i40iw_arp_table(iwdev, - cm_node->rem_addr, - false, - NULL, - I40IW_ARP_RESOLVE); - if (cm_node->ipv4) - arpindex = i40iw_addr_resolve_neigh(iwdev, - cm_info->loc_addr[0], - cm_info->rem_addr[0], - oldarpindex); - else if (IS_ENABLED(CONFIG_IPV6)) - arpindex = i40iw_addr_resolve_neigh_ipv6(iwdev, - cm_info->loc_addr, - cm_info->rem_addr, - oldarpindex); - else - arpindex = -EINVAL; - } - if (arpindex < 0) { - i40iw_pr_err("cm_node arpindex\n"); - kfree(cm_node); - return NULL; - } - ether_addr_copy(cm_node->rem_mac, iwdev->arp_table[arpindex].mac_addr); - i40iw_add_hte_node(cm_core, cm_node); - cm_core->stats_nodes_created++; - return cm_node; -} - -/** - * i40iw_rem_ref_cm_node - destroy an instance of a cm node - * @cm_node: connection's node - */ -static void i40iw_rem_ref_cm_node(struct i40iw_cm_node *cm_node) -{ - struct i40iw_cm_core *cm_core = cm_node->cm_core; - struct i40iw_qp *iwqp; - struct i40iw_cm_info nfo; - unsigned long flags; - - spin_lock_irqsave(&cm_node->cm_core->ht_lock, flags); - if (atomic_dec_return(&cm_node->ref_count)) { - spin_unlock_irqrestore(&cm_node->cm_core->ht_lock, flags); - return; - } - list_del(&cm_node->list); - spin_unlock_irqrestore(&cm_node->cm_core->ht_lock, flags); - - /* if the node is destroyed before connection was accelerated */ - if (!cm_node->accelerated && cm_node->accept_pend) { - pr_err("node destroyed before established\n"); - atomic_dec(&cm_node->listener->pend_accepts_cnt); - } - if (cm_node->close_entry) - i40iw_handle_close_entry(cm_node, 0); - if (cm_node->listener) { - i40iw_dec_refcnt_listen(cm_core, cm_node->listener, 0, true); - } else { - if (cm_node->apbvt_set) { - i40iw_manage_apbvt(cm_node->iwdev, - cm_node->loc_port, - I40IW_MANAGE_APBVT_DEL); - cm_node->apbvt_set = 0; - } - i40iw_get_addr_info(cm_node, &nfo); - if (cm_node->qhash_set) { - i40iw_manage_qhash(cm_node->iwdev, - &nfo, - I40IW_QHASH_TYPE_TCP_ESTABLISHED, - I40IW_QHASH_MANAGE_TYPE_DELETE, - NULL, - false); - cm_node->qhash_set = 0; - } - } - - iwqp = cm_node->iwqp; - if (iwqp) { - iwqp->cm_node = NULL; - i40iw_qp_rem_ref(&iwqp->ibqp); - cm_node->iwqp = NULL; - } else if (cm_node->qhash_set) { - i40iw_get_addr_info(cm_node, &nfo); - i40iw_manage_qhash(cm_node->iwdev, - &nfo, - I40IW_QHASH_TYPE_TCP_ESTABLISHED, - I40IW_QHASH_MANAGE_TYPE_DELETE, - NULL, - false); - cm_node->qhash_set = 0; - } - - cm_node->cm_core->stats_nodes_destroyed++; - kfree(cm_node); -} - -/** - * i40iw_handle_fin_pkt - FIN packet received - * @cm_node: connection's node - */ -static void i40iw_handle_fin_pkt(struct i40iw_cm_node *cm_node) -{ - u32 ret; - - switch (cm_node->state) { - case I40IW_CM_STATE_SYN_RCVD: - case I40IW_CM_STATE_SYN_SENT: - case I40IW_CM_STATE_ESTABLISHED: - case I40IW_CM_STATE_MPAREJ_RCVD: - cm_node->tcp_cntxt.rcv_nxt++; - i40iw_cleanup_retrans_entry(cm_node); - cm_node->state = I40IW_CM_STATE_LAST_ACK; - i40iw_send_fin(cm_node); - break; - case I40IW_CM_STATE_MPAREQ_SENT: - i40iw_create_event(cm_node, I40IW_CM_EVENT_ABORTED); - cm_node->tcp_cntxt.rcv_nxt++; - i40iw_cleanup_retrans_entry(cm_node); - cm_node->state = I40IW_CM_STATE_CLOSED; - atomic_inc(&cm_node->ref_count); - i40iw_send_reset(cm_node); - break; - case I40IW_CM_STATE_FIN_WAIT1: - cm_node->tcp_cntxt.rcv_nxt++; - i40iw_cleanup_retrans_entry(cm_node); - cm_node->state = I40IW_CM_STATE_CLOSING; - i40iw_send_ack(cm_node); - /* - * Wait for ACK as this is simultaneous close. - * After we receive ACK, do not send anything. - * Just rm the node. - */ - break; - case I40IW_CM_STATE_FIN_WAIT2: - cm_node->tcp_cntxt.rcv_nxt++; - i40iw_cleanup_retrans_entry(cm_node); - cm_node->state = I40IW_CM_STATE_TIME_WAIT; - i40iw_send_ack(cm_node); - ret = - i40iw_schedule_cm_timer(cm_node, NULL, I40IW_TIMER_TYPE_CLOSE, 1, 0); - if (ret) - i40iw_pr_err("node %p state = %d\n", cm_node, cm_node->state); - break; - case I40IW_CM_STATE_TIME_WAIT: - cm_node->tcp_cntxt.rcv_nxt++; - i40iw_cleanup_retrans_entry(cm_node); - cm_node->state = I40IW_CM_STATE_CLOSED; - i40iw_rem_ref_cm_node(cm_node); - break; - case I40IW_CM_STATE_OFFLOADED: - default: - i40iw_pr_err("bad state node %p state = %d\n", cm_node, cm_node->state); - break; - } -} - -/** - * i40iw_handle_rst_pkt - process received RST packet - * @cm_node: connection's node - * @rbuf: receive buffer - */ -static void i40iw_handle_rst_pkt(struct i40iw_cm_node *cm_node, - struct i40iw_puda_buf *rbuf) -{ - i40iw_cleanup_retrans_entry(cm_node); - switch (cm_node->state) { - case I40IW_CM_STATE_SYN_SENT: - case I40IW_CM_STATE_MPAREQ_SENT: - switch (cm_node->mpa_frame_rev) { - case IETF_MPA_V2: - cm_node->mpa_frame_rev = IETF_MPA_V1; - /* send a syn and goto syn sent state */ - cm_node->state = I40IW_CM_STATE_SYN_SENT; - if (i40iw_send_syn(cm_node, 0)) - i40iw_active_open_err(cm_node, false); - break; - case IETF_MPA_V1: - default: - i40iw_active_open_err(cm_node, false); - break; - } - break; - case I40IW_CM_STATE_MPAREQ_RCVD: - atomic_inc(&cm_node->passive_state); - break; - case I40IW_CM_STATE_ESTABLISHED: - case I40IW_CM_STATE_SYN_RCVD: - case I40IW_CM_STATE_LISTENING: - i40iw_pr_err("Bad state state = %d\n", cm_node->state); - i40iw_passive_open_err(cm_node, false); - break; - case I40IW_CM_STATE_OFFLOADED: - i40iw_active_open_err(cm_node, false); - break; - case I40IW_CM_STATE_CLOSED: - break; - case I40IW_CM_STATE_FIN_WAIT2: - case I40IW_CM_STATE_FIN_WAIT1: - case I40IW_CM_STATE_LAST_ACK: - cm_node->cm_id->rem_ref(cm_node->cm_id); - fallthrough; - case I40IW_CM_STATE_TIME_WAIT: - cm_node->state = I40IW_CM_STATE_CLOSED; - i40iw_rem_ref_cm_node(cm_node); - break; - default: - break; - } -} - -/** - * i40iw_handle_rcv_mpa - Process a recv'd mpa buffer - * @cm_node: connection's node - * @rbuf: receive buffer - */ -static void i40iw_handle_rcv_mpa(struct i40iw_cm_node *cm_node, - struct i40iw_puda_buf *rbuf) -{ - int ret; - int datasize = rbuf->datalen; - u8 *dataloc = rbuf->data; - - enum i40iw_cm_event_type type = I40IW_CM_EVENT_UNKNOWN; - u32 res_type; - - ret = i40iw_parse_mpa(cm_node, dataloc, &res_type, datasize); - if (ret) { - if (cm_node->state == I40IW_CM_STATE_MPAREQ_SENT) - i40iw_active_open_err(cm_node, true); - else - i40iw_passive_open_err(cm_node, true); - return; - } - - switch (cm_node->state) { - case I40IW_CM_STATE_ESTABLISHED: - if (res_type == I40IW_MPA_REQUEST_REJECT) - i40iw_pr_err("state for reject\n"); - cm_node->state = I40IW_CM_STATE_MPAREQ_RCVD; - type = I40IW_CM_EVENT_MPA_REQ; - i40iw_send_ack(cm_node); /* ACK received MPA request */ - atomic_set(&cm_node->passive_state, - I40IW_PASSIVE_STATE_INDICATED); - break; - case I40IW_CM_STATE_MPAREQ_SENT: - i40iw_cleanup_retrans_entry(cm_node); - if (res_type == I40IW_MPA_REQUEST_REJECT) { - type = I40IW_CM_EVENT_MPA_REJECT; - cm_node->state = I40IW_CM_STATE_MPAREJ_RCVD; - } else { - type = I40IW_CM_EVENT_CONNECTED; - cm_node->state = I40IW_CM_STATE_OFFLOADED; - } - i40iw_send_ack(cm_node); - break; - default: - pr_err("%s wrong cm_node state =%d\n", __func__, cm_node->state); - break; - } - i40iw_create_event(cm_node, type); -} - -/** - * i40iw_indicate_pkt_err - Send up err event to cm - * @cm_node: connection's node - */ -static void i40iw_indicate_pkt_err(struct i40iw_cm_node *cm_node) -{ - switch (cm_node->state) { - case I40IW_CM_STATE_SYN_SENT: - case I40IW_CM_STATE_MPAREQ_SENT: - i40iw_active_open_err(cm_node, true); - break; - case I40IW_CM_STATE_ESTABLISHED: - case I40IW_CM_STATE_SYN_RCVD: - i40iw_passive_open_err(cm_node, true); - break; - case I40IW_CM_STATE_OFFLOADED: - default: - break; - } -} - -/** - * i40iw_check_syn - Check for error on received syn ack - * @cm_node: connection's node - * @tcph: pointer tcp header - */ -static int i40iw_check_syn(struct i40iw_cm_node *cm_node, struct tcphdr *tcph) -{ - int err = 0; - - if (ntohl(tcph->ack_seq) != cm_node->tcp_cntxt.loc_seq_num) { - err = 1; - i40iw_active_open_err(cm_node, true); - } - return err; -} - -/** - * i40iw_check_seq - check seq numbers if OK - * @cm_node: connection's node - * @tcph: pointer tcp header - */ -static int i40iw_check_seq(struct i40iw_cm_node *cm_node, struct tcphdr *tcph) -{ - int err = 0; - u32 seq; - u32 ack_seq; - u32 loc_seq_num = cm_node->tcp_cntxt.loc_seq_num; - u32 rcv_nxt = cm_node->tcp_cntxt.rcv_nxt; - u32 rcv_wnd; - - seq = ntohl(tcph->seq); - ack_seq = ntohl(tcph->ack_seq); - rcv_wnd = cm_node->tcp_cntxt.rcv_wnd; - if (ack_seq != loc_seq_num) - err = -1; - else if (!between(seq, rcv_nxt, (rcv_nxt + rcv_wnd))) - err = -1; - if (err) { - i40iw_pr_err("seq number\n"); - i40iw_indicate_pkt_err(cm_node); - } - return err; -} - -/** - * i40iw_handle_syn_pkt - is for Passive node - * @cm_node: connection's node - * @rbuf: receive buffer - */ -static void i40iw_handle_syn_pkt(struct i40iw_cm_node *cm_node, - struct i40iw_puda_buf *rbuf) -{ - struct tcphdr *tcph = (struct tcphdr *)rbuf->tcph; - int ret; - u32 inc_sequence; - int optionsize; - struct i40iw_cm_info nfo; - - optionsize = (tcph->doff << 2) - sizeof(struct tcphdr); - inc_sequence = ntohl(tcph->seq); - - switch (cm_node->state) { - case I40IW_CM_STATE_SYN_SENT: - case I40IW_CM_STATE_MPAREQ_SENT: - /* Rcvd syn on active open connection */ - i40iw_active_open_err(cm_node, 1); - break; - case I40IW_CM_STATE_LISTENING: - /* Passive OPEN */ - if (atomic_read(&cm_node->listener->pend_accepts_cnt) > - cm_node->listener->backlog) { - cm_node->cm_core->stats_backlog_drops++; - i40iw_passive_open_err(cm_node, false); - break; - } - ret = i40iw_handle_tcp_options(cm_node, tcph, optionsize, 1); - if (ret) { - i40iw_passive_open_err(cm_node, false); - /* drop pkt */ - break; - } - cm_node->tcp_cntxt.rcv_nxt = inc_sequence + 1; - cm_node->accept_pend = 1; - atomic_inc(&cm_node->listener->pend_accepts_cnt); - - cm_node->state = I40IW_CM_STATE_SYN_RCVD; - i40iw_get_addr_info(cm_node, &nfo); - ret = i40iw_manage_qhash(cm_node->iwdev, - &nfo, - I40IW_QHASH_TYPE_TCP_ESTABLISHED, - I40IW_QHASH_MANAGE_TYPE_ADD, - (void *)cm_node, - false); - cm_node->qhash_set = true; - break; - case I40IW_CM_STATE_CLOSED: - i40iw_cleanup_retrans_entry(cm_node); - atomic_inc(&cm_node->ref_count); - i40iw_send_reset(cm_node); - break; - case I40IW_CM_STATE_OFFLOADED: - case I40IW_CM_STATE_ESTABLISHED: - case I40IW_CM_STATE_FIN_WAIT1: - case I40IW_CM_STATE_FIN_WAIT2: - case I40IW_CM_STATE_MPAREQ_RCVD: - case I40IW_CM_STATE_LAST_ACK: - case I40IW_CM_STATE_CLOSING: - case I40IW_CM_STATE_UNKNOWN: - default: - break; - } -} - -/** - * i40iw_handle_synack_pkt - Process SYN+ACK packet (active side) - * @cm_node: connection's node - * @rbuf: receive buffer - */ -static void i40iw_handle_synack_pkt(struct i40iw_cm_node *cm_node, - struct i40iw_puda_buf *rbuf) -{ - struct tcphdr *tcph = (struct tcphdr *)rbuf->tcph; - int ret; - u32 inc_sequence; - int optionsize; - - optionsize = (tcph->doff << 2) - sizeof(struct tcphdr); - inc_sequence = ntohl(tcph->seq); - switch (cm_node->state) { - case I40IW_CM_STATE_SYN_SENT: - i40iw_cleanup_retrans_entry(cm_node); - /* active open */ - if (i40iw_check_syn(cm_node, tcph)) { - i40iw_pr_err("check syn fail\n"); - return; - } - cm_node->tcp_cntxt.rem_ack_num = ntohl(tcph->ack_seq); - /* setup options */ - ret = i40iw_handle_tcp_options(cm_node, tcph, optionsize, 0); - if (ret) { - i40iw_debug(cm_node->dev, - I40IW_DEBUG_CM, - "cm_node=%p tcp_options failed\n", - cm_node); - break; - } - i40iw_cleanup_retrans_entry(cm_node); - cm_node->tcp_cntxt.rcv_nxt = inc_sequence + 1; - i40iw_send_ack(cm_node); /* ACK for the syn_ack */ - ret = i40iw_send_mpa_request(cm_node); - if (ret) { - i40iw_debug(cm_node->dev, - I40IW_DEBUG_CM, - "cm_node=%p i40iw_send_mpa_request failed\n", - cm_node); - break; - } - cm_node->state = I40IW_CM_STATE_MPAREQ_SENT; - break; - case I40IW_CM_STATE_MPAREQ_RCVD: - i40iw_passive_open_err(cm_node, true); - break; - case I40IW_CM_STATE_LISTENING: - cm_node->tcp_cntxt.loc_seq_num = ntohl(tcph->ack_seq); - i40iw_cleanup_retrans_entry(cm_node); - cm_node->state = I40IW_CM_STATE_CLOSED; - i40iw_send_reset(cm_node); - break; - case I40IW_CM_STATE_CLOSED: - cm_node->tcp_cntxt.loc_seq_num = ntohl(tcph->ack_seq); - i40iw_cleanup_retrans_entry(cm_node); - atomic_inc(&cm_node->ref_count); - i40iw_send_reset(cm_node); - break; - case I40IW_CM_STATE_ESTABLISHED: - case I40IW_CM_STATE_FIN_WAIT1: - case I40IW_CM_STATE_FIN_WAIT2: - case I40IW_CM_STATE_LAST_ACK: - case I40IW_CM_STATE_OFFLOADED: - case I40IW_CM_STATE_CLOSING: - case I40IW_CM_STATE_UNKNOWN: - case I40IW_CM_STATE_MPAREQ_SENT: - default: - break; - } -} - -/** - * i40iw_handle_ack_pkt - process packet with ACK - * @cm_node: connection's node - * @rbuf: receive buffer - */ -static int i40iw_handle_ack_pkt(struct i40iw_cm_node *cm_node, - struct i40iw_puda_buf *rbuf) -{ - struct tcphdr *tcph = (struct tcphdr *)rbuf->tcph; - u32 inc_sequence; - int ret = 0; - int optionsize; - u32 datasize = rbuf->datalen; - - optionsize = (tcph->doff << 2) - sizeof(struct tcphdr); - - if (i40iw_check_seq(cm_node, tcph)) - return -EINVAL; - - inc_sequence = ntohl(tcph->seq); - switch (cm_node->state) { - case I40IW_CM_STATE_SYN_RCVD: - i40iw_cleanup_retrans_entry(cm_node); - ret = i40iw_handle_tcp_options(cm_node, tcph, optionsize, 1); - if (ret) - break; - cm_node->tcp_cntxt.rem_ack_num = ntohl(tcph->ack_seq); - cm_node->state = I40IW_CM_STATE_ESTABLISHED; - if (datasize) { - cm_node->tcp_cntxt.rcv_nxt = inc_sequence + datasize; - i40iw_handle_rcv_mpa(cm_node, rbuf); - } - break; - case I40IW_CM_STATE_ESTABLISHED: - i40iw_cleanup_retrans_entry(cm_node); - if (datasize) { - cm_node->tcp_cntxt.rcv_nxt = inc_sequence + datasize; - i40iw_handle_rcv_mpa(cm_node, rbuf); - } - break; - case I40IW_CM_STATE_MPAREQ_SENT: - cm_node->tcp_cntxt.rem_ack_num = ntohl(tcph->ack_seq); - if (datasize) { - cm_node->tcp_cntxt.rcv_nxt = inc_sequence + datasize; - cm_node->ack_rcvd = false; - i40iw_handle_rcv_mpa(cm_node, rbuf); - } else { - cm_node->ack_rcvd = true; - } - break; - case I40IW_CM_STATE_LISTENING: - i40iw_cleanup_retrans_entry(cm_node); - cm_node->state = I40IW_CM_STATE_CLOSED; - i40iw_send_reset(cm_node); - break; - case I40IW_CM_STATE_CLOSED: - i40iw_cleanup_retrans_entry(cm_node); - atomic_inc(&cm_node->ref_count); - i40iw_send_reset(cm_node); - break; - case I40IW_CM_STATE_LAST_ACK: - case I40IW_CM_STATE_CLOSING: - i40iw_cleanup_retrans_entry(cm_node); - cm_node->state = I40IW_CM_STATE_CLOSED; - if (!cm_node->accept_pend) - cm_node->cm_id->rem_ref(cm_node->cm_id); - i40iw_rem_ref_cm_node(cm_node); - break; - case I40IW_CM_STATE_FIN_WAIT1: - i40iw_cleanup_retrans_entry(cm_node); - cm_node->state = I40IW_CM_STATE_FIN_WAIT2; - break; - case I40IW_CM_STATE_SYN_SENT: - case I40IW_CM_STATE_FIN_WAIT2: - case I40IW_CM_STATE_OFFLOADED: - case I40IW_CM_STATE_MPAREQ_RCVD: - case I40IW_CM_STATE_UNKNOWN: - default: - i40iw_cleanup_retrans_entry(cm_node); - break; - } - return ret; -} - -/** - * i40iw_process_packet - process cm packet - * @cm_node: connection's node - * @rbuf: receive buffer - */ -static void i40iw_process_packet(struct i40iw_cm_node *cm_node, - struct i40iw_puda_buf *rbuf) -{ - enum i40iw_tcpip_pkt_type pkt_type = I40IW_PKT_TYPE_UNKNOWN; - struct tcphdr *tcph = (struct tcphdr *)rbuf->tcph; - u32 fin_set = 0; - int ret; - - if (tcph->rst) { - pkt_type = I40IW_PKT_TYPE_RST; - } else if (tcph->syn) { - pkt_type = I40IW_PKT_TYPE_SYN; - if (tcph->ack) - pkt_type = I40IW_PKT_TYPE_SYNACK; - } else if (tcph->ack) { - pkt_type = I40IW_PKT_TYPE_ACK; - } - if (tcph->fin) - fin_set = 1; - - switch (pkt_type) { - case I40IW_PKT_TYPE_SYN: - i40iw_handle_syn_pkt(cm_node, rbuf); - break; - case I40IW_PKT_TYPE_SYNACK: - i40iw_handle_synack_pkt(cm_node, rbuf); - break; - case I40IW_PKT_TYPE_ACK: - ret = i40iw_handle_ack_pkt(cm_node, rbuf); - if (fin_set && !ret) - i40iw_handle_fin_pkt(cm_node); - break; - case I40IW_PKT_TYPE_RST: - i40iw_handle_rst_pkt(cm_node, rbuf); - break; - default: - if (fin_set && - (!i40iw_check_seq(cm_node, (struct tcphdr *)rbuf->tcph))) - i40iw_handle_fin_pkt(cm_node); - break; - } -} - -/** - * i40iw_make_listen_node - create a listen node with params - * @cm_core: cm's core - * @iwdev: iwarp device structure - * @cm_info: quad info for connection - */ -static struct i40iw_cm_listener *i40iw_make_listen_node( - struct i40iw_cm_core *cm_core, - struct i40iw_device *iwdev, - struct i40iw_cm_info *cm_info) -{ - struct i40iw_cm_listener *listener; - unsigned long flags; - - /* cannot have multiple matching listeners */ - listener = i40iw_find_listener(cm_core, cm_info->loc_addr, - cm_info->loc_port, - cm_info->vlan_id, - I40IW_CM_LISTENER_EITHER_STATE); - if (listener && - (listener->listener_state == I40IW_CM_LISTENER_ACTIVE_STATE)) { - atomic_dec(&listener->ref_count); - i40iw_debug(cm_core->dev, - I40IW_DEBUG_CM, - "Not creating listener since it already exists\n"); - return NULL; - } - - if (!listener) { - /* create a CM listen node (1/2 node to compare incoming traffic to) */ - listener = kzalloc(sizeof(*listener), GFP_KERNEL); - if (!listener) - return NULL; - cm_core->stats_listen_nodes_created++; - memcpy(listener->loc_addr, cm_info->loc_addr, sizeof(listener->loc_addr)); - listener->loc_port = cm_info->loc_port; - - INIT_LIST_HEAD(&listener->child_listen_list); - - atomic_set(&listener->ref_count, 1); - } else { - listener->reused_node = 1; - } - - listener->cm_id = cm_info->cm_id; - listener->ipv4 = cm_info->ipv4; - listener->vlan_id = cm_info->vlan_id; - atomic_set(&listener->pend_accepts_cnt, 0); - listener->cm_core = cm_core; - listener->iwdev = iwdev; - - listener->backlog = cm_info->backlog; - listener->listener_state = I40IW_CM_LISTENER_ACTIVE_STATE; - - if (!listener->reused_node) { - spin_lock_irqsave(&cm_core->listen_list_lock, flags); - list_add(&listener->list, &cm_core->listen_nodes); - spin_unlock_irqrestore(&cm_core->listen_list_lock, flags); - } - - return listener; -} - -/** - * i40iw_create_cm_node - make a connection node with params - * @cm_core: cm's core - * @iwdev: iwarp device structure - * @conn_param: upper layer connection parameters - * @cm_info: quad info for connection - */ -static struct i40iw_cm_node *i40iw_create_cm_node( - struct i40iw_cm_core *cm_core, - struct i40iw_device *iwdev, - struct iw_cm_conn_param *conn_param, - struct i40iw_cm_info *cm_info) -{ - struct i40iw_cm_node *cm_node; - struct i40iw_cm_listener *loopback_remotelistener; - struct i40iw_cm_node *loopback_remotenode; - struct i40iw_cm_info loopback_cm_info; - - u16 private_data_len = conn_param->private_data_len; - const void *private_data = conn_param->private_data; - - /* create a CM connection node */ - cm_node = i40iw_make_cm_node(cm_core, iwdev, cm_info, NULL); - if (!cm_node) - return ERR_PTR(-ENOMEM); - /* set our node side to client (active) side */ - cm_node->tcp_cntxt.client = 1; - cm_node->tcp_cntxt.rcv_wscale = I40IW_CM_DEFAULT_RCV_WND_SCALE; - - i40iw_record_ird_ord(cm_node, conn_param->ird, conn_param->ord); - - if (!memcmp(cm_info->loc_addr, cm_info->rem_addr, sizeof(cm_info->loc_addr))) { - loopback_remotelistener = i40iw_find_listener( - cm_core, - cm_info->rem_addr, - cm_node->rem_port, - cm_node->vlan_id, - I40IW_CM_LISTENER_ACTIVE_STATE); - if (!loopback_remotelistener) { - i40iw_rem_ref_cm_node(cm_node); - return ERR_PTR(-ECONNREFUSED); - } else { - loopback_cm_info = *cm_info; - loopback_cm_info.loc_port = cm_info->rem_port; - loopback_cm_info.rem_port = cm_info->loc_port; - loopback_cm_info.cm_id = loopback_remotelistener->cm_id; - loopback_cm_info.ipv4 = cm_info->ipv4; - loopback_remotenode = i40iw_make_cm_node(cm_core, - iwdev, - &loopback_cm_info, - loopback_remotelistener); - if (!loopback_remotenode) { - i40iw_rem_ref_cm_node(cm_node); - return ERR_PTR(-ENOMEM); - } - cm_core->stats_loopbacks++; - loopback_remotenode->loopbackpartner = cm_node; - loopback_remotenode->tcp_cntxt.rcv_wscale = - I40IW_CM_DEFAULT_RCV_WND_SCALE; - cm_node->loopbackpartner = loopback_remotenode; - memcpy(loopback_remotenode->pdata_buf, private_data, - private_data_len); - loopback_remotenode->pdata.size = private_data_len; - - if (loopback_remotenode->ord_size > cm_node->ird_size) - loopback_remotenode->ord_size = - cm_node->ird_size; - - cm_node->state = I40IW_CM_STATE_OFFLOADED; - cm_node->tcp_cntxt.rcv_nxt = - loopback_remotenode->tcp_cntxt.loc_seq_num; - loopback_remotenode->tcp_cntxt.rcv_nxt = - cm_node->tcp_cntxt.loc_seq_num; - cm_node->tcp_cntxt.max_snd_wnd = - loopback_remotenode->tcp_cntxt.rcv_wnd; - loopback_remotenode->tcp_cntxt.max_snd_wnd = cm_node->tcp_cntxt.rcv_wnd; - cm_node->tcp_cntxt.snd_wnd = loopback_remotenode->tcp_cntxt.rcv_wnd; - loopback_remotenode->tcp_cntxt.snd_wnd = cm_node->tcp_cntxt.rcv_wnd; - cm_node->tcp_cntxt.snd_wscale = loopback_remotenode->tcp_cntxt.rcv_wscale; - loopback_remotenode->tcp_cntxt.snd_wscale = cm_node->tcp_cntxt.rcv_wscale; - } - return cm_node; - } - - cm_node->pdata.size = private_data_len; - cm_node->pdata.addr = cm_node->pdata_buf; - - memcpy(cm_node->pdata_buf, private_data, private_data_len); - - cm_node->state = I40IW_CM_STATE_SYN_SENT; - return cm_node; -} - -/** - * i40iw_cm_reject - reject and teardown a connection - * @cm_node: connection's node - * @pdata: ptr to private data for reject - * @plen: size of private data - */ -static int i40iw_cm_reject(struct i40iw_cm_node *cm_node, const void *pdata, u8 plen) -{ - int ret = 0; - int err; - int passive_state; - struct iw_cm_id *cm_id = cm_node->cm_id; - struct i40iw_cm_node *loopback = cm_node->loopbackpartner; - - if (cm_node->tcp_cntxt.client) - return ret; - i40iw_cleanup_retrans_entry(cm_node); - - if (!loopback) { - passive_state = atomic_inc_return(&cm_node->passive_state); - if (passive_state == I40IW_SEND_RESET_EVENT) { - cm_node->state = I40IW_CM_STATE_CLOSED; - i40iw_rem_ref_cm_node(cm_node); - } else { - if (cm_node->state == I40IW_CM_STATE_LISTENER_DESTROYED) { - i40iw_rem_ref_cm_node(cm_node); - } else { - ret = i40iw_send_mpa_reject(cm_node, pdata, plen); - if (ret) { - cm_node->state = I40IW_CM_STATE_CLOSED; - err = i40iw_send_reset(cm_node); - if (err) - i40iw_pr_err("send reset failed\n"); - } else { - cm_id->add_ref(cm_id); - } - } - } - } else { - cm_node->cm_id = NULL; - if (cm_node->state == I40IW_CM_STATE_LISTENER_DESTROYED) { - i40iw_rem_ref_cm_node(cm_node); - i40iw_rem_ref_cm_node(loopback); - } else { - ret = i40iw_send_cm_event(loopback, - loopback->cm_id, - IW_CM_EVENT_CONNECT_REPLY, - -ECONNREFUSED); - i40iw_rem_ref_cm_node(cm_node); - loopback->state = I40IW_CM_STATE_CLOSING; - - cm_id = loopback->cm_id; - i40iw_rem_ref_cm_node(loopback); - cm_id->rem_ref(cm_id); - } - } - - return ret; -} - -/** - * i40iw_cm_close - close of cm connection - * @cm_node: connection's node - */ -static int i40iw_cm_close(struct i40iw_cm_node *cm_node) -{ - int ret = 0; - - if (!cm_node) - return -EINVAL; - - switch (cm_node->state) { - case I40IW_CM_STATE_SYN_RCVD: - case I40IW_CM_STATE_SYN_SENT: - case I40IW_CM_STATE_ONE_SIDE_ESTABLISHED: - case I40IW_CM_STATE_ESTABLISHED: - case I40IW_CM_STATE_ACCEPTING: - case I40IW_CM_STATE_MPAREQ_SENT: - case I40IW_CM_STATE_MPAREQ_RCVD: - i40iw_cleanup_retrans_entry(cm_node); - i40iw_send_reset(cm_node); - break; - case I40IW_CM_STATE_CLOSE_WAIT: - cm_node->state = I40IW_CM_STATE_LAST_ACK; - i40iw_send_fin(cm_node); - break; - case I40IW_CM_STATE_FIN_WAIT1: - case I40IW_CM_STATE_FIN_WAIT2: - case I40IW_CM_STATE_LAST_ACK: - case I40IW_CM_STATE_TIME_WAIT: - case I40IW_CM_STATE_CLOSING: - ret = -1; - break; - case I40IW_CM_STATE_LISTENING: - i40iw_cleanup_retrans_entry(cm_node); - i40iw_send_reset(cm_node); - break; - case I40IW_CM_STATE_MPAREJ_RCVD: - case I40IW_CM_STATE_UNKNOWN: - case I40IW_CM_STATE_INITED: - case I40IW_CM_STATE_CLOSED: - case I40IW_CM_STATE_LISTENER_DESTROYED: - i40iw_rem_ref_cm_node(cm_node); - break; - case I40IW_CM_STATE_OFFLOADED: - if (cm_node->send_entry) - i40iw_pr_err("send_entry\n"); - i40iw_rem_ref_cm_node(cm_node); - break; - } - return ret; -} - -/** - * i40iw_receive_ilq - recv an ETHERNET packet, and process it - * through CM - * @vsi: pointer to the vsi structure - * @rbuf: receive buffer - */ -void i40iw_receive_ilq(struct i40iw_sc_vsi *vsi, struct i40iw_puda_buf *rbuf) -{ - struct i40iw_cm_node *cm_node; - struct i40iw_cm_listener *listener; - struct iphdr *iph; - struct ipv6hdr *ip6h; - struct tcphdr *tcph; - struct i40iw_cm_info cm_info; - struct i40iw_sc_dev *dev = vsi->dev; - struct i40iw_device *iwdev = (struct i40iw_device *)dev->back_dev; - struct i40iw_cm_core *cm_core = &iwdev->cm_core; - struct vlan_ethhdr *ethh; - u16 vtag; - - /* if vlan, then maclen = 18 else 14 */ - iph = (struct iphdr *)rbuf->iph; - memset(&cm_info, 0, sizeof(cm_info)); - - i40iw_debug_buf(dev, - I40IW_DEBUG_ILQ, - "RECEIVE ILQ BUFFER", - rbuf->mem.va, - rbuf->totallen); - ethh = (struct vlan_ethhdr *)rbuf->mem.va; - - if (ethh->h_vlan_proto == htons(ETH_P_8021Q)) { - vtag = ntohs(ethh->h_vlan_TCI); - cm_info.user_pri = (vtag & VLAN_PRIO_MASK) >> VLAN_PRIO_SHIFT; - cm_info.vlan_id = vtag & VLAN_VID_MASK; - i40iw_debug(cm_core->dev, - I40IW_DEBUG_CM, - "%s vlan_id=%d\n", - __func__, - cm_info.vlan_id); - } else { - cm_info.vlan_id = I40IW_NO_VLAN; - } - tcph = (struct tcphdr *)rbuf->tcph; - - if (rbuf->ipv4) { - cm_info.loc_addr[0] = ntohl(iph->daddr); - cm_info.rem_addr[0] = ntohl(iph->saddr); - cm_info.ipv4 = true; - cm_info.tos = iph->tos; - } else { - ip6h = (struct ipv6hdr *)rbuf->iph; - i40iw_copy_ip_ntohl(cm_info.loc_addr, - ip6h->daddr.in6_u.u6_addr32); - i40iw_copy_ip_ntohl(cm_info.rem_addr, - ip6h->saddr.in6_u.u6_addr32); - cm_info.ipv4 = false; - cm_info.tos = (ip6h->priority << 4) | (ip6h->flow_lbl[0] >> 4); - } - cm_info.loc_port = ntohs(tcph->dest); - cm_info.rem_port = ntohs(tcph->source); - cm_node = i40iw_find_node(cm_core, - cm_info.rem_port, - cm_info.rem_addr, - cm_info.loc_port, - cm_info.loc_addr, - true, - false); - - if (!cm_node) { - /* Only type of packet accepted are for */ - /* the PASSIVE open (syn only) */ - if (!tcph->syn || tcph->ack) - return; - listener = - i40iw_find_listener(cm_core, - cm_info.loc_addr, - cm_info.loc_port, - cm_info.vlan_id, - I40IW_CM_LISTENER_ACTIVE_STATE); - if (!listener) { - cm_info.cm_id = NULL; - i40iw_debug(cm_core->dev, - I40IW_DEBUG_CM, - "%s no listener found\n", - __func__); - return; - } - cm_info.cm_id = listener->cm_id; - cm_node = i40iw_make_cm_node(cm_core, iwdev, &cm_info, listener); - if (!cm_node) { - i40iw_debug(cm_core->dev, - I40IW_DEBUG_CM, - "%s allocate node failed\n", - __func__); - atomic_dec(&listener->ref_count); - return; - } - if (!tcph->rst && !tcph->fin) { - cm_node->state = I40IW_CM_STATE_LISTENING; - } else { - i40iw_rem_ref_cm_node(cm_node); - return; - } - atomic_inc(&cm_node->ref_count); - } else if (cm_node->state == I40IW_CM_STATE_OFFLOADED) { - i40iw_rem_ref_cm_node(cm_node); - return; - } - i40iw_process_packet(cm_node, rbuf); - i40iw_rem_ref_cm_node(cm_node); -} - -/** - * i40iw_setup_cm_core - allocate a top level instance of a cm - * core - * @iwdev: iwarp device structure - */ -int i40iw_setup_cm_core(struct i40iw_device *iwdev) -{ - struct i40iw_cm_core *cm_core = &iwdev->cm_core; - - cm_core->iwdev = iwdev; - cm_core->dev = &iwdev->sc_dev; - - INIT_LIST_HEAD(&cm_core->accelerated_list); - INIT_LIST_HEAD(&cm_core->non_accelerated_list); - INIT_LIST_HEAD(&cm_core->listen_nodes); - - timer_setup(&cm_core->tcp_timer, i40iw_cm_timer_tick, 0); - - spin_lock_init(&cm_core->ht_lock); - spin_lock_init(&cm_core->listen_list_lock); - spin_lock_init(&cm_core->apbvt_lock); - - cm_core->event_wq = alloc_ordered_workqueue("iwewq", - WQ_MEM_RECLAIM); - if (!cm_core->event_wq) - goto error; - - cm_core->disconn_wq = alloc_ordered_workqueue("iwdwq", - WQ_MEM_RECLAIM); - if (!cm_core->disconn_wq) - goto error; - - return 0; -error: - i40iw_cleanup_cm_core(&iwdev->cm_core); - - return -ENOMEM; -} - -/** - * i40iw_cleanup_cm_core - deallocate a top level instance of a - * cm core - * @cm_core: cm's core - */ -void i40iw_cleanup_cm_core(struct i40iw_cm_core *cm_core) -{ - unsigned long flags; - - if (!cm_core) - return; - - spin_lock_irqsave(&cm_core->ht_lock, flags); - if (timer_pending(&cm_core->tcp_timer)) - del_timer_sync(&cm_core->tcp_timer); - spin_unlock_irqrestore(&cm_core->ht_lock, flags); - - if (cm_core->event_wq) - destroy_workqueue(cm_core->event_wq); - if (cm_core->disconn_wq) - destroy_workqueue(cm_core->disconn_wq); -} - -/** - * i40iw_init_tcp_ctx - setup qp context - * @cm_node: connection's node - * @tcp_info: offload info for tcp - * @iwqp: associate qp for the connection - */ -static void i40iw_init_tcp_ctx(struct i40iw_cm_node *cm_node, - struct i40iw_tcp_offload_info *tcp_info, - struct i40iw_qp *iwqp) -{ - tcp_info->ipv4 = cm_node->ipv4; - tcp_info->drop_ooo_seg = true; - tcp_info->wscale = true; - tcp_info->ignore_tcp_opt = true; - tcp_info->ignore_tcp_uns_opt = true; - tcp_info->no_nagle = false; - - tcp_info->ttl = I40IW_DEFAULT_TTL; - tcp_info->rtt_var = cpu_to_le32(I40IW_DEFAULT_RTT_VAR); - tcp_info->ss_thresh = cpu_to_le32(I40IW_DEFAULT_SS_THRESH); - tcp_info->rexmit_thresh = I40IW_DEFAULT_REXMIT_THRESH; - - tcp_info->tcp_state = I40IW_TCP_STATE_ESTABLISHED; - tcp_info->snd_wscale = cm_node->tcp_cntxt.snd_wscale; - tcp_info->rcv_wscale = cm_node->tcp_cntxt.rcv_wscale; - - tcp_info->snd_nxt = cpu_to_le32(cm_node->tcp_cntxt.loc_seq_num); - tcp_info->snd_wnd = cpu_to_le32(cm_node->tcp_cntxt.snd_wnd); - tcp_info->rcv_nxt = cpu_to_le32(cm_node->tcp_cntxt.rcv_nxt); - tcp_info->snd_max = cpu_to_le32(cm_node->tcp_cntxt.loc_seq_num); - - tcp_info->snd_una = cpu_to_le32(cm_node->tcp_cntxt.loc_seq_num); - tcp_info->cwnd = cpu_to_le32(2 * cm_node->tcp_cntxt.mss); - tcp_info->snd_wl1 = cpu_to_le32(cm_node->tcp_cntxt.rcv_nxt); - tcp_info->snd_wl2 = cpu_to_le32(cm_node->tcp_cntxt.loc_seq_num); - tcp_info->max_snd_window = cpu_to_le32(cm_node->tcp_cntxt.max_snd_wnd); - tcp_info->rcv_wnd = cpu_to_le32(cm_node->tcp_cntxt.rcv_wnd << - cm_node->tcp_cntxt.rcv_wscale); - - tcp_info->flow_label = 0; - tcp_info->snd_mss = cpu_to_le32(((u32)cm_node->tcp_cntxt.mss)); - if (cm_node->vlan_id <= VLAN_VID_MASK) { - tcp_info->insert_vlan_tag = true; - tcp_info->vlan_tag = cpu_to_le16(((u16)cm_node->user_pri << I40IW_VLAN_PRIO_SHIFT) | - cm_node->vlan_id); - } - if (cm_node->ipv4) { - tcp_info->src_port = cpu_to_le16(cm_node->loc_port); - tcp_info->dst_port = cpu_to_le16(cm_node->rem_port); - - tcp_info->dest_ip_addr3 = cpu_to_le32(cm_node->rem_addr[0]); - tcp_info->local_ipaddr3 = cpu_to_le32(cm_node->loc_addr[0]); - tcp_info->arp_idx = - cpu_to_le16((u16)i40iw_arp_table( - iwqp->iwdev, - &tcp_info->dest_ip_addr3, - true, - NULL, - I40IW_ARP_RESOLVE)); - } else { - tcp_info->src_port = cpu_to_le16(cm_node->loc_port); - tcp_info->dst_port = cpu_to_le16(cm_node->rem_port); - tcp_info->dest_ip_addr0 = cpu_to_le32(cm_node->rem_addr[0]); - tcp_info->dest_ip_addr1 = cpu_to_le32(cm_node->rem_addr[1]); - tcp_info->dest_ip_addr2 = cpu_to_le32(cm_node->rem_addr[2]); - tcp_info->dest_ip_addr3 = cpu_to_le32(cm_node->rem_addr[3]); - tcp_info->local_ipaddr0 = cpu_to_le32(cm_node->loc_addr[0]); - tcp_info->local_ipaddr1 = cpu_to_le32(cm_node->loc_addr[1]); - tcp_info->local_ipaddr2 = cpu_to_le32(cm_node->loc_addr[2]); - tcp_info->local_ipaddr3 = cpu_to_le32(cm_node->loc_addr[3]); - tcp_info->arp_idx = - cpu_to_le16((u16)i40iw_arp_table( - iwqp->iwdev, - &tcp_info->dest_ip_addr0, - false, - NULL, - I40IW_ARP_RESOLVE)); - } -} - -/** - * i40iw_cm_init_tsa_conn - setup qp for RTS - * @iwqp: associate qp for the connection - * @cm_node: connection's node - */ -static void i40iw_cm_init_tsa_conn(struct i40iw_qp *iwqp, - struct i40iw_cm_node *cm_node) -{ - struct i40iw_tcp_offload_info tcp_info; - struct i40iwarp_offload_info *iwarp_info; - struct i40iw_qp_host_ctx_info *ctx_info; - struct i40iw_device *iwdev = iwqp->iwdev; - struct i40iw_sc_dev *dev = &iwqp->iwdev->sc_dev; - - memset(&tcp_info, 0x00, sizeof(struct i40iw_tcp_offload_info)); - iwarp_info = &iwqp->iwarp_info; - ctx_info = &iwqp->ctx_info; - - ctx_info->tcp_info = &tcp_info; - ctx_info->send_cq_num = iwqp->iwscq->sc_cq.cq_uk.cq_id; - ctx_info->rcv_cq_num = iwqp->iwrcq->sc_cq.cq_uk.cq_id; - - iwarp_info->ord_size = cm_node->ord_size; - iwarp_info->ird_size = i40iw_derive_hw_ird_setting(cm_node->ird_size); - - if (iwarp_info->ord_size == 1) - iwarp_info->ord_size = 2; - - iwarp_info->rd_enable = true; - iwarp_info->rdmap_ver = 1; - iwarp_info->ddp_ver = 1; - - iwarp_info->pd_id = iwqp->iwpd->sc_pd.pd_id; - - ctx_info->tcp_info_valid = true; - ctx_info->iwarp_info_valid = true; - ctx_info->add_to_qoslist = true; - ctx_info->user_pri = cm_node->user_pri; - - i40iw_init_tcp_ctx(cm_node, &tcp_info, iwqp); - if (cm_node->snd_mark_en) { - iwarp_info->snd_mark_en = true; - iwarp_info->snd_mark_offset = (tcp_info.snd_nxt & - SNDMARKER_SEQNMASK) + cm_node->lsmm_size; - } - - cm_node->state = I40IW_CM_STATE_OFFLOADED; - tcp_info.tcp_state = I40IW_TCP_STATE_ESTABLISHED; - tcp_info.src_mac_addr_idx = iwdev->mac_ip_table_idx; - tcp_info.tos = cm_node->tos; - - dev->iw_priv_qp_ops->qp_setctx(&iwqp->sc_qp, (u64 *)(iwqp->host_ctx.va), ctx_info); - - /* once tcp_info is set, no need to do it again */ - ctx_info->tcp_info_valid = false; - ctx_info->iwarp_info_valid = false; - ctx_info->add_to_qoslist = false; -} - -/** - * i40iw_cm_disconn - when a connection is being closed - * @iwqp: associate qp for the connection - */ -void i40iw_cm_disconn(struct i40iw_qp *iwqp) -{ - struct disconn_work *work; - struct i40iw_device *iwdev = iwqp->iwdev; - struct i40iw_cm_core *cm_core = &iwdev->cm_core; - unsigned long flags; - - work = kzalloc(sizeof(*work), GFP_ATOMIC); - if (!work) - return; /* Timer will clean up */ - - spin_lock_irqsave(&iwdev->qptable_lock, flags); - if (!iwdev->qp_table[iwqp->ibqp.qp_num]) { - spin_unlock_irqrestore(&iwdev->qptable_lock, flags); - i40iw_debug(&iwdev->sc_dev, I40IW_DEBUG_CM, - "%s qp_id %d is already freed\n", - __func__, iwqp->ibqp.qp_num); - kfree(work); - return; - } - i40iw_qp_add_ref(&iwqp->ibqp); - spin_unlock_irqrestore(&iwdev->qptable_lock, flags); - - work->iwqp = iwqp; - INIT_WORK(&work->work, i40iw_disconnect_worker); - queue_work(cm_core->disconn_wq, &work->work); - return; -} - -/** - * i40iw_qp_disconnect - free qp and close cm - * @iwqp: associate qp for the connection - */ -static void i40iw_qp_disconnect(struct i40iw_qp *iwqp) -{ - struct i40iw_device *iwdev; - struct i40iw_ib_device *iwibdev; - - iwdev = to_iwdev(iwqp->ibqp.device); - if (!iwdev) { - i40iw_pr_err("iwdev == NULL\n"); - return; - } - - iwibdev = iwdev->iwibdev; - - if (iwqp->active_conn) { - /* indicate this connection is NOT active */ - iwqp->active_conn = 0; - } else { - /* Need to free the Last Streaming Mode Message */ - if (iwqp->ietf_mem.va) { - if (iwqp->lsmm_mr) - iwibdev->ibdev.ops.dereg_mr(iwqp->lsmm_mr, - NULL); - i40iw_free_dma_mem(iwdev->sc_dev.hw, &iwqp->ietf_mem); - } - } - - /* close the CM node down if it is still active */ - if (iwqp->cm_node) { - i40iw_debug(&iwdev->sc_dev, I40IW_DEBUG_CM, "%s Call close API\n", __func__); - i40iw_cm_close(iwqp->cm_node); - } -} - -/** - * i40iw_cm_disconn_true - called by worker thread to disconnect qp - * @iwqp: associate qp for the connection - */ -static void i40iw_cm_disconn_true(struct i40iw_qp *iwqp) -{ - struct iw_cm_id *cm_id; - struct i40iw_device *iwdev; - struct i40iw_sc_qp *qp = &iwqp->sc_qp; - u16 last_ae; - u8 original_hw_tcp_state; - u8 original_ibqp_state; - int disconn_status = 0; - int issue_disconn = 0; - int issue_close = 0; - int issue_flush = 0; - struct ib_event ibevent; - unsigned long flags; - int ret; - - if (!iwqp) { - i40iw_pr_err("iwqp == NULL\n"); - return; - } - - spin_lock_irqsave(&iwqp->lock, flags); - cm_id = iwqp->cm_id; - /* make sure we havent already closed this connection */ - if (!cm_id) { - spin_unlock_irqrestore(&iwqp->lock, flags); - return; - } - - iwdev = to_iwdev(iwqp->ibqp.device); - - original_hw_tcp_state = iwqp->hw_tcp_state; - original_ibqp_state = iwqp->ibqp_state; - last_ae = iwqp->last_aeq; - - if (qp->term_flags) { - issue_disconn = 1; - issue_close = 1; - iwqp->cm_id = NULL; - /*When term timer expires after cm_timer, don't want - *terminate-handler to issue cm_disconn which can re-free - *a QP even after its refcnt=0. - */ - i40iw_terminate_del_timer(qp); - if (!iwqp->flush_issued) { - iwqp->flush_issued = 1; - issue_flush = 1; - } - } else if ((original_hw_tcp_state == I40IW_TCP_STATE_CLOSE_WAIT) || - ((original_ibqp_state == IB_QPS_RTS) && - (last_ae == I40IW_AE_LLP_CONNECTION_RESET))) { - issue_disconn = 1; - if (last_ae == I40IW_AE_LLP_CONNECTION_RESET) - disconn_status = -ECONNRESET; - } - - if (((original_hw_tcp_state == I40IW_TCP_STATE_CLOSED) || - (original_hw_tcp_state == I40IW_TCP_STATE_TIME_WAIT) || - (last_ae == I40IW_AE_RDMAP_ROE_BAD_LLP_CLOSE) || - (last_ae == I40IW_AE_LLP_CONNECTION_RESET) || - iwdev->reset)) { - issue_close = 1; - iwqp->cm_id = NULL; - if (!iwqp->flush_issued) { - iwqp->flush_issued = 1; - issue_flush = 1; - } - } - - spin_unlock_irqrestore(&iwqp->lock, flags); - if (issue_flush && !iwqp->destroyed) { - /* Flush the queues */ - i40iw_flush_wqes(iwdev, iwqp); - - if (qp->term_flags && iwqp->ibqp.event_handler) { - ibevent.device = iwqp->ibqp.device; - ibevent.event = (qp->eventtype == TERM_EVENT_QP_FATAL) ? - IB_EVENT_QP_FATAL : IB_EVENT_QP_ACCESS_ERR; - ibevent.element.qp = &iwqp->ibqp; - iwqp->ibqp.event_handler(&ibevent, iwqp->ibqp.qp_context); - } - } - - if (cm_id && cm_id->event_handler) { - if (issue_disconn) { - ret = i40iw_send_cm_event(NULL, - cm_id, - IW_CM_EVENT_DISCONNECT, - disconn_status); - - if (ret) - i40iw_debug(&iwdev->sc_dev, - I40IW_DEBUG_CM, - "disconnect event failed %s: - cm_id = %p\n", - __func__, cm_id); - } - if (issue_close) { - i40iw_qp_disconnect(iwqp); - cm_id->provider_data = iwqp; - ret = i40iw_send_cm_event(NULL, cm_id, IW_CM_EVENT_CLOSE, 0); - if (ret) - i40iw_debug(&iwdev->sc_dev, - I40IW_DEBUG_CM, - "close event failed %s: - cm_id = %p\n", - __func__, cm_id); - cm_id->rem_ref(cm_id); - } - } -} - -/** - * i40iw_disconnect_worker - worker for connection close - * @work: points or disconn structure - */ -static void i40iw_disconnect_worker(struct work_struct *work) -{ - struct disconn_work *dwork = container_of(work, struct disconn_work, work); - struct i40iw_qp *iwqp = dwork->iwqp; - - kfree(dwork); - i40iw_cm_disconn_true(iwqp); - i40iw_qp_rem_ref(&iwqp->ibqp); -} - -/** - * i40iw_accept - registered call for connection to be accepted - * @cm_id: cm information for passive connection - * @conn_param: accpet parameters - */ -int i40iw_accept(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param) -{ - struct ib_qp *ibqp; - struct i40iw_qp *iwqp; - struct i40iw_device *iwdev; - struct i40iw_sc_dev *dev; - struct i40iw_cm_core *cm_core; - struct i40iw_cm_node *cm_node; - struct ib_qp_attr attr; - int passive_state; - struct ib_mr *ibmr; - struct i40iw_pd *iwpd; - u16 buf_len = 0; - struct i40iw_kmem_info accept; - enum i40iw_status_code status; - u64 tagged_offset; - unsigned long flags; - - memset(&attr, 0, sizeof(attr)); - ibqp = i40iw_get_qp(cm_id->device, conn_param->qpn); - if (!ibqp) - return -EINVAL; - - iwqp = to_iwqp(ibqp); - iwdev = iwqp->iwdev; - dev = &iwdev->sc_dev; - cm_core = &iwdev->cm_core; - cm_node = (struct i40iw_cm_node *)cm_id->provider_data; - - if (((struct sockaddr_in *)&cm_id->local_addr)->sin_family == AF_INET) { - cm_node->ipv4 = true; - cm_node->vlan_id = i40iw_get_vlan_ipv4(cm_node->loc_addr); - } else { - cm_node->ipv4 = false; - i40iw_netdev_vlan_ipv6(cm_node->loc_addr, &cm_node->vlan_id); - } - i40iw_debug(cm_node->dev, - I40IW_DEBUG_CM, - "Accept vlan_id=%d\n", - cm_node->vlan_id); - if (cm_node->state == I40IW_CM_STATE_LISTENER_DESTROYED) { - if (cm_node->loopbackpartner) - i40iw_rem_ref_cm_node(cm_node->loopbackpartner); - i40iw_rem_ref_cm_node(cm_node); - return -EINVAL; - } - - passive_state = atomic_inc_return(&cm_node->passive_state); - if (passive_state == I40IW_SEND_RESET_EVENT) { - i40iw_rem_ref_cm_node(cm_node); - return -ECONNRESET; - } - - cm_node->cm_core->stats_accepts++; - iwqp->cm_node = (void *)cm_node; - cm_node->iwqp = iwqp; - - buf_len = conn_param->private_data_len + I40IW_MAX_IETF_SIZE; - - status = i40iw_allocate_dma_mem(dev->hw, &iwqp->ietf_mem, buf_len, 1); - - if (status) - return -ENOMEM; - cm_node->pdata.size = conn_param->private_data_len; - accept.addr = iwqp->ietf_mem.va; - accept.size = i40iw_cm_build_mpa_frame(cm_node, &accept, MPA_KEY_REPLY); - memcpy(accept.addr + accept.size, conn_param->private_data, - conn_param->private_data_len); - - /* setup our first outgoing iWarp send WQE (the IETF frame response) */ - if ((cm_node->ipv4 && - !i40iw_ipv4_is_loopback(cm_node->loc_addr[0], cm_node->rem_addr[0])) || - (!cm_node->ipv4 && - !i40iw_ipv6_is_loopback(cm_node->loc_addr, cm_node->rem_addr))) { - iwpd = iwqp->iwpd; - tagged_offset = (uintptr_t)iwqp->ietf_mem.va; - ibmr = i40iw_reg_phys_mr(&iwpd->ibpd, - iwqp->ietf_mem.pa, - buf_len, - IB_ACCESS_LOCAL_WRITE, - &tagged_offset); - if (IS_ERR(ibmr)) { - i40iw_free_dma_mem(dev->hw, &iwqp->ietf_mem); - return -ENOMEM; - } - - ibmr->pd = &iwpd->ibpd; - ibmr->device = iwpd->ibpd.device; - iwqp->lsmm_mr = ibmr; - if (iwqp->page) - iwqp->sc_qp.qp_uk.sq_base = kmap(iwqp->page); - dev->iw_priv_qp_ops->qp_send_lsmm(&iwqp->sc_qp, - iwqp->ietf_mem.va, - (accept.size + conn_param->private_data_len), - ibmr->lkey); - - } else { - if (iwqp->page) - iwqp->sc_qp.qp_uk.sq_base = kmap(iwqp->page); - dev->iw_priv_qp_ops->qp_send_lsmm(&iwqp->sc_qp, NULL, 0, 0); - } - - if (iwqp->page) - kunmap(iwqp->page); - - iwqp->cm_id = cm_id; - cm_node->cm_id = cm_id; - - cm_id->provider_data = (void *)iwqp; - iwqp->active_conn = 0; - - cm_node->lsmm_size = accept.size + conn_param->private_data_len; - i40iw_cm_init_tsa_conn(iwqp, cm_node); - cm_id->add_ref(cm_id); - i40iw_qp_add_ref(&iwqp->ibqp); - - attr.qp_state = IB_QPS_RTS; - cm_node->qhash_set = false; - i40iw_modify_qp(&iwqp->ibqp, &attr, IB_QP_STATE, NULL); - - cm_node->accelerated = true; - spin_lock_irqsave(&cm_core->ht_lock, flags); - list_move_tail(&cm_node->list, &cm_core->accelerated_list); - spin_unlock_irqrestore(&cm_core->ht_lock, flags); - - status = - i40iw_send_cm_event(cm_node, cm_id, IW_CM_EVENT_ESTABLISHED, 0); - if (status) - i40iw_debug(dev, I40IW_DEBUG_CM, "error sending cm event - ESTABLISHED\n"); - - if (cm_node->loopbackpartner) { - cm_node->loopbackpartner->pdata.size = conn_param->private_data_len; - - /* copy entire MPA frame to our cm_node's frame */ - memcpy(cm_node->loopbackpartner->pdata_buf, - conn_param->private_data, - conn_param->private_data_len); - i40iw_create_event(cm_node->loopbackpartner, I40IW_CM_EVENT_CONNECTED); - } - - if (cm_node->accept_pend) { - atomic_dec(&cm_node->listener->pend_accepts_cnt); - cm_node->accept_pend = 0; - } - return 0; -} - -/** - * i40iw_reject - registered call for connection to be rejected - * @cm_id: cm information for passive connection - * @pdata: private data to be sent - * @pdata_len: private data length - */ -int i40iw_reject(struct iw_cm_id *cm_id, const void *pdata, u8 pdata_len) -{ - struct i40iw_device *iwdev; - struct i40iw_cm_node *cm_node; - struct i40iw_cm_node *loopback; - - cm_node = (struct i40iw_cm_node *)cm_id->provider_data; - loopback = cm_node->loopbackpartner; - cm_node->cm_id = cm_id; - cm_node->pdata.size = pdata_len; - - iwdev = to_iwdev(cm_id->device); - if (!iwdev) - return -EINVAL; - cm_node->cm_core->stats_rejects++; - - if (pdata_len + sizeof(struct ietf_mpa_v2) > MAX_CM_BUFFER) - return -EINVAL; - - if (loopback) { - memcpy(&loopback->pdata_buf, pdata, pdata_len); - loopback->pdata.size = pdata_len; - } - - return i40iw_cm_reject(cm_node, pdata, pdata_len); -} - -/** - * i40iw_connect - registered call for connection to be established - * @cm_id: cm information for passive connection - * @conn_param: Information about the connection - */ -int i40iw_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param) -{ - struct ib_qp *ibqp; - struct i40iw_qp *iwqp; - struct i40iw_device *iwdev; - struct i40iw_cm_node *cm_node; - struct i40iw_cm_info cm_info; - struct sockaddr_in *laddr; - struct sockaddr_in *raddr; - struct sockaddr_in6 *laddr6; - struct sockaddr_in6 *raddr6; - int ret = 0; - - ibqp = i40iw_get_qp(cm_id->device, conn_param->qpn); - if (!ibqp) - return -EINVAL; - iwqp = to_iwqp(ibqp); - if (!iwqp) - return -EINVAL; - iwdev = to_iwdev(iwqp->ibqp.device); - if (!iwdev) - return -EINVAL; - - laddr = (struct sockaddr_in *)&cm_id->m_local_addr; - raddr = (struct sockaddr_in *)&cm_id->m_remote_addr; - laddr6 = (struct sockaddr_in6 *)&cm_id->m_local_addr; - raddr6 = (struct sockaddr_in6 *)&cm_id->m_remote_addr; - - if (!(laddr->sin_port) || !(raddr->sin_port)) - return -EINVAL; - - iwqp->active_conn = 1; - iwqp->cm_id = NULL; - cm_id->provider_data = iwqp; - - /* set up the connection params for the node */ - if (cm_id->remote_addr.ss_family == AF_INET) { - cm_info.ipv4 = true; - memset(cm_info.loc_addr, 0, sizeof(cm_info.loc_addr)); - memset(cm_info.rem_addr, 0, sizeof(cm_info.rem_addr)); - cm_info.loc_addr[0] = ntohl(laddr->sin_addr.s_addr); - cm_info.rem_addr[0] = ntohl(raddr->sin_addr.s_addr); - cm_info.loc_port = ntohs(laddr->sin_port); - cm_info.rem_port = ntohs(raddr->sin_port); - cm_info.vlan_id = i40iw_get_vlan_ipv4(cm_info.loc_addr); - } else { - cm_info.ipv4 = false; - i40iw_copy_ip_ntohl(cm_info.loc_addr, - laddr6->sin6_addr.in6_u.u6_addr32); - i40iw_copy_ip_ntohl(cm_info.rem_addr, - raddr6->sin6_addr.in6_u.u6_addr32); - cm_info.loc_port = ntohs(laddr6->sin6_port); - cm_info.rem_port = ntohs(raddr6->sin6_port); - i40iw_netdev_vlan_ipv6(cm_info.loc_addr, &cm_info.vlan_id); - } - cm_info.cm_id = cm_id; - cm_info.tos = cm_id->tos; - cm_info.user_pri = rt_tos2priority(cm_id->tos); - i40iw_debug(&iwdev->sc_dev, I40IW_DEBUG_DCB, "%s TOS:[%d] UP:[%d]\n", - __func__, cm_id->tos, cm_info.user_pri); - cm_id->add_ref(cm_id); - cm_node = i40iw_create_cm_node(&iwdev->cm_core, iwdev, - conn_param, &cm_info); - - if (IS_ERR(cm_node)) { - ret = PTR_ERR(cm_node); - cm_id->rem_ref(cm_id); - return ret; - } - - if ((cm_info.ipv4 && (laddr->sin_addr.s_addr != raddr->sin_addr.s_addr)) || - (!cm_info.ipv4 && memcmp(laddr6->sin6_addr.in6_u.u6_addr32, - raddr6->sin6_addr.in6_u.u6_addr32, - sizeof(laddr6->sin6_addr.in6_u.u6_addr32)))) { - if (i40iw_manage_qhash(iwdev, &cm_info, I40IW_QHASH_TYPE_TCP_ESTABLISHED, - I40IW_QHASH_MANAGE_TYPE_ADD, NULL, true)) { - ret = -EINVAL; - goto err; - } - cm_node->qhash_set = true; - } - - if (i40iw_manage_apbvt(iwdev, cm_info.loc_port, - I40IW_MANAGE_APBVT_ADD)) { - ret = -EINVAL; - goto err; - } - - cm_node->apbvt_set = true; - iwqp->cm_node = cm_node; - cm_node->iwqp = iwqp; - iwqp->cm_id = cm_id; - i40iw_qp_add_ref(&iwqp->ibqp); - - if (cm_node->state != I40IW_CM_STATE_OFFLOADED) { - cm_node->state = I40IW_CM_STATE_SYN_SENT; - ret = i40iw_send_syn(cm_node, 0); - if (ret) - goto err; - } - - if (cm_node->loopbackpartner) { - cm_node->loopbackpartner->state = I40IW_CM_STATE_MPAREQ_RCVD; - i40iw_create_event(cm_node->loopbackpartner, - I40IW_CM_EVENT_MPA_REQ); - } - - i40iw_debug(cm_node->dev, - I40IW_DEBUG_CM, - "Api - connect(): port=0x%04x, cm_node=%p, cm_id = %p.\n", - cm_node->rem_port, - cm_node, - cm_node->cm_id); - - return 0; - -err: - if (cm_info.ipv4) - i40iw_debug(&iwdev->sc_dev, - I40IW_DEBUG_CM, - "Api - connect() FAILED: dest addr=%pI4", - cm_info.rem_addr); - else - i40iw_debug(&iwdev->sc_dev, - I40IW_DEBUG_CM, - "Api - connect() FAILED: dest addr=%pI6", - cm_info.rem_addr); - - i40iw_rem_ref_cm_node(cm_node); - cm_id->rem_ref(cm_id); - iwdev->cm_core.stats_connect_errs++; - return ret; -} - -/** - * i40iw_create_listen - registered call creating listener - * @cm_id: cm information for passive connection - * @backlog: to max accept pending count - */ -int i40iw_create_listen(struct iw_cm_id *cm_id, int backlog) -{ - struct i40iw_device *iwdev; - struct i40iw_cm_listener *cm_listen_node; - struct i40iw_cm_info cm_info; - enum i40iw_status_code ret; - struct sockaddr_in *laddr; - struct sockaddr_in6 *laddr6; - bool wildcard = false; - - iwdev = to_iwdev(cm_id->device); - if (!iwdev) - return -EINVAL; - - laddr = (struct sockaddr_in *)&cm_id->m_local_addr; - laddr6 = (struct sockaddr_in6 *)&cm_id->m_local_addr; - memset(&cm_info, 0, sizeof(cm_info)); - if (laddr->sin_family == AF_INET) { - cm_info.ipv4 = true; - cm_info.loc_addr[0] = ntohl(laddr->sin_addr.s_addr); - cm_info.loc_port = ntohs(laddr->sin_port); - - if (laddr->sin_addr.s_addr != INADDR_ANY) - cm_info.vlan_id = i40iw_get_vlan_ipv4(cm_info.loc_addr); - else - wildcard = true; - - } else { - cm_info.ipv4 = false; - i40iw_copy_ip_ntohl(cm_info.loc_addr, - laddr6->sin6_addr.in6_u.u6_addr32); - cm_info.loc_port = ntohs(laddr6->sin6_port); - if (ipv6_addr_type(&laddr6->sin6_addr) != IPV6_ADDR_ANY) - i40iw_netdev_vlan_ipv6(cm_info.loc_addr, - &cm_info.vlan_id); - else - wildcard = true; - } - cm_info.backlog = backlog; - cm_info.cm_id = cm_id; - - cm_listen_node = i40iw_make_listen_node(&iwdev->cm_core, iwdev, &cm_info); - if (!cm_listen_node) { - i40iw_pr_err("cm_listen_node == NULL\n"); - return -ENOMEM; - } - - cm_id->provider_data = cm_listen_node; - - cm_listen_node->tos = cm_id->tos; - cm_listen_node->user_pri = rt_tos2priority(cm_id->tos); - cm_info.user_pri = cm_listen_node->user_pri; - - if (!cm_listen_node->reused_node) { - if (wildcard) { - if (cm_info.ipv4) - ret = i40iw_add_mqh_4(iwdev, - &cm_info, - cm_listen_node); - else - ret = i40iw_add_mqh_6(iwdev, - &cm_info, - cm_listen_node); - if (ret) - goto error; - - ret = i40iw_manage_apbvt(iwdev, - cm_info.loc_port, - I40IW_MANAGE_APBVT_ADD); - - if (ret) - goto error; - } else { - ret = i40iw_manage_qhash(iwdev, - &cm_info, - I40IW_QHASH_TYPE_TCP_SYN, - I40IW_QHASH_MANAGE_TYPE_ADD, - NULL, - true); - if (ret) - goto error; - cm_listen_node->qhash_set = true; - ret = i40iw_manage_apbvt(iwdev, - cm_info.loc_port, - I40IW_MANAGE_APBVT_ADD); - if (ret) - goto error; - } - } - cm_id->add_ref(cm_id); - cm_listen_node->cm_core->stats_listen_created++; - return 0; - error: - i40iw_cm_del_listen(&iwdev->cm_core, (void *)cm_listen_node, false); - return -EINVAL; -} - -/** - * i40iw_destroy_listen - registered call to destroy listener - * @cm_id: cm information for passive connection - */ -int i40iw_destroy_listen(struct iw_cm_id *cm_id) -{ - struct i40iw_device *iwdev; - - iwdev = to_iwdev(cm_id->device); - if (cm_id->provider_data) - i40iw_cm_del_listen(&iwdev->cm_core, cm_id->provider_data, true); - else - i40iw_pr_err("cm_id->provider_data was NULL\n"); - - cm_id->rem_ref(cm_id); - - return 0; -} - -/** - * i40iw_cm_event_connected - handle connected active node - * @event: the info for cm_node of connection - */ -static void i40iw_cm_event_connected(struct i40iw_cm_event *event) -{ - struct i40iw_qp *iwqp; - struct i40iw_device *iwdev; - struct i40iw_cm_core *cm_core; - struct i40iw_cm_node *cm_node; - struct i40iw_sc_dev *dev; - struct ib_qp_attr attr; - struct iw_cm_id *cm_id; - unsigned long flags; - int status; - bool read0; - - cm_node = event->cm_node; - cm_id = cm_node->cm_id; - iwqp = (struct i40iw_qp *)cm_id->provider_data; - iwdev = to_iwdev(iwqp->ibqp.device); - dev = &iwdev->sc_dev; - cm_core = &iwdev->cm_core; - - if (iwqp->destroyed) { - status = -ETIMEDOUT; - goto error; - } - i40iw_cm_init_tsa_conn(iwqp, cm_node); - read0 = (cm_node->send_rdma0_op == SEND_RDMA_READ_ZERO); - if (iwqp->page) - iwqp->sc_qp.qp_uk.sq_base = kmap(iwqp->page); - dev->iw_priv_qp_ops->qp_send_rtt(&iwqp->sc_qp, read0); - if (iwqp->page) - kunmap(iwqp->page); - - memset(&attr, 0, sizeof(attr)); - attr.qp_state = IB_QPS_RTS; - cm_node->qhash_set = false; - i40iw_modify_qp(&iwqp->ibqp, &attr, IB_QP_STATE, NULL); - - cm_node->accelerated = true; - spin_lock_irqsave(&cm_core->ht_lock, flags); - list_move_tail(&cm_node->list, &cm_core->accelerated_list); - spin_unlock_irqrestore(&cm_core->ht_lock, flags); - status = i40iw_send_cm_event(cm_node, cm_id, IW_CM_EVENT_CONNECT_REPLY, - 0); - if (status) - i40iw_debug(dev, I40IW_DEBUG_CM, "error sending cm event - CONNECT_REPLY\n"); - - return; - -error: - iwqp->cm_id = NULL; - cm_id->provider_data = NULL; - i40iw_send_cm_event(event->cm_node, - cm_id, - IW_CM_EVENT_CONNECT_REPLY, - status); - cm_id->rem_ref(cm_id); - i40iw_rem_ref_cm_node(event->cm_node); -} - -/** - * i40iw_cm_event_reset - handle reset - * @event: the info for cm_node of connection - */ -static void i40iw_cm_event_reset(struct i40iw_cm_event *event) -{ - struct i40iw_cm_node *cm_node = event->cm_node; - struct iw_cm_id *cm_id = cm_node->cm_id; - struct i40iw_qp *iwqp; - - if (!cm_id) - return; - - iwqp = cm_id->provider_data; - if (!iwqp) - return; - - i40iw_debug(cm_node->dev, - I40IW_DEBUG_CM, - "reset event %p - cm_id = %p\n", - event->cm_node, cm_id); - iwqp->cm_id = NULL; - - i40iw_send_cm_event(cm_node, cm_node->cm_id, IW_CM_EVENT_DISCONNECT, -ECONNRESET); - i40iw_send_cm_event(cm_node, cm_node->cm_id, IW_CM_EVENT_CLOSE, 0); -} - -/** - * i40iw_cm_event_handler - worker thread callback to send event to cm upper layer - * @work: pointer of cm event info. - */ -static void i40iw_cm_event_handler(struct work_struct *work) -{ - struct i40iw_cm_event *event = container_of(work, - struct i40iw_cm_event, - event_work); - struct i40iw_cm_node *cm_node; - - if (!event || !event->cm_node || !event->cm_node->cm_core) - return; - - cm_node = event->cm_node; - - switch (event->type) { - case I40IW_CM_EVENT_MPA_REQ: - i40iw_send_cm_event(cm_node, - cm_node->cm_id, - IW_CM_EVENT_CONNECT_REQUEST, - 0); - break; - case I40IW_CM_EVENT_RESET: - i40iw_cm_event_reset(event); - break; - case I40IW_CM_EVENT_CONNECTED: - if (!event->cm_node->cm_id || - (event->cm_node->state != I40IW_CM_STATE_OFFLOADED)) - break; - i40iw_cm_event_connected(event); - break; - case I40IW_CM_EVENT_MPA_REJECT: - if (!event->cm_node->cm_id || - (cm_node->state == I40IW_CM_STATE_OFFLOADED)) - break; - i40iw_send_cm_event(cm_node, - cm_node->cm_id, - IW_CM_EVENT_CONNECT_REPLY, - -ECONNREFUSED); - break; - case I40IW_CM_EVENT_ABORTED: - if (!event->cm_node->cm_id || - (event->cm_node->state == I40IW_CM_STATE_OFFLOADED)) - break; - i40iw_event_connect_error(event); - break; - default: - i40iw_pr_err("event type = %d\n", event->type); - break; - } - - event->cm_info.cm_id->rem_ref(event->cm_info.cm_id); - i40iw_rem_ref_cm_node(event->cm_node); - kfree(event); -} - -/** - * i40iw_cm_post_event - queue event request for worker thread - * @event: cm node's info for up event call - */ -static void i40iw_cm_post_event(struct i40iw_cm_event *event) -{ - atomic_inc(&event->cm_node->ref_count); - event->cm_info.cm_id->add_ref(event->cm_info.cm_id); - INIT_WORK(&event->event_work, i40iw_cm_event_handler); - - queue_work(event->cm_node->cm_core->event_wq, &event->event_work); -} - -/** - * i40iw_qhash_ctrl - enable/disable qhash for list - * @iwdev: device pointer - * @parent_listen_node: parent listen node - * @nfo: cm info node - * @ipaddr: Pointer to IPv4 or IPv6 address - * @ipv4: flag indicating IPv4 when true - * @ifup: flag indicating interface up when true - * - * Enables or disables the qhash for the node in the child - * listen list that matches ipaddr. If no matching IP was found - * it will allocate and add a new child listen node to the - * parent listen node. The listen_list_lock is assumed to be - * held when called. - */ -static void i40iw_qhash_ctrl(struct i40iw_device *iwdev, - struct i40iw_cm_listener *parent_listen_node, - struct i40iw_cm_info *nfo, - u32 *ipaddr, bool ipv4, bool ifup) -{ - struct list_head *child_listen_list = &parent_listen_node->child_listen_list; - struct i40iw_cm_listener *child_listen_node; - struct list_head *pos, *tpos; - enum i40iw_status_code ret; - bool node_allocated = false; - enum i40iw_quad_hash_manage_type op = - ifup ? I40IW_QHASH_MANAGE_TYPE_ADD : I40IW_QHASH_MANAGE_TYPE_DELETE; - - list_for_each_safe(pos, tpos, child_listen_list) { - child_listen_node = - list_entry(pos, - struct i40iw_cm_listener, - child_listen_list); - if (!memcmp(child_listen_node->loc_addr, ipaddr, ipv4 ? 4 : 16)) - goto set_qhash; - } - - /* if not found then add a child listener if interface is going up */ - if (!ifup) - return; - child_listen_node = kmemdup(parent_listen_node, - sizeof(*child_listen_node), GFP_ATOMIC); - if (!child_listen_node) - return; - node_allocated = true; - - memcpy(child_listen_node->loc_addr, ipaddr, ipv4 ? 4 : 16); - -set_qhash: - memcpy(nfo->loc_addr, - child_listen_node->loc_addr, - sizeof(nfo->loc_addr)); - nfo->vlan_id = child_listen_node->vlan_id; - ret = i40iw_manage_qhash(iwdev, nfo, - I40IW_QHASH_TYPE_TCP_SYN, - op, - NULL, false); - if (!ret) { - child_listen_node->qhash_set = ifup; - if (node_allocated) - list_add(&child_listen_node->child_listen_list, - &parent_listen_node->child_listen_list); - } else if (node_allocated) { - kfree(child_listen_node); - } -} - -/** - * i40iw_cm_teardown_connections - teardown QPs - * @iwdev: device pointer - * @ipaddr: Pointer to IPv4 or IPv6 address - * @nfo: cm info node - * @disconnect_all: flag indicating disconnect all QPs - * teardown QPs where source or destination addr matches ip addr - */ -void i40iw_cm_teardown_connections(struct i40iw_device *iwdev, u32 *ipaddr, - struct i40iw_cm_info *nfo, - bool disconnect_all) -{ - struct i40iw_cm_core *cm_core = &iwdev->cm_core; - struct list_head *list_core_temp; - struct list_head *list_node; - struct i40iw_cm_node *cm_node; - unsigned long flags; - struct list_head teardown_list; - struct ib_qp_attr attr; - - INIT_LIST_HEAD(&teardown_list); - spin_lock_irqsave(&cm_core->ht_lock, flags); - list_for_each_safe(list_node, list_core_temp, - &cm_core->accelerated_list) { - cm_node = container_of(list_node, struct i40iw_cm_node, list); - if (disconnect_all || - (nfo->vlan_id == cm_node->vlan_id && - (!memcmp(cm_node->loc_addr, ipaddr, nfo->ipv4 ? 4 : 16) || - !memcmp(cm_node->rem_addr, ipaddr, nfo->ipv4 ? 4 : 16)))) { - atomic_inc(&cm_node->ref_count); - list_add(&cm_node->teardown_entry, &teardown_list); - } - } - list_for_each_safe(list_node, list_core_temp, - &cm_core->non_accelerated_list) { - cm_node = container_of(list_node, struct i40iw_cm_node, list); - if (disconnect_all || - (nfo->vlan_id == cm_node->vlan_id && - (!memcmp(cm_node->loc_addr, ipaddr, nfo->ipv4 ? 4 : 16) || - !memcmp(cm_node->rem_addr, ipaddr, nfo->ipv4 ? 4 : 16)))) { - atomic_inc(&cm_node->ref_count); - list_add(&cm_node->teardown_entry, &teardown_list); - } - } - spin_unlock_irqrestore(&cm_core->ht_lock, flags); - - list_for_each_safe(list_node, list_core_temp, &teardown_list) { - cm_node = container_of(list_node, struct i40iw_cm_node, - teardown_entry); - attr.qp_state = IB_QPS_ERR; - i40iw_modify_qp(&cm_node->iwqp->ibqp, &attr, IB_QP_STATE, NULL); - if (iwdev->reset) - i40iw_cm_disconn(cm_node->iwqp); - i40iw_rem_ref_cm_node(cm_node); - } -} - -/** - * i40iw_if_notify - process an ifdown on an interface - * @iwdev: device pointer - * @netdev: network interface device structure - * @ipaddr: Pointer to IPv4 or IPv6 address - * @ipv4: flag indicating IPv4 when true - * @ifup: flag indicating interface up when true - */ -void i40iw_if_notify(struct i40iw_device *iwdev, struct net_device *netdev, - u32 *ipaddr, bool ipv4, bool ifup) -{ - struct i40iw_cm_core *cm_core = &iwdev->cm_core; - unsigned long flags; - struct i40iw_cm_listener *listen_node; - static const u32 ip_zero[4] = { 0, 0, 0, 0 }; - struct i40iw_cm_info nfo; - u16 vlan_id = rdma_vlan_dev_vlan_id(netdev); - enum i40iw_status_code ret; - enum i40iw_quad_hash_manage_type op = - ifup ? I40IW_QHASH_MANAGE_TYPE_ADD : I40IW_QHASH_MANAGE_TYPE_DELETE; - - nfo.vlan_id = vlan_id; - nfo.ipv4 = ipv4; - - /* Disable or enable qhash for listeners */ - spin_lock_irqsave(&cm_core->listen_list_lock, flags); - list_for_each_entry(listen_node, &cm_core->listen_nodes, list) { - if (vlan_id == listen_node->vlan_id && - (!memcmp(listen_node->loc_addr, ipaddr, ipv4 ? 4 : 16) || - !memcmp(listen_node->loc_addr, ip_zero, ipv4 ? 4 : 16))) { - memcpy(nfo.loc_addr, listen_node->loc_addr, - sizeof(nfo.loc_addr)); - nfo.loc_port = listen_node->loc_port; - nfo.user_pri = listen_node->user_pri; - if (!list_empty(&listen_node->child_listen_list)) { - i40iw_qhash_ctrl(iwdev, - listen_node, - &nfo, - ipaddr, ipv4, ifup); - } else if (memcmp(listen_node->loc_addr, ip_zero, - ipv4 ? 4 : 16)) { - ret = i40iw_manage_qhash(iwdev, - &nfo, - I40IW_QHASH_TYPE_TCP_SYN, - op, - NULL, - false); - if (!ret) - listen_node->qhash_set = ifup; - } - } - } - spin_unlock_irqrestore(&cm_core->listen_list_lock, flags); - - /* teardown connected qp's on ifdown */ - if (!ifup) - i40iw_cm_teardown_connections(iwdev, ipaddr, &nfo, false); -} diff --git a/drivers/infiniband/hw/i40iw/i40iw_cm.h b/drivers/infiniband/hw/i40iw/i40iw_cm.h deleted file mode 100644 index 6e43e4d730f4..000000000000 --- a/drivers/infiniband/hw/i40iw/i40iw_cm.h +++ /dev/null @@ -1,462 +0,0 @@ -/******************************************************************************* -* -* Copyright (c) 2015-2016 Intel Corporation. All rights reserved. -* -* This software is available to you under a choice of one of two -* licenses. You may choose to be licensed under the terms of the GNU -* General Public License (GPL) Version 2, available from the file -* COPYING in the main directory of this source tree, or the -* OpenFabrics.org BSD license below: -* -* Redistribution and use in source and binary forms, with or -* without modification, are permitted provided that the following -* conditions are met: -* -* - Redistributions of source code must retain the above -* copyright notice, this list of conditions and the following -* disclaimer. -* -* - Redistributions in binary form must reproduce the above -* copyright notice, this list of conditions and the following -* disclaimer in the documentation and/or other materials -* provided with the distribution. -* -* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS -* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN -* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -* SOFTWARE. -* -*******************************************************************************/ - -#ifndef I40IW_CM_H -#define I40IW_CM_H - -#define QUEUE_EVENTS - -#define I40IW_MANAGE_APBVT_DEL 0 -#define I40IW_MANAGE_APBVT_ADD 1 - -#define I40IW_MPA_REQUEST_ACCEPT 1 -#define I40IW_MPA_REQUEST_REJECT 2 - -/* IETF MPA -- defines, enums, structs */ -#define IEFT_MPA_KEY_REQ "MPA ID Req Frame" -#define IEFT_MPA_KEY_REP "MPA ID Rep Frame" -#define IETF_MPA_KEY_SIZE 16 -#define IETF_MPA_VERSION 1 -#define IETF_MAX_PRIV_DATA_LEN 512 -#define IETF_MPA_FRAME_SIZE 20 -#define IETF_RTR_MSG_SIZE 4 -#define IETF_MPA_V2_FLAG 0x10 -#define SNDMARKER_SEQNMASK 0x000001FF - -#define I40IW_MAX_IETF_SIZE 32 - -/* IETF RTR MSG Fields */ -#define IETF_PEER_TO_PEER 0x8000 -#define IETF_FLPDU_ZERO_LEN 0x4000 -#define IETF_RDMA0_WRITE 0x8000 -#define IETF_RDMA0_READ 0x4000 -#define IETF_NO_IRD_ORD 0x3FFF - -/* HW-supported IRD sizes*/ -#define I40IW_HW_IRD_SETTING_2 2 -#define I40IW_HW_IRD_SETTING_4 4 -#define I40IW_HW_IRD_SETTING_8 8 -#define I40IW_HW_IRD_SETTING_16 16 -#define I40IW_HW_IRD_SETTING_32 32 -#define I40IW_HW_IRD_SETTING_64 64 - -#define MAX_PORTS 65536 -#define I40IW_VLAN_PRIO_SHIFT 13 - -enum ietf_mpa_flags { - IETF_MPA_FLAGS_MARKERS = 0x80, /* receive Markers */ - IETF_MPA_FLAGS_CRC = 0x40, /* receive Markers */ - IETF_MPA_FLAGS_REJECT = 0x20, /* Reject */ -}; - -struct ietf_mpa_v1 { - u8 key[IETF_MPA_KEY_SIZE]; - u8 flags; - u8 rev; - __be16 priv_data_len; - u8 priv_data[]; -}; - -#define ietf_mpa_req_resp_frame ietf_mpa_frame - -struct ietf_rtr_msg { - __be16 ctrl_ird; - __be16 ctrl_ord; -}; - -struct ietf_mpa_v2 { - u8 key[IETF_MPA_KEY_SIZE]; - u8 flags; - u8 rev; - __be16 priv_data_len; - struct ietf_rtr_msg rtr_msg; - u8 priv_data[]; -}; - -struct i40iw_cm_node; -enum i40iw_timer_type { - I40IW_TIMER_TYPE_SEND, - I40IW_TIMER_TYPE_RECV, - I40IW_TIMER_NODE_CLEANUP, - I40IW_TIMER_TYPE_CLOSE, -}; - -#define I40IW_PASSIVE_STATE_INDICATED 0 -#define I40IW_DO_NOT_SEND_RESET_EVENT 1 -#define I40IW_SEND_RESET_EVENT 2 - -#define MAX_I40IW_IFS 4 - -#define SET_ACK 0x1 -#define SET_SYN 0x2 -#define SET_FIN 0x4 -#define SET_RST 0x8 - -#define TCP_OPTIONS_PADDING 3 - -struct option_base { - u8 optionnum; - u8 length; -}; - -enum option_numbers { - OPTION_NUMBER_END, - OPTION_NUMBER_NONE, - OPTION_NUMBER_MSS, - OPTION_NUMBER_WINDOW_SCALE, - OPTION_NUMBER_SACK_PERM, - OPTION_NUMBER_SACK, - OPTION_NUMBER_WRITE0 = 0xbc -}; - -struct option_mss { - u8 optionnum; - u8 length; - __be16 mss; -}; - -struct option_windowscale { - u8 optionnum; - u8 length; - u8 shiftcount; -}; - -union all_known_options { - char as_end; - struct option_base as_base; - struct option_mss as_mss; - struct option_windowscale as_windowscale; -}; - -struct i40iw_timer_entry { - struct list_head list; - unsigned long timetosend; /* jiffies */ - struct i40iw_puda_buf *sqbuf; - u32 type; - u32 retrycount; - u32 retranscount; - u32 context; - u32 send_retrans; - int close_when_complete; -}; - -#define I40IW_DEFAULT_RETRYS 64 -#define I40IW_DEFAULT_RETRANS 8 -#define I40IW_DEFAULT_TTL 0x40 -#define I40IW_DEFAULT_RTT_VAR 0x6 -#define I40IW_DEFAULT_SS_THRESH 0x3FFFFFFF -#define I40IW_DEFAULT_REXMIT_THRESH 8 - -#define I40IW_RETRY_TIMEOUT HZ -#define I40IW_SHORT_TIME 10 -#define I40IW_LONG_TIME (2 * HZ) -#define I40IW_MAX_TIMEOUT ((unsigned long)(12 * HZ)) - -#define I40IW_CM_HASHTABLE_SIZE 1024 -#define I40IW_CM_TCP_TIMER_INTERVAL 3000 -#define I40IW_CM_DEFAULT_MTU 1540 -#define I40IW_CM_DEFAULT_FRAME_CNT 10 -#define I40IW_CM_THREAD_STACK_SIZE 256 -#define I40IW_CM_DEFAULT_RCV_WND 64240 -#define I40IW_CM_DEFAULT_RCV_WND_SCALED 0x3fffc -#define I40IW_CM_DEFAULT_RCV_WND_SCALE 2 -#define I40IW_CM_DEFAULT_FREE_PKTS 0x000A -#define I40IW_CM_FREE_PKT_LO_WATERMARK 2 - -#define I40IW_CM_DEFAULT_MSS 536 - -#define I40IW_CM_DEF_SEQ 0x159bf75f -#define I40IW_CM_DEF_LOCAL_ID 0x3b47 - -#define I40IW_CM_DEF_SEQ2 0x18ed5740 -#define I40IW_CM_DEF_LOCAL_ID2 0xb807 -#define MAX_CM_BUFFER (I40IW_MAX_IETF_SIZE + IETF_MAX_PRIV_DATA_LEN) - -typedef u32 i40iw_addr_t; - -#define i40iw_cm_tsa_context i40iw_qp_context - -struct i40iw_qp; - -/* cm node transition states */ -enum i40iw_cm_node_state { - I40IW_CM_STATE_UNKNOWN, - I40IW_CM_STATE_INITED, - I40IW_CM_STATE_LISTENING, - I40IW_CM_STATE_SYN_RCVD, - I40IW_CM_STATE_SYN_SENT, - I40IW_CM_STATE_ONE_SIDE_ESTABLISHED, - I40IW_CM_STATE_ESTABLISHED, - I40IW_CM_STATE_ACCEPTING, - I40IW_CM_STATE_MPAREQ_SENT, - I40IW_CM_STATE_MPAREQ_RCVD, - I40IW_CM_STATE_MPAREJ_RCVD, - I40IW_CM_STATE_OFFLOADED, - I40IW_CM_STATE_FIN_WAIT1, - I40IW_CM_STATE_FIN_WAIT2, - I40IW_CM_STATE_CLOSE_WAIT, - I40IW_CM_STATE_TIME_WAIT, - I40IW_CM_STATE_LAST_ACK, - I40IW_CM_STATE_CLOSING, - I40IW_CM_STATE_LISTENER_DESTROYED, - I40IW_CM_STATE_CLOSED -}; - -enum mpa_frame_version { - IETF_MPA_V1 = 1, - IETF_MPA_V2 = 2 -}; - -enum mpa_frame_key { - MPA_KEY_REQUEST, - MPA_KEY_REPLY -}; - -enum send_rdma0 { - SEND_RDMA_READ_ZERO = 1, - SEND_RDMA_WRITE_ZERO = 2 -}; - -enum i40iw_tcpip_pkt_type { - I40IW_PKT_TYPE_UNKNOWN, - I40IW_PKT_TYPE_SYN, - I40IW_PKT_TYPE_SYNACK, - I40IW_PKT_TYPE_ACK, - I40IW_PKT_TYPE_FIN, - I40IW_PKT_TYPE_RST -}; - -/* CM context params */ -struct i40iw_cm_tcp_context { - u8 client; - - u32 loc_seq_num; - u32 loc_ack_num; - u32 rem_ack_num; - u32 rcv_nxt; - - u32 loc_id; - u32 rem_id; - - u32 snd_wnd; - u32 max_snd_wnd; - - u32 rcv_wnd; - u32 mss; - u8 snd_wscale; - u8 rcv_wscale; -}; - -enum i40iw_cm_listener_state { - I40IW_CM_LISTENER_PASSIVE_STATE = 1, - I40IW_CM_LISTENER_ACTIVE_STATE = 2, - I40IW_CM_LISTENER_EITHER_STATE = 3 -}; - -struct i40iw_cm_listener { - struct list_head list; - struct i40iw_cm_core *cm_core; - u8 loc_mac[ETH_ALEN]; - u32 loc_addr[4]; - u16 loc_port; - struct iw_cm_id *cm_id; - atomic_t ref_count; - struct i40iw_device *iwdev; - atomic_t pend_accepts_cnt; - int backlog; - enum i40iw_cm_listener_state listener_state; - u32 reused_node; - u8 user_pri; - u8 tos; - u16 vlan_id; - bool qhash_set; - bool ipv4; - struct list_head child_listen_list; - -}; - -struct i40iw_kmem_info { - void *addr; - u32 size; -}; - -/* per connection node and node state information */ -struct i40iw_cm_node { - u32 loc_addr[4], rem_addr[4]; - u16 loc_port, rem_port; - u16 vlan_id; - enum i40iw_cm_node_state state; - u8 loc_mac[ETH_ALEN]; - u8 rem_mac[ETH_ALEN]; - atomic_t ref_count; - struct i40iw_qp *iwqp; - struct i40iw_device *iwdev; - struct i40iw_sc_dev *dev; - struct i40iw_cm_tcp_context tcp_cntxt; - struct i40iw_cm_core *cm_core; - struct i40iw_cm_node *loopbackpartner; - struct i40iw_timer_entry *send_entry; - struct i40iw_timer_entry *close_entry; - spinlock_t retrans_list_lock; /* cm transmit packet */ - enum send_rdma0 send_rdma0_op; - u16 ird_size; - u16 ord_size; - u16 mpav2_ird_ord; - struct iw_cm_id *cm_id; - struct list_head list; - bool accelerated; - struct i40iw_cm_listener *listener; - int apbvt_set; - int accept_pend; - struct list_head timer_entry; - struct list_head reset_entry; - struct list_head teardown_entry; - atomic_t passive_state; - bool qhash_set; - u8 user_pri; - u8 tos; - bool ipv4; - bool snd_mark_en; - u16 lsmm_size; - enum mpa_frame_version mpa_frame_rev; - struct i40iw_kmem_info pdata; - union { - struct ietf_mpa_v1 mpa_frame; - struct ietf_mpa_v2 mpa_v2_frame; - }; - - u8 pdata_buf[IETF_MAX_PRIV_DATA_LEN]; - struct i40iw_kmem_info mpa_hdr; - bool ack_rcvd; -}; - -/* structure for client or CM to fill when making CM api calls. */ -/* - only need to set relevant data, based on op. */ -struct i40iw_cm_info { - struct iw_cm_id *cm_id; - u16 loc_port; - u16 rem_port; - u32 loc_addr[4]; - u32 rem_addr[4]; - u16 vlan_id; - int backlog; - u8 user_pri; - u8 tos; - bool ipv4; -}; - -/* CM event codes */ -enum i40iw_cm_event_type { - I40IW_CM_EVENT_UNKNOWN, - I40IW_CM_EVENT_ESTABLISHED, - I40IW_CM_EVENT_MPA_REQ, - I40IW_CM_EVENT_MPA_CONNECT, - I40IW_CM_EVENT_MPA_ACCEPT, - I40IW_CM_EVENT_MPA_REJECT, - I40IW_CM_EVENT_MPA_ESTABLISHED, - I40IW_CM_EVENT_CONNECTED, - I40IW_CM_EVENT_RESET, - I40IW_CM_EVENT_ABORTED -}; - -/* event to post to CM event handler */ -struct i40iw_cm_event { - enum i40iw_cm_event_type type; - struct i40iw_cm_info cm_info; - struct work_struct event_work; - struct i40iw_cm_node *cm_node; -}; - -struct i40iw_cm_core { - struct i40iw_device *iwdev; - struct i40iw_sc_dev *dev; - - struct list_head listen_nodes; - struct list_head accelerated_list; - struct list_head non_accelerated_list; - - struct timer_list tcp_timer; - - struct workqueue_struct *event_wq; - struct workqueue_struct *disconn_wq; - - spinlock_t ht_lock; /* manage hash table */ - spinlock_t listen_list_lock; /* listen list */ - spinlock_t apbvt_lock; /*manage apbvt entries*/ - - unsigned long ports_in_use[BITS_TO_LONGS(MAX_PORTS)]; - - u64 stats_nodes_created; - u64 stats_nodes_destroyed; - u64 stats_listen_created; - u64 stats_listen_destroyed; - u64 stats_listen_nodes_created; - u64 stats_listen_nodes_destroyed; - u64 stats_loopbacks; - u64 stats_accepts; - u64 stats_rejects; - u64 stats_connect_errs; - u64 stats_passive_errs; - u64 stats_pkt_retrans; - u64 stats_backlog_drops; -}; - -int i40iw_schedule_cm_timer(struct i40iw_cm_node *cm_node, - struct i40iw_puda_buf *sqbuf, - enum i40iw_timer_type type, - int send_retrans, - int close_when_complete); - -int i40iw_accept(struct iw_cm_id *, struct iw_cm_conn_param *); -int i40iw_reject(struct iw_cm_id *, const void *, u8); -int i40iw_connect(struct iw_cm_id *, struct iw_cm_conn_param *); -int i40iw_create_listen(struct iw_cm_id *, int); -int i40iw_destroy_listen(struct iw_cm_id *); - -int i40iw_cm_start(struct i40iw_device *); -int i40iw_cm_stop(struct i40iw_device *); - -int i40iw_arp_table(struct i40iw_device *iwdev, - u32 *ip_addr, - bool ipv4, - u8 *mac_addr, - u32 action); - -void i40iw_if_notify(struct i40iw_device *iwdev, struct net_device *netdev, - u32 *ipaddr, bool ipv4, bool ifup); -void i40iw_cm_teardown_connections(struct i40iw_device *iwdev, u32 *ipaddr, - struct i40iw_cm_info *nfo, - bool disconnect_all); -bool i40iw_port_in_use(struct i40iw_cm_core *cm_core, u16 port); -#endif /* I40IW_CM_H */ diff --git a/drivers/infiniband/hw/i40iw/i40iw_ctrl.c b/drivers/infiniband/hw/i40iw/i40iw_ctrl.c deleted file mode 100644 index eaea5d545eb8..000000000000 --- a/drivers/infiniband/hw/i40iw/i40iw_ctrl.c +++ /dev/null @@ -1,5243 +0,0 @@ -/******************************************************************************* -* -* Copyright (c) 2015-2016 Intel Corporation. All rights reserved. -* -* This software is available to you under a choice of one of two -* licenses. You may choose to be licensed under the terms of the GNU -* General Public License (GPL) Version 2, available from the file -* COPYING in the main directory of this source tree, or the -* OpenFabrics.org BSD license below: -* -* Redistribution and use in source and binary forms, with or -* without modification, are permitted provided that the following -* conditions are met: -* -* - Redistributions of source code must retain the above -* copyright notice, this list of conditions and the following -* disclaimer. -* -* - Redistributions in binary form must reproduce the above -* copyright notice, this list of conditions and the following -* disclaimer in the documentation and/or other materials -* provided with the distribution. -* -* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS -* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN -* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -* SOFTWARE. -* -*******************************************************************************/ - -#include "i40iw_osdep.h" -#include "i40iw_register.h" -#include "i40iw_status.h" -#include "i40iw_hmc.h" - -#include "i40iw_d.h" -#include "i40iw_type.h" -#include "i40iw_p.h" -#include "i40iw_vf.h" -#include "i40iw_virtchnl.h" - -/** - * i40iw_insert_wqe_hdr - write wqe header - * @wqe: cqp wqe for header - * @header: header for the cqp wqe - */ -void i40iw_insert_wqe_hdr(u64 *wqe, u64 header) -{ - wmb(); /* make sure WQE is populated before polarity is set */ - set_64bit_val(wqe, 24, header); -} - -void i40iw_check_cqp_progress(struct i40iw_cqp_timeout *cqp_timeout, struct i40iw_sc_dev *dev) -{ - if (cqp_timeout->compl_cqp_cmds != dev->cqp_cmd_stats[OP_COMPLETED_COMMANDS]) { - cqp_timeout->compl_cqp_cmds = dev->cqp_cmd_stats[OP_COMPLETED_COMMANDS]; - cqp_timeout->count = 0; - } else { - if (dev->cqp_cmd_stats[OP_REQUESTED_COMMANDS] != cqp_timeout->compl_cqp_cmds) - cqp_timeout->count++; - } -} - -/** - * i40iw_get_cqp_reg_info - get head and tail for cqp using registers - * @cqp: struct for cqp hw - * @val: cqp tail register value - * @tail:wqtail register value - * @error: cqp processing err - */ -static inline void i40iw_get_cqp_reg_info(struct i40iw_sc_cqp *cqp, - u32 *val, - u32 *tail, - u32 *error) -{ - if (cqp->dev->is_pf) { - *val = i40iw_rd32(cqp->dev->hw, I40E_PFPE_CQPTAIL); - *tail = RS_32(*val, I40E_PFPE_CQPTAIL_WQTAIL); - *error = RS_32(*val, I40E_PFPE_CQPTAIL_CQP_OP_ERR); - } else { - *val = i40iw_rd32(cqp->dev->hw, I40E_VFPE_CQPTAIL1); - *tail = RS_32(*val, I40E_VFPE_CQPTAIL_WQTAIL); - *error = RS_32(*val, I40E_VFPE_CQPTAIL_CQP_OP_ERR); - } -} - -/** - * i40iw_cqp_poll_registers - poll cqp registers - * @cqp: struct for cqp hw - * @tail:wqtail register value - * @count: how many times to try for completion - */ -static enum i40iw_status_code i40iw_cqp_poll_registers( - struct i40iw_sc_cqp *cqp, - u32 tail, - u32 count) -{ - u32 i = 0; - u32 newtail, error, val; - - while (i < count) { - i++; - i40iw_get_cqp_reg_info(cqp, &val, &newtail, &error); - if (error) { - error = (cqp->dev->is_pf) ? - i40iw_rd32(cqp->dev->hw, I40E_PFPE_CQPERRCODES) : - i40iw_rd32(cqp->dev->hw, I40E_VFPE_CQPERRCODES1); - return I40IW_ERR_CQP_COMPL_ERROR; - } - if (newtail != tail) { - /* SUCCESS */ - I40IW_RING_MOVE_TAIL(cqp->sq_ring); - cqp->dev->cqp_cmd_stats[OP_COMPLETED_COMMANDS]++; - return 0; - } - udelay(I40IW_SLEEP_COUNT); - } - return I40IW_ERR_TIMEOUT; -} - -/** - * i40iw_sc_parse_fpm_commit_buf - parse fpm commit buffer - * @buf: ptr to fpm commit buffer - * @info: ptr to i40iw_hmc_obj_info struct - * @sd: number of SDs for HMC objects - * - * parses fpm commit info and copy base value - * of hmc objects in hmc_info - */ -static enum i40iw_status_code i40iw_sc_parse_fpm_commit_buf( - u64 *buf, - struct i40iw_hmc_obj_info *info, - u32 *sd) -{ - u64 temp; - u64 size; - u64 base = 0; - u32 i, j; - u32 k = 0; - - /* copy base values in obj_info */ - for (i = I40IW_HMC_IW_QP, j = 0; i <= I40IW_HMC_IW_PBLE; i++, j += 8) { - if ((i == I40IW_HMC_IW_SRQ) || - (i == I40IW_HMC_IW_FSIMC) || - (i == I40IW_HMC_IW_FSIAV)) { - info[i].base = 0; - info[i].cnt = 0; - continue; - } - get_64bit_val(buf, j, &temp); - info[i].base = RS_64_1(temp, 32) * 512; - if (info[i].base > base) { - base = info[i].base; - k = i; - } - if (i == I40IW_HMC_IW_APBVT_ENTRY) { - info[i].cnt = 1; - continue; - } - if (i == I40IW_HMC_IW_QP) - info[i].cnt = (u32)RS_64(temp, I40IW_QUERY_FPM_MAX_QPS); - else if (i == I40IW_HMC_IW_CQ) - info[i].cnt = (u32)RS_64(temp, I40IW_QUERY_FPM_MAX_CQS); - else - info[i].cnt = (u32)(temp); - } - size = info[k].cnt * info[k].size + info[k].base; - if (size & 0x1FFFFF) - *sd = (u32)((size >> 21) + 1); /* add 1 for remainder */ - else - *sd = (u32)(size >> 21); - - return 0; -} - -/** - * i40iw_sc_decode_fpm_query() - Decode a 64 bit value into max count and size - * @buf: ptr to fpm query buffer - * @buf_idx: index into buf - * @obj_info: ptr to i40iw_hmc_obj_info struct - * @rsrc_idx: resource index into info - * - * Decode a 64 bit value from fpm query buffer into max count and size - */ -static u64 i40iw_sc_decode_fpm_query(u64 *buf, - u32 buf_idx, - struct i40iw_hmc_obj_info *obj_info, - u32 rsrc_idx) -{ - u64 temp; - u32 size; - - get_64bit_val(buf, buf_idx, &temp); - obj_info[rsrc_idx].max_cnt = (u32)temp; - size = (u32)RS_64_1(temp, 32); - obj_info[rsrc_idx].size = LS_64_1(1, size); - - return temp; -} - -/** - * i40iw_sc_parse_fpm_query_buf() - parses fpm query buffer - * @buf: ptr to fpm query buffer - * @hmc_info: ptr to i40iw_hmc_obj_info struct - * @hmc_fpm_misc: ptr to fpm data - * - * parses fpm query buffer and copy max_cnt and - * size value of hmc objects in hmc_info - */ -static enum i40iw_status_code i40iw_sc_parse_fpm_query_buf( - u64 *buf, - struct i40iw_hmc_info *hmc_info, - struct i40iw_hmc_fpm_misc *hmc_fpm_misc) -{ - struct i40iw_hmc_obj_info *obj_info; - u64 temp; - u32 size; - u16 max_pe_sds; - - obj_info = hmc_info->hmc_obj; - - get_64bit_val(buf, 0, &temp); - hmc_info->first_sd_index = (u16)RS_64(temp, I40IW_QUERY_FPM_FIRST_PE_SD_INDEX); - max_pe_sds = (u16)RS_64(temp, I40IW_QUERY_FPM_MAX_PE_SDS); - - /* Reduce SD count for VFs by 1 to account for PBLE backing page rounding */ - if (hmc_info->hmc_fn_id >= I40IW_FIRST_VF_FPM_ID) - max_pe_sds--; - hmc_fpm_misc->max_sds = max_pe_sds; - hmc_info->sd_table.sd_cnt = max_pe_sds + hmc_info->first_sd_index; - - get_64bit_val(buf, 8, &temp); - obj_info[I40IW_HMC_IW_QP].max_cnt = (u32)RS_64(temp, I40IW_QUERY_FPM_MAX_QPS); - size = (u32)RS_64_1(temp, 32); - obj_info[I40IW_HMC_IW_QP].size = LS_64_1(1, size); - - get_64bit_val(buf, 16, &temp); - obj_info[I40IW_HMC_IW_CQ].max_cnt = (u32)RS_64(temp, I40IW_QUERY_FPM_MAX_CQS); - size = (u32)RS_64_1(temp, 32); - obj_info[I40IW_HMC_IW_CQ].size = LS_64_1(1, size); - - i40iw_sc_decode_fpm_query(buf, 32, obj_info, I40IW_HMC_IW_HTE); - i40iw_sc_decode_fpm_query(buf, 40, obj_info, I40IW_HMC_IW_ARP); - - obj_info[I40IW_HMC_IW_APBVT_ENTRY].size = 8192; - obj_info[I40IW_HMC_IW_APBVT_ENTRY].max_cnt = 1; - - i40iw_sc_decode_fpm_query(buf, 48, obj_info, I40IW_HMC_IW_MR); - i40iw_sc_decode_fpm_query(buf, 56, obj_info, I40IW_HMC_IW_XF); - - get_64bit_val(buf, 64, &temp); - obj_info[I40IW_HMC_IW_XFFL].max_cnt = (u32)temp; - obj_info[I40IW_HMC_IW_XFFL].size = 4; - hmc_fpm_misc->xf_block_size = RS_64(temp, I40IW_QUERY_FPM_XFBLOCKSIZE); - if (!hmc_fpm_misc->xf_block_size) - return I40IW_ERR_INVALID_SIZE; - - i40iw_sc_decode_fpm_query(buf, 72, obj_info, I40IW_HMC_IW_Q1); - - get_64bit_val(buf, 80, &temp); - obj_info[I40IW_HMC_IW_Q1FL].max_cnt = (u32)temp; - obj_info[I40IW_HMC_IW_Q1FL].size = 4; - hmc_fpm_misc->q1_block_size = RS_64(temp, I40IW_QUERY_FPM_Q1BLOCKSIZE); - if (!hmc_fpm_misc->q1_block_size) - return I40IW_ERR_INVALID_SIZE; - - i40iw_sc_decode_fpm_query(buf, 88, obj_info, I40IW_HMC_IW_TIMER); - - get_64bit_val(buf, 112, &temp); - obj_info[I40IW_HMC_IW_PBLE].max_cnt = (u32)temp; - obj_info[I40IW_HMC_IW_PBLE].size = 8; - - get_64bit_val(buf, 120, &temp); - hmc_fpm_misc->max_ceqs = (u8)RS_64(temp, I40IW_QUERY_FPM_MAX_CEQS); - hmc_fpm_misc->ht_multiplier = RS_64(temp, I40IW_QUERY_FPM_HTMULTIPLIER); - hmc_fpm_misc->timer_bucket = RS_64(temp, I40IW_QUERY_FPM_TIMERBUCKET); - - return 0; -} - -/** - * i40iw_fill_qos_list - Change all unknown qs handles to available ones - * @qs_list: list of qs_handles to be fixed with valid qs_handles - */ -static void i40iw_fill_qos_list(u16 *qs_list) -{ - u16 qshandle = qs_list[0]; - int i; - - for (i = 0; i < I40IW_MAX_USER_PRIORITY; i++) { - if (qs_list[i] == QS_HANDLE_UNKNOWN) - qs_list[i] = qshandle; - else - qshandle = qs_list[i]; - } -} - -/** - * i40iw_qp_from_entry - Given entry, get to the qp structure - * @entry: Points to list of qp structure - */ -static struct i40iw_sc_qp *i40iw_qp_from_entry(struct list_head *entry) -{ - if (!entry) - return NULL; - - return (struct i40iw_sc_qp *)((char *)entry - offsetof(struct i40iw_sc_qp, list)); -} - -/** - * i40iw_get_qp - get the next qp from the list given current qp - * @head: Listhead of qp's - * @qp: current qp - */ -static struct i40iw_sc_qp *i40iw_get_qp(struct list_head *head, struct i40iw_sc_qp *qp) -{ - struct list_head *entry = NULL; - struct list_head *lastentry; - - if (list_empty(head)) - return NULL; - - if (!qp) { - entry = head->next; - } else { - lastentry = &qp->list; - entry = (lastentry != head) ? lastentry->next : NULL; - } - - return i40iw_qp_from_entry(entry); -} - -/** - * i40iw_change_l2params - given the new l2 parameters, change all qp - * @vsi: pointer to the vsi structure - * @l2params: New paramaters from l2 - */ -void i40iw_change_l2params(struct i40iw_sc_vsi *vsi, struct i40iw_l2params *l2params) -{ - struct i40iw_sc_dev *dev = vsi->dev; - struct i40iw_sc_qp *qp = NULL; - bool qs_handle_change = false; - unsigned long flags; - u16 qs_handle; - int i; - - if (vsi->mtu != l2params->mtu) { - vsi->mtu = l2params->mtu; - i40iw_reinitialize_ieq(dev); - } - - i40iw_fill_qos_list(l2params->qs_handle_list); - for (i = 0; i < I40IW_MAX_USER_PRIORITY; i++) { - qs_handle = l2params->qs_handle_list[i]; - if (vsi->qos[i].qs_handle != qs_handle) - qs_handle_change = true; - spin_lock_irqsave(&vsi->qos[i].lock, flags); - qp = i40iw_get_qp(&vsi->qos[i].qplist, qp); - while (qp) { - if (qs_handle_change) { - qp->qs_handle = qs_handle; - /* issue cqp suspend command */ - i40iw_qp_suspend_resume(dev, qp, true); - } - qp = i40iw_get_qp(&vsi->qos[i].qplist, qp); - } - spin_unlock_irqrestore(&vsi->qos[i].lock, flags); - vsi->qos[i].qs_handle = qs_handle; - } -} - -/** - * i40iw_qp_rem_qos - remove qp from qos lists during destroy qp - * @qp: qp to be removed from qos - */ -void i40iw_qp_rem_qos(struct i40iw_sc_qp *qp) -{ - struct i40iw_sc_vsi *vsi = qp->vsi; - unsigned long flags; - - if (!qp->on_qoslist) - return; - spin_lock_irqsave(&vsi->qos[qp->user_pri].lock, flags); - list_del(&qp->list); - spin_unlock_irqrestore(&vsi->qos[qp->user_pri].lock, flags); -} - -/** - * i40iw_qp_add_qos - called during setctx fot qp to be added to qos - * @qp: qp to be added to qos - */ -void i40iw_qp_add_qos(struct i40iw_sc_qp *qp) -{ - struct i40iw_sc_vsi *vsi = qp->vsi; - unsigned long flags; - - if (qp->on_qoslist) - return; - spin_lock_irqsave(&vsi->qos[qp->user_pri].lock, flags); - qp->qs_handle = vsi->qos[qp->user_pri].qs_handle; - list_add(&qp->list, &vsi->qos[qp->user_pri].qplist); - qp->on_qoslist = true; - spin_unlock_irqrestore(&vsi->qos[qp->user_pri].lock, flags); -} - -/** - * i40iw_sc_pd_init - initialize sc pd struct - * @dev: sc device struct - * @pd: sc pd ptr - * @pd_id: pd_id for allocated pd - * @abi_ver: ABI version from user context, -1 if not valid - */ -static void i40iw_sc_pd_init(struct i40iw_sc_dev *dev, - struct i40iw_sc_pd *pd, - u16 pd_id, - int abi_ver) -{ - pd->size = sizeof(*pd); - pd->pd_id = pd_id; - pd->abi_ver = abi_ver; - pd->dev = dev; -} - -/** - * i40iw_get_encoded_wqe_size - given wq size, returns hardware encoded size - * @wqsize: size of the wq (sq, rq, srq) to encoded_size - * @cqpsq: encoded size for sq for cqp as its encoded size is 1+ other wq's - */ -u8 i40iw_get_encoded_wqe_size(u32 wqsize, bool cqpsq) -{ - u8 encoded_size = 0; - - /* cqp sq's hw coded value starts from 1 for size of 4 - * while it starts from 0 for qp' wq's. - */ - if (cqpsq) - encoded_size = 1; - wqsize >>= 2; - while (wqsize >>= 1) - encoded_size++; - return encoded_size; -} - -/** - * i40iw_sc_cqp_init - Initialize buffers for a control Queue Pair - * @cqp: IWARP control queue pair pointer - * @info: IWARP control queue pair init info pointer - * - * Initializes the object and context buffers for a control Queue Pair. - */ -static enum i40iw_status_code i40iw_sc_cqp_init(struct i40iw_sc_cqp *cqp, - struct i40iw_cqp_init_info *info) -{ - u8 hw_sq_size; - - if ((info->sq_size > I40IW_CQP_SW_SQSIZE_2048) || - (info->sq_size < I40IW_CQP_SW_SQSIZE_4) || - ((info->sq_size & (info->sq_size - 1)))) - return I40IW_ERR_INVALID_SIZE; - - hw_sq_size = i40iw_get_encoded_wqe_size(info->sq_size, true); - cqp->size = sizeof(*cqp); - cqp->sq_size = info->sq_size; - cqp->hw_sq_size = hw_sq_size; - cqp->sq_base = info->sq; - cqp->host_ctx = info->host_ctx; - cqp->sq_pa = info->sq_pa; - cqp->host_ctx_pa = info->host_ctx_pa; - cqp->dev = info->dev; - cqp->struct_ver = info->struct_ver; - cqp->scratch_array = info->scratch_array; - cqp->polarity = 0; - cqp->en_datacenter_tcp = info->en_datacenter_tcp; - cqp->enabled_vf_count = info->enabled_vf_count; - cqp->hmc_profile = info->hmc_profile; - info->dev->cqp = cqp; - - I40IW_RING_INIT(cqp->sq_ring, cqp->sq_size); - cqp->dev->cqp_cmd_stats[OP_REQUESTED_COMMANDS] = 0; - cqp->dev->cqp_cmd_stats[OP_COMPLETED_COMMANDS] = 0; - INIT_LIST_HEAD(&cqp->dev->cqp_cmd_head); /* for the cqp commands backlog. */ - - i40iw_wr32(cqp->dev->hw, I40E_PFPE_CQPTAIL, 0); - i40iw_wr32(cqp->dev->hw, I40E_PFPE_CQPDB, 0); - - i40iw_debug(cqp->dev, I40IW_DEBUG_WQE, - "%s: sq_size[%04d] hw_sq_size[%04d] sq_base[%p] sq_pa[%llxh] cqp[%p] polarity[x%04X]\n", - __func__, cqp->sq_size, cqp->hw_sq_size, - cqp->sq_base, cqp->sq_pa, cqp, cqp->polarity); - return 0; -} - -/** - * i40iw_sc_cqp_create - create cqp during bringup - * @cqp: struct for cqp hw - * @maj_err: If error, major err number - * @min_err: If error, minor err number - */ -static enum i40iw_status_code i40iw_sc_cqp_create(struct i40iw_sc_cqp *cqp, - u16 *maj_err, - u16 *min_err) -{ - u64 temp; - u32 cnt = 0, p1, p2, val = 0, err_code; - enum i40iw_status_code ret_code; - - *maj_err = 0; - *min_err = 0; - - ret_code = i40iw_allocate_dma_mem(cqp->dev->hw, - &cqp->sdbuf, - I40IW_UPDATE_SD_BUF_SIZE * cqp->sq_size, - I40IW_SD_BUF_ALIGNMENT); - - if (ret_code) - goto exit; - - temp = LS_64(cqp->hw_sq_size, I40IW_CQPHC_SQSIZE) | - LS_64(cqp->struct_ver, I40IW_CQPHC_SVER); - - set_64bit_val(cqp->host_ctx, 0, temp); - set_64bit_val(cqp->host_ctx, 8, cqp->sq_pa); - temp = LS_64(cqp->enabled_vf_count, I40IW_CQPHC_ENABLED_VFS) | - LS_64(cqp->hmc_profile, I40IW_CQPHC_HMC_PROFILE); - set_64bit_val(cqp->host_ctx, 16, temp); - set_64bit_val(cqp->host_ctx, 24, (uintptr_t)cqp); - set_64bit_val(cqp->host_ctx, 32, 0); - set_64bit_val(cqp->host_ctx, 40, 0); - set_64bit_val(cqp->host_ctx, 48, 0); - set_64bit_val(cqp->host_ctx, 56, 0); - - i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "CQP_HOST_CTX", - cqp->host_ctx, I40IW_CQP_CTX_SIZE * 8); - - p1 = RS_32_1(cqp->host_ctx_pa, 32); - p2 = (u32)cqp->host_ctx_pa; - - if (cqp->dev->is_pf) { - i40iw_wr32(cqp->dev->hw, I40E_PFPE_CCQPHIGH, p1); - i40iw_wr32(cqp->dev->hw, I40E_PFPE_CCQPLOW, p2); - } else { - i40iw_wr32(cqp->dev->hw, I40E_VFPE_CCQPHIGH1, p1); - i40iw_wr32(cqp->dev->hw, I40E_VFPE_CCQPLOW1, p2); - } - do { - if (cnt++ > I40IW_DONE_COUNT) { - i40iw_free_dma_mem(cqp->dev->hw, &cqp->sdbuf); - ret_code = I40IW_ERR_TIMEOUT; - /* - * read PFPE_CQPERRORCODES register to get the minor - * and major error code - */ - if (cqp->dev->is_pf) - err_code = i40iw_rd32(cqp->dev->hw, I40E_PFPE_CQPERRCODES); - else - err_code = i40iw_rd32(cqp->dev->hw, I40E_VFPE_CQPERRCODES1); - *min_err = RS_32(err_code, I40E_PFPE_CQPERRCODES_CQP_MINOR_CODE); - *maj_err = RS_32(err_code, I40E_PFPE_CQPERRCODES_CQP_MAJOR_CODE); - goto exit; - } - udelay(I40IW_SLEEP_COUNT); - if (cqp->dev->is_pf) - val = i40iw_rd32(cqp->dev->hw, I40E_PFPE_CCQPSTATUS); - else - val = i40iw_rd32(cqp->dev->hw, I40E_VFPE_CCQPSTATUS1); - } while (!val); - -exit: - if (!ret_code) - cqp->process_cqp_sds = i40iw_update_sds_noccq; - return ret_code; -} - -/** - * i40iw_sc_cqp_post_sq - post of cqp's sq - * @cqp: struct for cqp hw - */ -void i40iw_sc_cqp_post_sq(struct i40iw_sc_cqp *cqp) -{ - if (cqp->dev->is_pf) - i40iw_wr32(cqp->dev->hw, I40E_PFPE_CQPDB, I40IW_RING_GETCURRENT_HEAD(cqp->sq_ring)); - else - i40iw_wr32(cqp->dev->hw, I40E_VFPE_CQPDB1, I40IW_RING_GETCURRENT_HEAD(cqp->sq_ring)); - - i40iw_debug(cqp->dev, - I40IW_DEBUG_WQE, - "%s: HEAD_TAIL[%04d,%04d,%04d]\n", - __func__, - cqp->sq_ring.head, - cqp->sq_ring.tail, - cqp->sq_ring.size); -} - -/** - * i40iw_sc_cqp_get_next_send_wqe_idx - get next WQE on CQP SQ and pass back the index - * @cqp: pointer to CQP structure - * @scratch: private data for CQP WQE - * @wqe_idx: WQE index for next WQE on CQP SQ - */ -static u64 *i40iw_sc_cqp_get_next_send_wqe_idx(struct i40iw_sc_cqp *cqp, - u64 scratch, u32 *wqe_idx) -{ - u64 *wqe = NULL; - enum i40iw_status_code ret_code; - - if (I40IW_RING_FULL_ERR(cqp->sq_ring)) { - i40iw_debug(cqp->dev, - I40IW_DEBUG_WQE, - "%s: ring is full head %x tail %x size %x\n", - __func__, - cqp->sq_ring.head, - cqp->sq_ring.tail, - cqp->sq_ring.size); - return NULL; - } - I40IW_ATOMIC_RING_MOVE_HEAD(cqp->sq_ring, *wqe_idx, ret_code); - cqp->dev->cqp_cmd_stats[OP_REQUESTED_COMMANDS]++; - if (ret_code) - return NULL; - if (!*wqe_idx) - cqp->polarity = !cqp->polarity; - - wqe = cqp->sq_base[*wqe_idx].elem; - cqp->scratch_array[*wqe_idx] = scratch; - I40IW_CQP_INIT_WQE(wqe); - - return wqe; -} - -/** - * i40iw_sc_cqp_get_next_send_wqe - get next wqe on cqp sq - * @cqp: struct for cqp hw - * @scratch: private data for CQP WQE - */ -u64 *i40iw_sc_cqp_get_next_send_wqe(struct i40iw_sc_cqp *cqp, u64 scratch) -{ - u32 wqe_idx; - - return i40iw_sc_cqp_get_next_send_wqe_idx(cqp, scratch, &wqe_idx); -} - -/** - * i40iw_sc_cqp_destroy - destroy cqp during close - * @cqp: struct for cqp hw - */ -static enum i40iw_status_code i40iw_sc_cqp_destroy(struct i40iw_sc_cqp *cqp) -{ - u32 cnt = 0, val = 1; - enum i40iw_status_code ret_code = 0; - u32 cqpstat_addr; - - if (cqp->dev->is_pf) { - i40iw_wr32(cqp->dev->hw, I40E_PFPE_CCQPHIGH, 0); - i40iw_wr32(cqp->dev->hw, I40E_PFPE_CCQPLOW, 0); - cqpstat_addr = I40E_PFPE_CCQPSTATUS; - } else { - i40iw_wr32(cqp->dev->hw, I40E_VFPE_CCQPHIGH1, 0); - i40iw_wr32(cqp->dev->hw, I40E_VFPE_CCQPLOW1, 0); - cqpstat_addr = I40E_VFPE_CCQPSTATUS1; - } - do { - if (cnt++ > I40IW_DONE_COUNT) { - ret_code = I40IW_ERR_TIMEOUT; - break; - } - udelay(I40IW_SLEEP_COUNT); - val = i40iw_rd32(cqp->dev->hw, cqpstat_addr); - } while (val); - - i40iw_free_dma_mem(cqp->dev->hw, &cqp->sdbuf); - return ret_code; -} - -/** - * i40iw_sc_ccq_arm - enable intr for control cq - * @ccq: ccq sc struct - */ -static void i40iw_sc_ccq_arm(struct i40iw_sc_cq *ccq) -{ - u64 temp_val; - u16 sw_cq_sel; - u8 arm_next_se; - u8 arm_seq_num; - - /* write to cq doorbell shadow area */ - /* arm next se should always be zero */ - get_64bit_val(ccq->cq_uk.shadow_area, 32, &temp_val); - - sw_cq_sel = (u16)RS_64(temp_val, I40IW_CQ_DBSA_SW_CQ_SELECT); - arm_next_se = (u8)RS_64(temp_val, I40IW_CQ_DBSA_ARM_NEXT_SE); - - arm_seq_num = (u8)RS_64(temp_val, I40IW_CQ_DBSA_ARM_SEQ_NUM); - arm_seq_num++; - - temp_val = LS_64(arm_seq_num, I40IW_CQ_DBSA_ARM_SEQ_NUM) | - LS_64(sw_cq_sel, I40IW_CQ_DBSA_SW_CQ_SELECT) | - LS_64(arm_next_se, I40IW_CQ_DBSA_ARM_NEXT_SE) | - LS_64(1, I40IW_CQ_DBSA_ARM_NEXT); - - set_64bit_val(ccq->cq_uk.shadow_area, 32, temp_val); - - wmb(); /* make sure shadow area is updated before arming */ - - if (ccq->dev->is_pf) - i40iw_wr32(ccq->dev->hw, I40E_PFPE_CQARM, ccq->cq_uk.cq_id); - else - i40iw_wr32(ccq->dev->hw, I40E_VFPE_CQARM1, ccq->cq_uk.cq_id); -} - -/** - * i40iw_sc_ccq_get_cqe_info - get ccq's cq entry - * @ccq: ccq sc struct - * @info: completion q entry to return - */ -static enum i40iw_status_code i40iw_sc_ccq_get_cqe_info( - struct i40iw_sc_cq *ccq, - struct i40iw_ccq_cqe_info *info) -{ - u64 qp_ctx, temp, temp1; - u64 *cqe; - struct i40iw_sc_cqp *cqp; - u32 wqe_idx; - u8 polarity; - enum i40iw_status_code ret_code = 0; - - if (ccq->cq_uk.avoid_mem_cflct) - cqe = (u64 *)I40IW_GET_CURRENT_EXTENDED_CQ_ELEMENT(&ccq->cq_uk); - else - cqe = (u64 *)I40IW_GET_CURRENT_CQ_ELEMENT(&ccq->cq_uk); - - get_64bit_val(cqe, 24, &temp); - polarity = (u8)RS_64(temp, I40IW_CQ_VALID); - if (polarity != ccq->cq_uk.polarity) - return I40IW_ERR_QUEUE_EMPTY; - - get_64bit_val(cqe, 8, &qp_ctx); - cqp = (struct i40iw_sc_cqp *)(unsigned long)qp_ctx; - info->error = (bool)RS_64(temp, I40IW_CQ_ERROR); - info->min_err_code = (u16)RS_64(temp, I40IW_CQ_MINERR); - if (info->error) { - info->maj_err_code = (u16)RS_64(temp, I40IW_CQ_MAJERR); - info->min_err_code = (u16)RS_64(temp, I40IW_CQ_MINERR); - } - wqe_idx = (u32)RS_64(temp, I40IW_CQ_WQEIDX); - info->scratch = cqp->scratch_array[wqe_idx]; - - get_64bit_val(cqe, 16, &temp1); - info->op_ret_val = (u32)RS_64(temp1, I40IW_CCQ_OPRETVAL); - get_64bit_val(cqp->sq_base[wqe_idx].elem, 24, &temp1); - info->op_code = (u8)RS_64(temp1, I40IW_CQPSQ_OPCODE); - info->cqp = cqp; - - /* move the head for cq */ - I40IW_RING_MOVE_HEAD(ccq->cq_uk.cq_ring, ret_code); - if (I40IW_RING_GETCURRENT_HEAD(ccq->cq_uk.cq_ring) == 0) - ccq->cq_uk.polarity ^= 1; - - /* update cq tail in cq shadow memory also */ - I40IW_RING_MOVE_TAIL(ccq->cq_uk.cq_ring); - set_64bit_val(ccq->cq_uk.shadow_area, - 0, - I40IW_RING_GETCURRENT_HEAD(ccq->cq_uk.cq_ring)); - wmb(); /* write shadow area before tail */ - I40IW_RING_MOVE_TAIL(cqp->sq_ring); - ccq->dev->cqp_cmd_stats[OP_COMPLETED_COMMANDS]++; - - return ret_code; -} - -/** - * i40iw_sc_poll_for_cqp_op_done - Waits for last write to complete in CQP SQ - * @cqp: struct for cqp hw - * @op_code: cqp opcode for completion - * @compl_info: completion q entry to return - */ -static enum i40iw_status_code i40iw_sc_poll_for_cqp_op_done( - struct i40iw_sc_cqp *cqp, - u8 op_code, - struct i40iw_ccq_cqe_info *compl_info) -{ - struct i40iw_ccq_cqe_info info; - struct i40iw_sc_cq *ccq; - enum i40iw_status_code ret_code = 0; - u32 cnt = 0; - - memset(&info, 0, sizeof(info)); - ccq = cqp->dev->ccq; - while (1) { - if (cnt++ > I40IW_DONE_COUNT) - return I40IW_ERR_TIMEOUT; - - if (i40iw_sc_ccq_get_cqe_info(ccq, &info)) { - udelay(I40IW_SLEEP_COUNT); - continue; - } - - if (info.error) { - ret_code = I40IW_ERR_CQP_COMPL_ERROR; - break; - } - /* check if opcode is cq create */ - if (op_code != info.op_code) { - i40iw_debug(cqp->dev, I40IW_DEBUG_WQE, - "%s: opcode mismatch for my op code 0x%x, returned opcode %x\n", - __func__, op_code, info.op_code); - } - /* success, exit out of the loop */ - if (op_code == info.op_code) - break; - } - - if (compl_info) - memcpy(compl_info, &info, sizeof(*compl_info)); - - return ret_code; -} - -/** - * i40iw_sc_manage_hmc_pm_func_table - manage of function table - * @cqp: struct for cqp hw - * @scratch: u64 saved to be used during cqp completion - * @vf_index: vf index for cqp - * @free_pm_fcn: function number - * @post_sq: flag for cqp db to ring - */ -static enum i40iw_status_code i40iw_sc_manage_hmc_pm_func_table( - struct i40iw_sc_cqp *cqp, - u64 scratch, - u8 vf_index, - bool free_pm_fcn, - bool post_sq) -{ - u64 *wqe; - u64 header; - - if (vf_index >= I40IW_MAX_VF_PER_PF) - return I40IW_ERR_INVALID_VF_ID; - wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch); - if (!wqe) - return I40IW_ERR_RING_FULL; - - header = LS_64(vf_index, I40IW_CQPSQ_MHMC_VFIDX) | - LS_64(I40IW_CQP_OP_MANAGE_HMC_PM_FUNC_TABLE, I40IW_CQPSQ_OPCODE) | - LS_64(free_pm_fcn, I40IW_CQPSQ_MHMC_FREEPMFN) | - LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID); - - i40iw_insert_wqe_hdr(wqe, header); - i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "MANAGE_HMC_PM_FUNC_TABLE WQE", - wqe, I40IW_CQP_WQE_SIZE * 8); - if (post_sq) - i40iw_sc_cqp_post_sq(cqp); - return 0; -} - -/** - * i40iw_sc_set_hmc_resource_profile - cqp wqe for hmc profile - * @cqp: struct for cqp hw - * @scratch: u64 saved to be used during cqp completion - * @hmc_profile_type: type of profile to set - * @vf_num: vf number for profile - * @post_sq: flag for cqp db to ring - * @poll_registers: flag to poll register for cqp completion - */ -static enum i40iw_status_code i40iw_sc_set_hmc_resource_profile( - struct i40iw_sc_cqp *cqp, - u64 scratch, - u8 hmc_profile_type, - u8 vf_num, bool post_sq, - bool poll_registers) -{ - u64 *wqe; - u64 header; - u32 val, tail, error; - enum i40iw_status_code ret_code = 0; - - wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch); - if (!wqe) - return I40IW_ERR_RING_FULL; - - set_64bit_val(wqe, 16, - (LS_64(hmc_profile_type, I40IW_CQPSQ_SHMCRP_HMC_PROFILE) | - LS_64(vf_num, I40IW_CQPSQ_SHMCRP_VFNUM))); - - header = LS_64(I40IW_CQP_OP_SET_HMC_RESOURCE_PROFILE, I40IW_CQPSQ_OPCODE) | - LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID); - - i40iw_insert_wqe_hdr(wqe, header); - - i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "MANAGE_HMC_PM_FUNC_TABLE WQE", - wqe, I40IW_CQP_WQE_SIZE * 8); - - i40iw_get_cqp_reg_info(cqp, &val, &tail, &error); - if (error) - return I40IW_ERR_CQP_COMPL_ERROR; - - if (post_sq) { - i40iw_sc_cqp_post_sq(cqp); - if (poll_registers) - ret_code = i40iw_cqp_poll_registers(cqp, tail, 1000000); - else - ret_code = i40iw_sc_poll_for_cqp_op_done(cqp, - I40IW_CQP_OP_SHMC_PAGES_ALLOCATED, - NULL); - } - - return ret_code; -} - -/** - * i40iw_sc_manage_hmc_pm_func_table_done - wait for cqp wqe completion for function table - * @cqp: struct for cqp hw - */ -static enum i40iw_status_code i40iw_sc_manage_hmc_pm_func_table_done(struct i40iw_sc_cqp *cqp) -{ - return i40iw_sc_poll_for_cqp_op_done(cqp, I40IW_CQP_OP_MANAGE_HMC_PM_FUNC_TABLE, NULL); -} - -/** - * i40iw_sc_commit_fpm_values_done - wait for cqp eqe completion for fpm commit - * @cqp: struct for cqp hw - */ -static enum i40iw_status_code i40iw_sc_commit_fpm_values_done(struct i40iw_sc_cqp *cqp) -{ - return i40iw_sc_poll_for_cqp_op_done(cqp, I40IW_CQP_OP_COMMIT_FPM_VALUES, NULL); -} - -/** - * i40iw_sc_commit_fpm_values - cqp wqe for commit fpm values - * @cqp: struct for cqp hw - * @scratch: u64 saved to be used during cqp completion - * @hmc_fn_id: hmc function id - * @commit_fpm_mem: Memory for fpm values - * @post_sq: flag for cqp db to ring - * @wait_type: poll ccq or cqp registers for cqp completion - */ -static enum i40iw_status_code i40iw_sc_commit_fpm_values( - struct i40iw_sc_cqp *cqp, - u64 scratch, - u8 hmc_fn_id, - struct i40iw_dma_mem *commit_fpm_mem, - bool post_sq, - u8 wait_type) -{ - u64 *wqe; - u64 header; - u32 tail, val, error; - enum i40iw_status_code ret_code = 0; - - wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch); - if (!wqe) - return I40IW_ERR_RING_FULL; - - set_64bit_val(wqe, 16, hmc_fn_id); - set_64bit_val(wqe, 32, commit_fpm_mem->pa); - - header = LS_64(I40IW_CQP_OP_COMMIT_FPM_VALUES, I40IW_CQPSQ_OPCODE) | - LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID); - - i40iw_insert_wqe_hdr(wqe, header); - - i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "COMMIT_FPM_VALUES WQE", - wqe, I40IW_CQP_WQE_SIZE * 8); - - i40iw_get_cqp_reg_info(cqp, &val, &tail, &error); - if (error) - return I40IW_ERR_CQP_COMPL_ERROR; - - if (post_sq) { - i40iw_sc_cqp_post_sq(cqp); - - if (wait_type == I40IW_CQP_WAIT_POLL_REGS) - ret_code = i40iw_cqp_poll_registers(cqp, tail, I40IW_DONE_COUNT); - else if (wait_type == I40IW_CQP_WAIT_POLL_CQ) - ret_code = i40iw_sc_commit_fpm_values_done(cqp); - } - - return ret_code; -} - -/** - * i40iw_sc_query_rdma_features_done - poll cqp for query features done - * @cqp: struct for cqp hw - */ -static enum i40iw_status_code -i40iw_sc_query_rdma_features_done(struct i40iw_sc_cqp *cqp) -{ - return i40iw_sc_poll_for_cqp_op_done( - cqp, I40IW_CQP_OP_QUERY_RDMA_FEATURES, NULL); -} - -/** - * i40iw_sc_query_rdma_features - query rdma features - * @cqp: struct for cqp hw - * @feat_mem: holds PA for HW to use - * @scratch: u64 saved to be used during cqp completion - */ -static enum i40iw_status_code -i40iw_sc_query_rdma_features(struct i40iw_sc_cqp *cqp, - struct i40iw_dma_mem *feat_mem, u64 scratch) -{ - u64 *wqe; - u64 header; - - wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch); - if (!wqe) - return I40IW_ERR_RING_FULL; - - set_64bit_val(wqe, 32, feat_mem->pa); - - header = LS_64(I40IW_CQP_OP_QUERY_RDMA_FEATURES, I40IW_CQPSQ_OPCODE) | - LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID) | feat_mem->size; - - i40iw_insert_wqe_hdr(wqe, header); - - i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "QUERY RDMA FEATURES WQE", - wqe, I40IW_CQP_WQE_SIZE * 8); - - i40iw_sc_cqp_post_sq(cqp); - - return 0; -} - -/** - * i40iw_get_rdma_features - get RDMA features - * @dev: sc device struct - */ -enum i40iw_status_code i40iw_get_rdma_features(struct i40iw_sc_dev *dev) -{ - enum i40iw_status_code ret_code; - struct i40iw_dma_mem feat_buf; - u64 temp; - u16 byte_idx, feat_type, feat_cnt; - - ret_code = i40iw_allocate_dma_mem(dev->hw, &feat_buf, - I40IW_FEATURE_BUF_SIZE, - I40IW_FEATURE_BUF_ALIGNMENT); - - if (ret_code) - return I40IW_ERR_NO_MEMORY; - - ret_code = i40iw_sc_query_rdma_features(dev->cqp, &feat_buf, 0); - if (!ret_code) - ret_code = i40iw_sc_query_rdma_features_done(dev->cqp); - - if (ret_code) - goto exit; - - get_64bit_val(feat_buf.va, 0, &temp); - feat_cnt = RS_64(temp, I40IW_FEATURE_CNT); - if (feat_cnt < I40IW_MAX_FEATURES) { - ret_code = I40IW_ERR_INVALID_FEAT_CNT; - goto exit; - } else if (feat_cnt > I40IW_MAX_FEATURES) { - i40iw_debug(dev, I40IW_DEBUG_CQP, - "features buf size insufficient\n"); - } - - for (byte_idx = 0, feat_type = 0; feat_type < I40IW_MAX_FEATURES; - feat_type++, byte_idx += 8) { - get_64bit_val((u64 *)feat_buf.va, byte_idx, &temp); - dev->feature_info[feat_type] = RS_64(temp, I40IW_FEATURE_INFO); - } -exit: - i40iw_free_dma_mem(dev->hw, &feat_buf); - - return ret_code; -} - -/** - * i40iw_sc_query_fpm_values_done - poll for cqp wqe completion for query fpm - * @cqp: struct for cqp hw - */ -static enum i40iw_status_code i40iw_sc_query_fpm_values_done(struct i40iw_sc_cqp *cqp) -{ - return i40iw_sc_poll_for_cqp_op_done(cqp, I40IW_CQP_OP_QUERY_FPM_VALUES, NULL); -} - -/** - * i40iw_sc_query_fpm_values - cqp wqe query fpm values - * @cqp: struct for cqp hw - * @scratch: u64 saved to be used during cqp completion - * @hmc_fn_id: hmc function id - * @query_fpm_mem: memory for return fpm values - * @post_sq: flag for cqp db to ring - * @wait_type: poll ccq or cqp registers for cqp completion - */ -static enum i40iw_status_code i40iw_sc_query_fpm_values( - struct i40iw_sc_cqp *cqp, - u64 scratch, - u8 hmc_fn_id, - struct i40iw_dma_mem *query_fpm_mem, - bool post_sq, - u8 wait_type) -{ - u64 *wqe; - u64 header; - u32 tail, val, error; - enum i40iw_status_code ret_code = 0; - - wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch); - if (!wqe) - return I40IW_ERR_RING_FULL; - - set_64bit_val(wqe, 16, hmc_fn_id); - set_64bit_val(wqe, 32, query_fpm_mem->pa); - - header = LS_64(I40IW_CQP_OP_QUERY_FPM_VALUES, I40IW_CQPSQ_OPCODE) | - LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID); - - i40iw_insert_wqe_hdr(wqe, header); - - i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "QUERY_FPM WQE", - wqe, I40IW_CQP_WQE_SIZE * 8); - - /* read the tail from CQP_TAIL register */ - i40iw_get_cqp_reg_info(cqp, &val, &tail, &error); - - if (error) - return I40IW_ERR_CQP_COMPL_ERROR; - - if (post_sq) { - i40iw_sc_cqp_post_sq(cqp); - if (wait_type == I40IW_CQP_WAIT_POLL_REGS) - ret_code = i40iw_cqp_poll_registers(cqp, tail, I40IW_DONE_COUNT); - else if (wait_type == I40IW_CQP_WAIT_POLL_CQ) - ret_code = i40iw_sc_query_fpm_values_done(cqp); - } - - return ret_code; -} - -/** - * i40iw_sc_add_arp_cache_entry - cqp wqe add arp cache entry - * @cqp: struct for cqp hw - * @info: arp entry information - * @scratch: u64 saved to be used during cqp completion - * @post_sq: flag for cqp db to ring - */ -static enum i40iw_status_code i40iw_sc_add_arp_cache_entry( - struct i40iw_sc_cqp *cqp, - struct i40iw_add_arp_cache_entry_info *info, - u64 scratch, - bool post_sq) -{ - u64 *wqe; - u64 temp, header; - - wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch); - if (!wqe) - return I40IW_ERR_RING_FULL; - set_64bit_val(wqe, 8, info->reach_max); - - temp = info->mac_addr[5] | - LS_64_1(info->mac_addr[4], 8) | - LS_64_1(info->mac_addr[3], 16) | - LS_64_1(info->mac_addr[2], 24) | - LS_64_1(info->mac_addr[1], 32) | - LS_64_1(info->mac_addr[0], 40); - - set_64bit_val(wqe, 16, temp); - - header = info->arp_index | - LS_64(I40IW_CQP_OP_MANAGE_ARP, I40IW_CQPSQ_OPCODE) | - LS_64((info->permanent ? 1 : 0), I40IW_CQPSQ_MAT_PERMANENT) | - LS_64(1, I40IW_CQPSQ_MAT_ENTRYVALID) | - LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID); - - i40iw_insert_wqe_hdr(wqe, header); - - i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "ARP_CACHE_ENTRY WQE", - wqe, I40IW_CQP_WQE_SIZE * 8); - - if (post_sq) - i40iw_sc_cqp_post_sq(cqp); - return 0; -} - -/** - * i40iw_sc_del_arp_cache_entry - dele arp cache entry - * @cqp: struct for cqp hw - * @scratch: u64 saved to be used during cqp completion - * @arp_index: arp index to delete arp entry - * @post_sq: flag for cqp db to ring - */ -static enum i40iw_status_code i40iw_sc_del_arp_cache_entry( - struct i40iw_sc_cqp *cqp, - u64 scratch, - u16 arp_index, - bool post_sq) -{ - u64 *wqe; - u64 header; - - wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch); - if (!wqe) - return I40IW_ERR_RING_FULL; - - header = arp_index | - LS_64(I40IW_CQP_OP_MANAGE_ARP, I40IW_CQPSQ_OPCODE) | - LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID); - i40iw_insert_wqe_hdr(wqe, header); - - i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "ARP_CACHE_DEL_ENTRY WQE", - wqe, I40IW_CQP_WQE_SIZE * 8); - - if (post_sq) - i40iw_sc_cqp_post_sq(cqp); - return 0; -} - -/** - * i40iw_sc_query_arp_cache_entry - cqp wqe to query arp and arp index - * @cqp: struct for cqp hw - * @scratch: u64 saved to be used during cqp completion - * @arp_index: arp index to delete arp entry - * @post_sq: flag for cqp db to ring - */ -static enum i40iw_status_code i40iw_sc_query_arp_cache_entry( - struct i40iw_sc_cqp *cqp, - u64 scratch, - u16 arp_index, - bool post_sq) -{ - u64 *wqe; - u64 header; - - wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch); - if (!wqe) - return I40IW_ERR_RING_FULL; - - header = arp_index | - LS_64(I40IW_CQP_OP_MANAGE_ARP, I40IW_CQPSQ_OPCODE) | - LS_64(1, I40IW_CQPSQ_MAT_QUERY) | - LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID); - - i40iw_insert_wqe_hdr(wqe, header); - - i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "QUERY_ARP_CACHE_ENTRY WQE", - wqe, I40IW_CQP_WQE_SIZE * 8); - - if (post_sq) - i40iw_sc_cqp_post_sq(cqp); - return 0; -} - -/** - * i40iw_sc_manage_apbvt_entry - for adding and deleting apbvt entries - * @cqp: struct for cqp hw - * @info: info for apbvt entry to add or delete - * @scratch: u64 saved to be used during cqp completion - * @post_sq: flag for cqp db to ring - */ -static enum i40iw_status_code i40iw_sc_manage_apbvt_entry( - struct i40iw_sc_cqp *cqp, - struct i40iw_apbvt_info *info, - u64 scratch, - bool post_sq) -{ - u64 *wqe; - u64 header; - - wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch); - if (!wqe) - return I40IW_ERR_RING_FULL; - - set_64bit_val(wqe, 16, info->port); - - header = LS_64(I40IW_CQP_OP_MANAGE_APBVT, I40IW_CQPSQ_OPCODE) | - LS_64(info->add, I40IW_CQPSQ_MAPT_ADDPORT) | - LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID); - - i40iw_insert_wqe_hdr(wqe, header); - - i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "MANAGE_APBVT WQE", - wqe, I40IW_CQP_WQE_SIZE * 8); - - if (post_sq) - i40iw_sc_cqp_post_sq(cqp); - return 0; -} - -/** - * i40iw_sc_manage_qhash_table_entry - manage quad hash entries - * @cqp: struct for cqp hw - * @info: info for quad hash to manage - * @scratch: u64 saved to be used during cqp completion - * @post_sq: flag for cqp db to ring - * - * This is called before connection establishment is started. For passive connections, when - * listener is created, it will call with entry type of I40IW_QHASH_TYPE_TCP_SYN with local - * ip address and tcp port. When SYN is received (passive connections) or - * sent (active connections), this routine is called with entry type of - * I40IW_QHASH_TYPE_TCP_ESTABLISHED and quad is passed in info. - * - * When iwarp connection is done and its state moves to RTS, the quad hash entry in - * the hardware will point to iwarp's qp number and requires no calls from the driver. - */ -static enum i40iw_status_code i40iw_sc_manage_qhash_table_entry( - struct i40iw_sc_cqp *cqp, - struct i40iw_qhash_table_info *info, - u64 scratch, - bool post_sq) -{ - u64 *wqe; - u64 qw1 = 0; - u64 qw2 = 0; - u64 temp; - struct i40iw_sc_vsi *vsi = info->vsi; - - wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch); - if (!wqe) - return I40IW_ERR_RING_FULL; - - temp = info->mac_addr[5] | - LS_64_1(info->mac_addr[4], 8) | - LS_64_1(info->mac_addr[3], 16) | - LS_64_1(info->mac_addr[2], 24) | - LS_64_1(info->mac_addr[1], 32) | - LS_64_1(info->mac_addr[0], 40); - - set_64bit_val(wqe, 0, temp); - - qw1 = LS_64(info->qp_num, I40IW_CQPSQ_QHASH_QPN) | - LS_64(info->dest_port, I40IW_CQPSQ_QHASH_DEST_PORT); - if (info->ipv4_valid) { - set_64bit_val(wqe, - 48, - LS_64(info->dest_ip[0], I40IW_CQPSQ_QHASH_ADDR3)); - } else { - set_64bit_val(wqe, - 56, - LS_64(info->dest_ip[0], I40IW_CQPSQ_QHASH_ADDR0) | - LS_64(info->dest_ip[1], I40IW_CQPSQ_QHASH_ADDR1)); - - set_64bit_val(wqe, - 48, - LS_64(info->dest_ip[2], I40IW_CQPSQ_QHASH_ADDR2) | - LS_64(info->dest_ip[3], I40IW_CQPSQ_QHASH_ADDR3)); - } - qw2 = LS_64(vsi->qos[info->user_pri].qs_handle, I40IW_CQPSQ_QHASH_QS_HANDLE); - if (info->vlan_valid) - qw2 |= LS_64(info->vlan_id, I40IW_CQPSQ_QHASH_VLANID); - set_64bit_val(wqe, 16, qw2); - if (info->entry_type == I40IW_QHASH_TYPE_TCP_ESTABLISHED) { - qw1 |= LS_64(info->src_port, I40IW_CQPSQ_QHASH_SRC_PORT); - if (!info->ipv4_valid) { - set_64bit_val(wqe, - 40, - LS_64(info->src_ip[0], I40IW_CQPSQ_QHASH_ADDR0) | - LS_64(info->src_ip[1], I40IW_CQPSQ_QHASH_ADDR1)); - set_64bit_val(wqe, - 32, - LS_64(info->src_ip[2], I40IW_CQPSQ_QHASH_ADDR2) | - LS_64(info->src_ip[3], I40IW_CQPSQ_QHASH_ADDR3)); - } else { - set_64bit_val(wqe, - 32, - LS_64(info->src_ip[0], I40IW_CQPSQ_QHASH_ADDR3)); - } - } - - set_64bit_val(wqe, 8, qw1); - temp = LS_64(cqp->polarity, I40IW_CQPSQ_QHASH_WQEVALID) | - LS_64(I40IW_CQP_OP_MANAGE_QUAD_HASH_TABLE_ENTRY, I40IW_CQPSQ_QHASH_OPCODE) | - LS_64(info->manage, I40IW_CQPSQ_QHASH_MANAGE) | - LS_64(info->ipv4_valid, I40IW_CQPSQ_QHASH_IPV4VALID) | - LS_64(info->vlan_valid, I40IW_CQPSQ_QHASH_VLANVALID) | - LS_64(info->entry_type, I40IW_CQPSQ_QHASH_ENTRYTYPE); - - i40iw_insert_wqe_hdr(wqe, temp); - - i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "MANAGE_QHASH WQE", - wqe, I40IW_CQP_WQE_SIZE * 8); - - if (post_sq) - i40iw_sc_cqp_post_sq(cqp); - return 0; -} - -/** - * i40iw_sc_alloc_local_mac_ipaddr_entry - cqp wqe for loc mac entry - * @cqp: struct for cqp hw - * @scratch: u64 saved to be used during cqp completion - * @post_sq: flag for cqp db to ring - */ -static enum i40iw_status_code i40iw_sc_alloc_local_mac_ipaddr_entry( - struct i40iw_sc_cqp *cqp, - u64 scratch, - bool post_sq) -{ - u64 *wqe; - u64 header; - - wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch); - if (!wqe) - return I40IW_ERR_RING_FULL; - header = LS_64(I40IW_CQP_OP_ALLOCATE_LOC_MAC_IP_TABLE_ENTRY, I40IW_CQPSQ_OPCODE) | - LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID); - - i40iw_insert_wqe_hdr(wqe, header); - i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "ALLOCATE_LOCAL_MAC_IPADDR WQE", - wqe, I40IW_CQP_WQE_SIZE * 8); - if (post_sq) - i40iw_sc_cqp_post_sq(cqp); - return 0; -} - -/** - * i40iw_sc_add_local_mac_ipaddr_entry - add mac enry - * @cqp: struct for cqp hw - * @info:mac addr info - * @scratch: u64 saved to be used during cqp completion - * @post_sq: flag for cqp db to ring - */ -static enum i40iw_status_code i40iw_sc_add_local_mac_ipaddr_entry( - struct i40iw_sc_cqp *cqp, - struct i40iw_local_mac_ipaddr_entry_info *info, - u64 scratch, - bool post_sq) -{ - u64 *wqe; - u64 temp, header; - - wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch); - if (!wqe) - return I40IW_ERR_RING_FULL; - temp = info->mac_addr[5] | - LS_64_1(info->mac_addr[4], 8) | - LS_64_1(info->mac_addr[3], 16) | - LS_64_1(info->mac_addr[2], 24) | - LS_64_1(info->mac_addr[1], 32) | - LS_64_1(info->mac_addr[0], 40); - - set_64bit_val(wqe, 32, temp); - - header = LS_64(info->entry_idx, I40IW_CQPSQ_MLIPA_IPTABLEIDX) | - LS_64(I40IW_CQP_OP_MANAGE_LOC_MAC_IP_TABLE, I40IW_CQPSQ_OPCODE) | - LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID); - - i40iw_insert_wqe_hdr(wqe, header); - - i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "ADD_LOCAL_MAC_IPADDR WQE", - wqe, I40IW_CQP_WQE_SIZE * 8); - - if (post_sq) - i40iw_sc_cqp_post_sq(cqp); - return 0; -} - -/** - * i40iw_sc_del_local_mac_ipaddr_entry - cqp wqe to dele local mac - * @cqp: struct for cqp hw - * @scratch: u64 saved to be used during cqp completion - * @entry_idx: index of mac entry - * @ignore_ref_count: to force mac adde delete - * @post_sq: flag for cqp db to ring - */ -static enum i40iw_status_code i40iw_sc_del_local_mac_ipaddr_entry( - struct i40iw_sc_cqp *cqp, - u64 scratch, - u8 entry_idx, - u8 ignore_ref_count, - bool post_sq) -{ - u64 *wqe; - u64 header; - - wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch); - if (!wqe) - return I40IW_ERR_RING_FULL; - header = LS_64(entry_idx, I40IW_CQPSQ_MLIPA_IPTABLEIDX) | - LS_64(I40IW_CQP_OP_MANAGE_LOC_MAC_IP_TABLE, I40IW_CQPSQ_OPCODE) | - LS_64(1, I40IW_CQPSQ_MLIPA_FREEENTRY) | - LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID) | - LS_64(ignore_ref_count, I40IW_CQPSQ_MLIPA_IGNORE_REF_CNT); - - i40iw_insert_wqe_hdr(wqe, header); - - i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "DEL_LOCAL_MAC_IPADDR WQE", - wqe, I40IW_CQP_WQE_SIZE * 8); - - if (post_sq) - i40iw_sc_cqp_post_sq(cqp); - return 0; -} - -/** - * i40iw_sc_cqp_nop - send a nop wqe - * @cqp: struct for cqp hw - * @scratch: u64 saved to be used during cqp completion - * @post_sq: flag for cqp db to ring - */ -static enum i40iw_status_code i40iw_sc_cqp_nop(struct i40iw_sc_cqp *cqp, - u64 scratch, - bool post_sq) -{ - u64 *wqe; - u64 header; - - wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch); - if (!wqe) - return I40IW_ERR_RING_FULL; - header = LS_64(I40IW_CQP_OP_NOP, I40IW_CQPSQ_OPCODE) | - LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID); - i40iw_insert_wqe_hdr(wqe, header); - i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "NOP WQE", - wqe, I40IW_CQP_WQE_SIZE * 8); - - if (post_sq) - i40iw_sc_cqp_post_sq(cqp); - return 0; -} - -/** - * i40iw_sc_ceq_init - initialize ceq - * @ceq: ceq sc structure - * @info: ceq initialization info - */ -static enum i40iw_status_code i40iw_sc_ceq_init(struct i40iw_sc_ceq *ceq, - struct i40iw_ceq_init_info *info) -{ - u32 pble_obj_cnt; - - if ((info->elem_cnt < I40IW_MIN_CEQ_ENTRIES) || - (info->elem_cnt > I40IW_MAX_CEQ_ENTRIES)) - return I40IW_ERR_INVALID_SIZE; - - if (info->ceq_id >= I40IW_MAX_CEQID) - return I40IW_ERR_INVALID_CEQ_ID; - - pble_obj_cnt = info->dev->hmc_info->hmc_obj[I40IW_HMC_IW_PBLE].cnt; - - if (info->virtual_map && (info->first_pm_pbl_idx >= pble_obj_cnt)) - return I40IW_ERR_INVALID_PBLE_INDEX; - - ceq->size = sizeof(*ceq); - ceq->ceqe_base = (struct i40iw_ceqe *)info->ceqe_base; - ceq->ceq_id = info->ceq_id; - ceq->dev = info->dev; - ceq->elem_cnt = info->elem_cnt; - ceq->ceq_elem_pa = info->ceqe_pa; - ceq->virtual_map = info->virtual_map; - - ceq->pbl_chunk_size = (ceq->virtual_map ? info->pbl_chunk_size : 0); - ceq->first_pm_pbl_idx = (ceq->virtual_map ? info->first_pm_pbl_idx : 0); - ceq->pbl_list = (ceq->virtual_map ? info->pbl_list : NULL); - - ceq->tph_en = info->tph_en; - ceq->tph_val = info->tph_val; - ceq->polarity = 1; - I40IW_RING_INIT(ceq->ceq_ring, ceq->elem_cnt); - ceq->dev->ceq[info->ceq_id] = ceq; - - return 0; -} - -/** - * i40iw_sc_ceq_create - create ceq wqe - * @ceq: ceq sc structure - * @scratch: u64 saved to be used during cqp completion - * @post_sq: flag for cqp db to ring - */ -static enum i40iw_status_code i40iw_sc_ceq_create(struct i40iw_sc_ceq *ceq, - u64 scratch, - bool post_sq) -{ - struct i40iw_sc_cqp *cqp; - u64 *wqe; - u64 header; - - cqp = ceq->dev->cqp; - wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch); - if (!wqe) - return I40IW_ERR_RING_FULL; - set_64bit_val(wqe, 16, ceq->elem_cnt); - set_64bit_val(wqe, 32, (ceq->virtual_map ? 0 : ceq->ceq_elem_pa)); - set_64bit_val(wqe, 48, (ceq->virtual_map ? ceq->first_pm_pbl_idx : 0)); - set_64bit_val(wqe, 56, LS_64(ceq->tph_val, I40IW_CQPSQ_TPHVAL)); - - header = ceq->ceq_id | - LS_64(I40IW_CQP_OP_CREATE_CEQ, I40IW_CQPSQ_OPCODE) | - LS_64(ceq->pbl_chunk_size, I40IW_CQPSQ_CEQ_LPBLSIZE) | - LS_64(ceq->virtual_map, I40IW_CQPSQ_CEQ_VMAP) | - LS_64(ceq->tph_en, I40IW_CQPSQ_TPHEN) | - LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID); - - i40iw_insert_wqe_hdr(wqe, header); - - i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "CEQ_CREATE WQE", - wqe, I40IW_CQP_WQE_SIZE * 8); - - if (post_sq) - i40iw_sc_cqp_post_sq(cqp); - return 0; -} - -/** - * i40iw_sc_cceq_create_done - poll for control ceq wqe to complete - * @ceq: ceq sc structure - */ -static enum i40iw_status_code i40iw_sc_cceq_create_done(struct i40iw_sc_ceq *ceq) -{ - struct i40iw_sc_cqp *cqp; - - cqp = ceq->dev->cqp; - return i40iw_sc_poll_for_cqp_op_done(cqp, I40IW_CQP_OP_CREATE_CEQ, NULL); -} - -/** - * i40iw_sc_cceq_destroy_done - poll for destroy cceq to complete - * @ceq: ceq sc structure - */ -static enum i40iw_status_code i40iw_sc_cceq_destroy_done(struct i40iw_sc_ceq *ceq) -{ - struct i40iw_sc_cqp *cqp; - - cqp = ceq->dev->cqp; - cqp->process_cqp_sds = i40iw_update_sds_noccq; - return i40iw_sc_poll_for_cqp_op_done(cqp, I40IW_CQP_OP_DESTROY_CEQ, NULL); -} - -/** - * i40iw_sc_cceq_create - create cceq - * @ceq: ceq sc structure - * @scratch: u64 saved to be used during cqp completion - */ -static enum i40iw_status_code i40iw_sc_cceq_create(struct i40iw_sc_ceq *ceq, u64 scratch) -{ - enum i40iw_status_code ret_code; - - ret_code = i40iw_sc_ceq_create(ceq, scratch, true); - if (!ret_code) - ret_code = i40iw_sc_cceq_create_done(ceq); - return ret_code; -} - -/** - * i40iw_sc_ceq_destroy - destroy ceq - * @ceq: ceq sc structure - * @scratch: u64 saved to be used during cqp completion - * @post_sq: flag for cqp db to ring - */ -static enum i40iw_status_code i40iw_sc_ceq_destroy(struct i40iw_sc_ceq *ceq, - u64 scratch, - bool post_sq) -{ - struct i40iw_sc_cqp *cqp; - u64 *wqe; - u64 header; - - cqp = ceq->dev->cqp; - wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch); - if (!wqe) - return I40IW_ERR_RING_FULL; - set_64bit_val(wqe, 16, ceq->elem_cnt); - set_64bit_val(wqe, 48, ceq->first_pm_pbl_idx); - header = ceq->ceq_id | - LS_64(I40IW_CQP_OP_DESTROY_CEQ, I40IW_CQPSQ_OPCODE) | - LS_64(ceq->pbl_chunk_size, I40IW_CQPSQ_CEQ_LPBLSIZE) | - LS_64(ceq->virtual_map, I40IW_CQPSQ_CEQ_VMAP) | - LS_64(ceq->tph_en, I40IW_CQPSQ_TPHEN) | - LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID); - i40iw_insert_wqe_hdr(wqe, header); - i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "CEQ_DESTROY WQE", - wqe, I40IW_CQP_WQE_SIZE * 8); - - if (post_sq) - i40iw_sc_cqp_post_sq(cqp); - return 0; -} - -/** - * i40iw_sc_process_ceq - process ceq - * @dev: sc device struct - * @ceq: ceq sc structure - */ -static void *i40iw_sc_process_ceq(struct i40iw_sc_dev *dev, struct i40iw_sc_ceq *ceq) -{ - u64 temp; - u64 *ceqe; - struct i40iw_sc_cq *cq = NULL; - u8 polarity; - - ceqe = (u64 *)I40IW_GET_CURRENT_CEQ_ELEMENT(ceq); - get_64bit_val(ceqe, 0, &temp); - polarity = (u8)RS_64(temp, I40IW_CEQE_VALID); - if (polarity != ceq->polarity) - return cq; - - cq = (struct i40iw_sc_cq *)(unsigned long)LS_64_1(temp, 1); - - I40IW_RING_MOVE_TAIL(ceq->ceq_ring); - if (I40IW_RING_GETCURRENT_TAIL(ceq->ceq_ring) == 0) - ceq->polarity ^= 1; - - if (dev->is_pf) - i40iw_wr32(dev->hw, I40E_PFPE_CQACK, cq->cq_uk.cq_id); - else - i40iw_wr32(dev->hw, I40E_VFPE_CQACK1, cq->cq_uk.cq_id); - - return cq; -} - -/** - * i40iw_sc_aeq_init - initialize aeq - * @aeq: aeq structure ptr - * @info: aeq initialization info - */ -static enum i40iw_status_code i40iw_sc_aeq_init(struct i40iw_sc_aeq *aeq, - struct i40iw_aeq_init_info *info) -{ - u32 pble_obj_cnt; - - if ((info->elem_cnt < I40IW_MIN_AEQ_ENTRIES) || - (info->elem_cnt > I40IW_MAX_AEQ_ENTRIES)) - return I40IW_ERR_INVALID_SIZE; - pble_obj_cnt = info->dev->hmc_info->hmc_obj[I40IW_HMC_IW_PBLE].cnt; - - if (info->virtual_map && (info->first_pm_pbl_idx >= pble_obj_cnt)) - return I40IW_ERR_INVALID_PBLE_INDEX; - - aeq->size = sizeof(*aeq); - aeq->polarity = 1; - aeq->aeqe_base = (struct i40iw_sc_aeqe *)info->aeqe_base; - aeq->dev = info->dev; - aeq->elem_cnt = info->elem_cnt; - - aeq->aeq_elem_pa = info->aeq_elem_pa; - I40IW_RING_INIT(aeq->aeq_ring, aeq->elem_cnt); - info->dev->aeq = aeq; - - aeq->virtual_map = info->virtual_map; - aeq->pbl_list = (aeq->virtual_map ? info->pbl_list : NULL); - aeq->pbl_chunk_size = (aeq->virtual_map ? info->pbl_chunk_size : 0); - aeq->first_pm_pbl_idx = (aeq->virtual_map ? info->first_pm_pbl_idx : 0); - info->dev->aeq = aeq; - return 0; -} - -/** - * i40iw_sc_aeq_create - create aeq - * @aeq: aeq structure ptr - * @scratch: u64 saved to be used during cqp completion - * @post_sq: flag for cqp db to ring - */ -static enum i40iw_status_code i40iw_sc_aeq_create(struct i40iw_sc_aeq *aeq, - u64 scratch, - bool post_sq) -{ - u64 *wqe; - struct i40iw_sc_cqp *cqp; - u64 header; - - cqp = aeq->dev->cqp; - wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch); - if (!wqe) - return I40IW_ERR_RING_FULL; - set_64bit_val(wqe, 16, aeq->elem_cnt); - set_64bit_val(wqe, 32, - (aeq->virtual_map ? 0 : aeq->aeq_elem_pa)); - set_64bit_val(wqe, 48, - (aeq->virtual_map ? aeq->first_pm_pbl_idx : 0)); - - header = LS_64(I40IW_CQP_OP_CREATE_AEQ, I40IW_CQPSQ_OPCODE) | - LS_64(aeq->pbl_chunk_size, I40IW_CQPSQ_AEQ_LPBLSIZE) | - LS_64(aeq->virtual_map, I40IW_CQPSQ_AEQ_VMAP) | - LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID); - - i40iw_insert_wqe_hdr(wqe, header); - i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "AEQ_CREATE WQE", - wqe, I40IW_CQP_WQE_SIZE * 8); - if (post_sq) - i40iw_sc_cqp_post_sq(cqp); - return 0; -} - -/** - * i40iw_sc_aeq_destroy - destroy aeq during close - * @aeq: aeq structure ptr - * @scratch: u64 saved to be used during cqp completion - * @post_sq: flag for cqp db to ring - */ -static enum i40iw_status_code i40iw_sc_aeq_destroy(struct i40iw_sc_aeq *aeq, - u64 scratch, - bool post_sq) -{ - u64 *wqe; - struct i40iw_sc_cqp *cqp; - u64 header; - - cqp = aeq->dev->cqp; - wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch); - if (!wqe) - return I40IW_ERR_RING_FULL; - set_64bit_val(wqe, 16, aeq->elem_cnt); - set_64bit_val(wqe, 48, aeq->first_pm_pbl_idx); - header = LS_64(I40IW_CQP_OP_DESTROY_AEQ, I40IW_CQPSQ_OPCODE) | - LS_64(aeq->pbl_chunk_size, I40IW_CQPSQ_AEQ_LPBLSIZE) | - LS_64(aeq->virtual_map, I40IW_CQPSQ_AEQ_VMAP) | - LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID); - i40iw_insert_wqe_hdr(wqe, header); - - i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "AEQ_DESTROY WQE", - wqe, I40IW_CQP_WQE_SIZE * 8); - if (post_sq) - i40iw_sc_cqp_post_sq(cqp); - return 0; -} - -/** - * i40iw_sc_get_next_aeqe - get next aeq entry - * @aeq: aeq structure ptr - * @info: aeqe info to be returned - */ -static enum i40iw_status_code i40iw_sc_get_next_aeqe(struct i40iw_sc_aeq *aeq, - struct i40iw_aeqe_info *info) -{ - u64 temp, compl_ctx; - u64 *aeqe; - u16 wqe_idx; - u8 ae_src; - u8 polarity; - - aeqe = (u64 *)I40IW_GET_CURRENT_AEQ_ELEMENT(aeq); - get_64bit_val(aeqe, 0, &compl_ctx); - get_64bit_val(aeqe, 8, &temp); - polarity = (u8)RS_64(temp, I40IW_AEQE_VALID); - - if (aeq->polarity != polarity) - return I40IW_ERR_QUEUE_EMPTY; - - i40iw_debug_buf(aeq->dev, I40IW_DEBUG_WQE, "AEQ_ENTRY", aeqe, 16); - - ae_src = (u8)RS_64(temp, I40IW_AEQE_AESRC); - wqe_idx = (u16)RS_64(temp, I40IW_AEQE_WQDESCIDX); - info->qp_cq_id = (u32)RS_64(temp, I40IW_AEQE_QPCQID); - info->ae_id = (u16)RS_64(temp, I40IW_AEQE_AECODE); - info->tcp_state = (u8)RS_64(temp, I40IW_AEQE_TCPSTATE); - info->iwarp_state = (u8)RS_64(temp, I40IW_AEQE_IWSTATE); - info->q2_data_written = (u8)RS_64(temp, I40IW_AEQE_Q2DATA); - info->aeqe_overflow = (bool)RS_64(temp, I40IW_AEQE_OVERFLOW); - - switch (info->ae_id) { - case I40IW_AE_PRIV_OPERATION_DENIED: - case I40IW_AE_UDA_XMIT_DGRAM_TOO_LONG: - case I40IW_AE_UDA_XMIT_DGRAM_TOO_SHORT: - case I40IW_AE_BAD_CLOSE: - case I40IW_AE_RDMAP_ROE_BAD_LLP_CLOSE: - case I40IW_AE_RDMA_READ_WHILE_ORD_ZERO: - case I40IW_AE_STAG_ZERO_INVALID: - case I40IW_AE_IB_RREQ_AND_Q1_FULL: - case I40IW_AE_WQE_UNEXPECTED_OPCODE: - case I40IW_AE_DDP_UBE_INVALID_DDP_VERSION: - case I40IW_AE_DDP_UBE_INVALID_MO: - case I40IW_AE_DDP_UBE_INVALID_QN: - case I40IW_AE_DDP_NO_L_BIT: - case I40IW_AE_RDMAP_ROE_INVALID_RDMAP_VERSION: - case I40IW_AE_RDMAP_ROE_UNEXPECTED_OPCODE: - case I40IW_AE_ROE_INVALID_RDMA_READ_REQUEST: - case I40IW_AE_ROE_INVALID_RDMA_WRITE_OR_READ_RESP: - case I40IW_AE_INVALID_ARP_ENTRY: - case I40IW_AE_INVALID_TCP_OPTION_RCVD: - case I40IW_AE_STALE_ARP_ENTRY: - case I40IW_AE_LLP_CLOSE_COMPLETE: - case I40IW_AE_LLP_CONNECTION_RESET: - case I40IW_AE_LLP_FIN_RECEIVED: - case I40IW_AE_LLP_RECEIVED_MPA_CRC_ERROR: - case I40IW_AE_LLP_SEGMENT_TOO_SMALL: - case I40IW_AE_LLP_SYN_RECEIVED: - case I40IW_AE_LLP_TERMINATE_RECEIVED: - case I40IW_AE_LLP_TOO_MANY_RETRIES: - case I40IW_AE_LLP_DOUBT_REACHABILITY: - case I40IW_AE_RESET_SENT: - case I40IW_AE_TERMINATE_SENT: - case I40IW_AE_RESET_NOT_SENT: - case I40IW_AE_LCE_QP_CATASTROPHIC: - case I40IW_AE_QP_SUSPEND_COMPLETE: - info->qp = true; - info->compl_ctx = compl_ctx; - ae_src = I40IW_AE_SOURCE_RSVD; - break; - case I40IW_AE_LCE_CQ_CATASTROPHIC: - info->cq = true; - info->compl_ctx = LS_64_1(compl_ctx, 1); - ae_src = I40IW_AE_SOURCE_RSVD; - break; - } - - switch (ae_src) { - case I40IW_AE_SOURCE_RQ: - case I40IW_AE_SOURCE_RQ_0011: - info->qp = true; - info->wqe_idx = wqe_idx; - info->compl_ctx = compl_ctx; - break; - case I40IW_AE_SOURCE_CQ: - case I40IW_AE_SOURCE_CQ_0110: - case I40IW_AE_SOURCE_CQ_1010: - case I40IW_AE_SOURCE_CQ_1110: - info->cq = true; - info->compl_ctx = LS_64_1(compl_ctx, 1); - break; - case I40IW_AE_SOURCE_SQ: - case I40IW_AE_SOURCE_SQ_0111: - info->qp = true; - info->sq = true; - info->wqe_idx = wqe_idx; - info->compl_ctx = compl_ctx; - break; - case I40IW_AE_SOURCE_IN_RR_WR: - case I40IW_AE_SOURCE_IN_RR_WR_1011: - info->qp = true; - info->compl_ctx = compl_ctx; - info->in_rdrsp_wr = true; - break; - case I40IW_AE_SOURCE_OUT_RR: - case I40IW_AE_SOURCE_OUT_RR_1111: - info->qp = true; - info->compl_ctx = compl_ctx; - info->out_rdrsp = true; - break; - case I40IW_AE_SOURCE_RSVD: - default: - break; - } - I40IW_RING_MOVE_TAIL(aeq->aeq_ring); - if (I40IW_RING_GETCURRENT_TAIL(aeq->aeq_ring) == 0) - aeq->polarity ^= 1; - return 0; -} - -/** - * i40iw_sc_repost_aeq_entries - repost completed aeq entries - * @dev: sc device struct - * @count: allocate count - */ -static enum i40iw_status_code i40iw_sc_repost_aeq_entries(struct i40iw_sc_dev *dev, - u32 count) -{ - - if (dev->is_pf) - i40iw_wr32(dev->hw, I40E_PFPE_AEQALLOC, count); - else - i40iw_wr32(dev->hw, I40E_VFPE_AEQALLOC1, count); - - return 0; -} - -/** - * i40iw_sc_aeq_create_done - create aeq - * @aeq: aeq structure ptr - */ -static enum i40iw_status_code i40iw_sc_aeq_create_done(struct i40iw_sc_aeq *aeq) -{ - struct i40iw_sc_cqp *cqp; - - cqp = aeq->dev->cqp; - return i40iw_sc_poll_for_cqp_op_done(cqp, I40IW_CQP_OP_CREATE_AEQ, NULL); -} - -/** - * i40iw_sc_aeq_destroy_done - destroy of aeq during close - * @aeq: aeq structure ptr - */ -static enum i40iw_status_code i40iw_sc_aeq_destroy_done(struct i40iw_sc_aeq *aeq) -{ - struct i40iw_sc_cqp *cqp; - - cqp = aeq->dev->cqp; - return i40iw_sc_poll_for_cqp_op_done(cqp, I40IW_CQP_OP_DESTROY_AEQ, NULL); -} - -/** - * i40iw_sc_ccq_init - initialize control cq - * @cq: sc's cq ctruct - * @info: info for control cq initialization - */ -static enum i40iw_status_code i40iw_sc_ccq_init(struct i40iw_sc_cq *cq, - struct i40iw_ccq_init_info *info) -{ - u32 pble_obj_cnt; - - if (info->num_elem < I40IW_MIN_CQ_SIZE || info->num_elem > I40IW_MAX_CQ_SIZE) - return I40IW_ERR_INVALID_SIZE; - - if (info->ceq_id > I40IW_MAX_CEQID) - return I40IW_ERR_INVALID_CEQ_ID; - - pble_obj_cnt = info->dev->hmc_info->hmc_obj[I40IW_HMC_IW_PBLE].cnt; - - if (info->virtual_map && (info->first_pm_pbl_idx >= pble_obj_cnt)) - return I40IW_ERR_INVALID_PBLE_INDEX; - - cq->cq_pa = info->cq_pa; - cq->cq_uk.cq_base = info->cq_base; - cq->shadow_area_pa = info->shadow_area_pa; - cq->cq_uk.shadow_area = info->shadow_area; - cq->shadow_read_threshold = info->shadow_read_threshold; - cq->dev = info->dev; - cq->ceq_id = info->ceq_id; - cq->cq_uk.cq_size = info->num_elem; - cq->cq_type = I40IW_CQ_TYPE_CQP; - cq->ceqe_mask = info->ceqe_mask; - I40IW_RING_INIT(cq->cq_uk.cq_ring, info->num_elem); - - cq->cq_uk.cq_id = 0; /* control cq is id 0 always */ - cq->ceq_id_valid = info->ceq_id_valid; - cq->tph_en = info->tph_en; - cq->tph_val = info->tph_val; - cq->cq_uk.avoid_mem_cflct = info->avoid_mem_cflct; - - cq->pbl_list = info->pbl_list; - cq->virtual_map = info->virtual_map; - cq->pbl_chunk_size = info->pbl_chunk_size; - cq->first_pm_pbl_idx = info->first_pm_pbl_idx; - cq->cq_uk.polarity = true; - - /* following are only for iw cqs so initialize them to zero */ - cq->cq_uk.cqe_alloc_reg = NULL; - info->dev->ccq = cq; - return 0; -} - -/** - * i40iw_sc_ccq_create_done - poll cqp for ccq create - * @ccq: ccq sc struct - */ -static enum i40iw_status_code i40iw_sc_ccq_create_done(struct i40iw_sc_cq *ccq) -{ - struct i40iw_sc_cqp *cqp; - - cqp = ccq->dev->cqp; - return i40iw_sc_poll_for_cqp_op_done(cqp, I40IW_CQP_OP_CREATE_CQ, NULL); -} - -/** - * i40iw_sc_ccq_create - create control cq - * @ccq: ccq sc struct - * @scratch: u64 saved to be used during cqp completion - * @check_overflow: overlow flag for ccq - * @post_sq: flag for cqp db to ring - */ -static enum i40iw_status_code i40iw_sc_ccq_create(struct i40iw_sc_cq *ccq, - u64 scratch, - bool check_overflow, - bool post_sq) -{ - u64 *wqe; - struct i40iw_sc_cqp *cqp; - u64 header; - enum i40iw_status_code ret_code; - - cqp = ccq->dev->cqp; - wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch); - if (!wqe) - return I40IW_ERR_RING_FULL; - set_64bit_val(wqe, 0, ccq->cq_uk.cq_size); - set_64bit_val(wqe, 8, RS_64_1(ccq, 1)); - set_64bit_val(wqe, 16, - LS_64(ccq->shadow_read_threshold, I40IW_CQPSQ_CQ_SHADOW_READ_THRESHOLD)); - set_64bit_val(wqe, 32, (ccq->virtual_map ? 0 : ccq->cq_pa)); - set_64bit_val(wqe, 40, ccq->shadow_area_pa); - set_64bit_val(wqe, 48, - (ccq->virtual_map ? ccq->first_pm_pbl_idx : 0)); - set_64bit_val(wqe, 56, - LS_64(ccq->tph_val, I40IW_CQPSQ_TPHVAL)); - - header = ccq->cq_uk.cq_id | - LS_64((ccq->ceq_id_valid ? ccq->ceq_id : 0), I40IW_CQPSQ_CQ_CEQID) | - LS_64(I40IW_CQP_OP_CREATE_CQ, I40IW_CQPSQ_OPCODE) | - LS_64(ccq->pbl_chunk_size, I40IW_CQPSQ_CQ_LPBLSIZE) | - LS_64(check_overflow, I40IW_CQPSQ_CQ_CHKOVERFLOW) | - LS_64(ccq->virtual_map, I40IW_CQPSQ_CQ_VIRTMAP) | - LS_64(ccq->ceqe_mask, I40IW_CQPSQ_CQ_ENCEQEMASK) | - LS_64(ccq->ceq_id_valid, I40IW_CQPSQ_CQ_CEQIDVALID) | - LS_64(ccq->tph_en, I40IW_CQPSQ_TPHEN) | - LS_64(ccq->cq_uk.avoid_mem_cflct, I40IW_CQPSQ_CQ_AVOIDMEMCNFLCT) | - LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID); - - i40iw_insert_wqe_hdr(wqe, header); - - i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "CCQ_CREATE WQE", - wqe, I40IW_CQP_WQE_SIZE * 8); - - if (post_sq) { - i40iw_sc_cqp_post_sq(cqp); - ret_code = i40iw_sc_ccq_create_done(ccq); - if (ret_code) - return ret_code; - } - cqp->process_cqp_sds = i40iw_cqp_sds_cmd; - - return 0; -} - -/** - * i40iw_sc_ccq_destroy - destroy ccq during close - * @ccq: ccq sc struct - * @scratch: u64 saved to be used during cqp completion - * @post_sq: flag for cqp db to ring - */ -static enum i40iw_status_code i40iw_sc_ccq_destroy(struct i40iw_sc_cq *ccq, - u64 scratch, - bool post_sq) -{ - struct i40iw_sc_cqp *cqp; - u64 *wqe; - u64 header; - enum i40iw_status_code ret_code = 0; - u32 tail, val, error; - - cqp = ccq->dev->cqp; - wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch); - if (!wqe) - return I40IW_ERR_RING_FULL; - set_64bit_val(wqe, 0, ccq->cq_uk.cq_size); - set_64bit_val(wqe, 8, RS_64_1(ccq, 1)); - set_64bit_val(wqe, 40, ccq->shadow_area_pa); - - header = ccq->cq_uk.cq_id | - LS_64((ccq->ceq_id_valid ? ccq->ceq_id : 0), I40IW_CQPSQ_CQ_CEQID) | - LS_64(I40IW_CQP_OP_DESTROY_CQ, I40IW_CQPSQ_OPCODE) | - LS_64(ccq->ceqe_mask, I40IW_CQPSQ_CQ_ENCEQEMASK) | - LS_64(ccq->ceq_id_valid, I40IW_CQPSQ_CQ_CEQIDVALID) | - LS_64(ccq->tph_en, I40IW_CQPSQ_TPHEN) | - LS_64(ccq->cq_uk.avoid_mem_cflct, I40IW_CQPSQ_CQ_AVOIDMEMCNFLCT) | - LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID); - - i40iw_insert_wqe_hdr(wqe, header); - - i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "CCQ_DESTROY WQE", - wqe, I40IW_CQP_WQE_SIZE * 8); - - i40iw_get_cqp_reg_info(cqp, &val, &tail, &error); - if (error) - return I40IW_ERR_CQP_COMPL_ERROR; - - if (post_sq) { - i40iw_sc_cqp_post_sq(cqp); - ret_code = i40iw_cqp_poll_registers(cqp, tail, 1000); - } - - cqp->process_cqp_sds = i40iw_update_sds_noccq; - - return ret_code; -} - -/** - * i40iw_sc_cq_init - initialize completion q - * @cq: cq struct - * @info: cq initialization info - */ -static enum i40iw_status_code i40iw_sc_cq_init(struct i40iw_sc_cq *cq, - struct i40iw_cq_init_info *info) -{ - u32 __iomem *cqe_alloc_reg = NULL; - enum i40iw_status_code ret_code; - u32 pble_obj_cnt; - u32 arm_offset; - - pble_obj_cnt = info->dev->hmc_info->hmc_obj[I40IW_HMC_IW_PBLE].cnt; - - if (info->virtual_map && (info->first_pm_pbl_idx >= pble_obj_cnt)) - return I40IW_ERR_INVALID_PBLE_INDEX; - - cq->cq_pa = info->cq_base_pa; - cq->dev = info->dev; - cq->ceq_id = info->ceq_id; - arm_offset = (info->dev->is_pf) ? I40E_PFPE_CQARM : I40E_VFPE_CQARM1; - if (i40iw_get_hw_addr(cq->dev)) - cqe_alloc_reg = (u32 __iomem *)(i40iw_get_hw_addr(cq->dev) + - arm_offset); - info->cq_uk_init_info.cqe_alloc_reg = cqe_alloc_reg; - ret_code = i40iw_cq_uk_init(&cq->cq_uk, &info->cq_uk_init_info); - if (ret_code) - return ret_code; - cq->virtual_map = info->virtual_map; - cq->pbl_chunk_size = info->pbl_chunk_size; - cq->ceqe_mask = info->ceqe_mask; - cq->cq_type = (info->type) ? info->type : I40IW_CQ_TYPE_IWARP; - - cq->shadow_area_pa = info->shadow_area_pa; - cq->shadow_read_threshold = info->shadow_read_threshold; - - cq->ceq_id_valid = info->ceq_id_valid; - cq->tph_en = info->tph_en; - cq->tph_val = info->tph_val; - - cq->first_pm_pbl_idx = info->first_pm_pbl_idx; - - return 0; -} - -/** - * i40iw_sc_cq_create - create completion q - * @cq: cq struct - * @scratch: u64 saved to be used during cqp completion - * @check_overflow: flag for overflow check - * @post_sq: flag for cqp db to ring - */ -static enum i40iw_status_code i40iw_sc_cq_create(struct i40iw_sc_cq *cq, - u64 scratch, - bool check_overflow, - bool post_sq) -{ - u64 *wqe; - struct i40iw_sc_cqp *cqp; - u64 header; - - if (cq->cq_uk.cq_id > I40IW_MAX_CQID) - return I40IW_ERR_INVALID_CQ_ID; - - if (cq->ceq_id > I40IW_MAX_CEQID) - return I40IW_ERR_INVALID_CEQ_ID; - - cqp = cq->dev->cqp; - wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch); - if (!wqe) - return I40IW_ERR_RING_FULL; - - set_64bit_val(wqe, 0, cq->cq_uk.cq_size); - set_64bit_val(wqe, 8, RS_64_1(cq, 1)); - set_64bit_val(wqe, - 16, - LS_64(cq->shadow_read_threshold, I40IW_CQPSQ_CQ_SHADOW_READ_THRESHOLD)); - - set_64bit_val(wqe, 32, (cq->virtual_map ? 0 : cq->cq_pa)); - - set_64bit_val(wqe, 40, cq->shadow_area_pa); - set_64bit_val(wqe, 48, (cq->virtual_map ? cq->first_pm_pbl_idx : 0)); - set_64bit_val(wqe, 56, LS_64(cq->tph_val, I40IW_CQPSQ_TPHVAL)); - - header = cq->cq_uk.cq_id | - LS_64((cq->ceq_id_valid ? cq->ceq_id : 0), I40IW_CQPSQ_CQ_CEQID) | - LS_64(I40IW_CQP_OP_CREATE_CQ, I40IW_CQPSQ_OPCODE) | - LS_64(cq->pbl_chunk_size, I40IW_CQPSQ_CQ_LPBLSIZE) | - LS_64(check_overflow, I40IW_CQPSQ_CQ_CHKOVERFLOW) | - LS_64(cq->virtual_map, I40IW_CQPSQ_CQ_VIRTMAP) | - LS_64(cq->ceqe_mask, I40IW_CQPSQ_CQ_ENCEQEMASK) | - LS_64(cq->ceq_id_valid, I40IW_CQPSQ_CQ_CEQIDVALID) | - LS_64(cq->tph_en, I40IW_CQPSQ_TPHEN) | - LS_64(cq->cq_uk.avoid_mem_cflct, I40IW_CQPSQ_CQ_AVOIDMEMCNFLCT) | - LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID); - - i40iw_insert_wqe_hdr(wqe, header); - - i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "CQ_CREATE WQE", - wqe, I40IW_CQP_WQE_SIZE * 8); - - if (post_sq) - i40iw_sc_cqp_post_sq(cqp); - return 0; -} - -/** - * i40iw_sc_cq_destroy - destroy completion q - * @cq: cq struct - * @scratch: u64 saved to be used during cqp completion - * @post_sq: flag for cqp db to ring - */ -static enum i40iw_status_code i40iw_sc_cq_destroy(struct i40iw_sc_cq *cq, - u64 scratch, - bool post_sq) -{ - struct i40iw_sc_cqp *cqp; - u64 *wqe; - u64 header; - - cqp = cq->dev->cqp; - wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch); - if (!wqe) - return I40IW_ERR_RING_FULL; - set_64bit_val(wqe, 0, cq->cq_uk.cq_size); - set_64bit_val(wqe, 8, RS_64_1(cq, 1)); - set_64bit_val(wqe, 40, cq->shadow_area_pa); - set_64bit_val(wqe, 48, (cq->virtual_map ? cq->first_pm_pbl_idx : 0)); - - header = cq->cq_uk.cq_id | - LS_64((cq->ceq_id_valid ? cq->ceq_id : 0), I40IW_CQPSQ_CQ_CEQID) | - LS_64(I40IW_CQP_OP_DESTROY_CQ, I40IW_CQPSQ_OPCODE) | - LS_64(cq->pbl_chunk_size, I40IW_CQPSQ_CQ_LPBLSIZE) | - LS_64(cq->virtual_map, I40IW_CQPSQ_CQ_VIRTMAP) | - LS_64(cq->ceqe_mask, I40IW_CQPSQ_CQ_ENCEQEMASK) | - LS_64(cq->ceq_id_valid, I40IW_CQPSQ_CQ_CEQIDVALID) | - LS_64(cq->tph_en, I40IW_CQPSQ_TPHEN) | - LS_64(cq->cq_uk.avoid_mem_cflct, I40IW_CQPSQ_CQ_AVOIDMEMCNFLCT) | - LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID); - - i40iw_insert_wqe_hdr(wqe, header); - - i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "CQ_DESTROY WQE", - wqe, I40IW_CQP_WQE_SIZE * 8); - - if (post_sq) - i40iw_sc_cqp_post_sq(cqp); - return 0; -} - -/** - * i40iw_sc_cq_modify - modify a Completion Queue - * @cq: cq struct - * @info: modification info struct - * @scratch: u64 saved to be used during cqp completion - * @post_sq: flag to post to sq - */ -static enum i40iw_status_code i40iw_sc_cq_modify(struct i40iw_sc_cq *cq, - struct i40iw_modify_cq_info *info, - u64 scratch, - bool post_sq) -{ - struct i40iw_sc_cqp *cqp; - u64 *wqe; - u64 header; - u32 cq_size, ceq_id, first_pm_pbl_idx; - u8 pbl_chunk_size; - bool virtual_map, ceq_id_valid, check_overflow; - u32 pble_obj_cnt; - - if (info->ceq_valid && (info->ceq_id > I40IW_MAX_CEQID)) - return I40IW_ERR_INVALID_CEQ_ID; - - pble_obj_cnt = cq->dev->hmc_info->hmc_obj[I40IW_HMC_IW_PBLE].cnt; - - if (info->cq_resize && info->virtual_map && - (info->first_pm_pbl_idx >= pble_obj_cnt)) - return I40IW_ERR_INVALID_PBLE_INDEX; - - cqp = cq->dev->cqp; - wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch); - if (!wqe) - return I40IW_ERR_RING_FULL; - - cq->pbl_list = info->pbl_list; - cq->cq_pa = info->cq_pa; - cq->first_pm_pbl_idx = info->first_pm_pbl_idx; - - cq_size = info->cq_resize ? info->cq_size : cq->cq_uk.cq_size; - if (info->ceq_change) { - ceq_id_valid = true; - ceq_id = info->ceq_id; - } else { - ceq_id_valid = cq->ceq_id_valid; - ceq_id = ceq_id_valid ? cq->ceq_id : 0; - } - virtual_map = info->cq_resize ? info->virtual_map : cq->virtual_map; - first_pm_pbl_idx = (info->cq_resize ? - (info->virtual_map ? info->first_pm_pbl_idx : 0) : - (cq->virtual_map ? cq->first_pm_pbl_idx : 0)); - pbl_chunk_size = (info->cq_resize ? - (info->virtual_map ? info->pbl_chunk_size : 0) : - (cq->virtual_map ? cq->pbl_chunk_size : 0)); - check_overflow = info->check_overflow_change ? info->check_overflow : - cq->check_overflow; - cq->cq_uk.cq_size = cq_size; - cq->ceq_id_valid = ceq_id_valid; - cq->ceq_id = ceq_id; - cq->virtual_map = virtual_map; - cq->first_pm_pbl_idx = first_pm_pbl_idx; - cq->pbl_chunk_size = pbl_chunk_size; - cq->check_overflow = check_overflow; - - set_64bit_val(wqe, 0, cq_size); - set_64bit_val(wqe, 8, RS_64_1(cq, 1)); - set_64bit_val(wqe, 16, - LS_64(info->shadow_read_threshold, I40IW_CQPSQ_CQ_SHADOW_READ_THRESHOLD)); - set_64bit_val(wqe, 32, (cq->virtual_map ? 0 : cq->cq_pa)); - set_64bit_val(wqe, 40, cq->shadow_area_pa); - set_64bit_val(wqe, 48, (cq->virtual_map ? first_pm_pbl_idx : 0)); - set_64bit_val(wqe, 56, LS_64(cq->tph_val, I40IW_CQPSQ_TPHVAL)); - - header = cq->cq_uk.cq_id | - LS_64(ceq_id, I40IW_CQPSQ_CQ_CEQID) | - LS_64(I40IW_CQP_OP_MODIFY_CQ, I40IW_CQPSQ_OPCODE) | - LS_64(info->cq_resize, I40IW_CQPSQ_CQ_CQRESIZE) | - LS_64(pbl_chunk_size, I40IW_CQPSQ_CQ_LPBLSIZE) | - LS_64(check_overflow, I40IW_CQPSQ_CQ_CHKOVERFLOW) | - LS_64(virtual_map, I40IW_CQPSQ_CQ_VIRTMAP) | - LS_64(cq->ceqe_mask, I40IW_CQPSQ_CQ_ENCEQEMASK) | - LS_64(ceq_id_valid, I40IW_CQPSQ_CQ_CEQIDVALID) | - LS_64(cq->tph_en, I40IW_CQPSQ_TPHEN) | - LS_64(cq->cq_uk.avoid_mem_cflct, I40IW_CQPSQ_CQ_AVOIDMEMCNFLCT) | - LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID); - - i40iw_insert_wqe_hdr(wqe, header); - - i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "CQ_MODIFY WQE", - wqe, I40IW_CQP_WQE_SIZE * 8); - - if (post_sq) - i40iw_sc_cqp_post_sq(cqp); - return 0; -} - -/** - * i40iw_sc_qp_init - initialize qp - * @qp: sc qp - * @info: initialization qp info - */ -static enum i40iw_status_code i40iw_sc_qp_init(struct i40iw_sc_qp *qp, - struct i40iw_qp_init_info *info) -{ - u32 __iomem *wqe_alloc_reg = NULL; - enum i40iw_status_code ret_code; - u32 pble_obj_cnt; - u8 wqe_size; - u32 offset; - - qp->dev = info->pd->dev; - qp->vsi = info->vsi; - qp->sq_pa = info->sq_pa; - qp->rq_pa = info->rq_pa; - qp->hw_host_ctx_pa = info->host_ctx_pa; - qp->q2_pa = info->q2_pa; - qp->shadow_area_pa = info->shadow_area_pa; - - qp->q2_buf = info->q2; - qp->pd = info->pd; - qp->hw_host_ctx = info->host_ctx; - offset = (qp->pd->dev->is_pf) ? I40E_PFPE_WQEALLOC : I40E_VFPE_WQEALLOC1; - if (i40iw_get_hw_addr(qp->pd->dev)) - wqe_alloc_reg = (u32 __iomem *)(i40iw_get_hw_addr(qp->pd->dev) + - offset); - - info->qp_uk_init_info.wqe_alloc_reg = wqe_alloc_reg; - info->qp_uk_init_info.abi_ver = qp->pd->abi_ver; - ret_code = i40iw_qp_uk_init(&qp->qp_uk, &info->qp_uk_init_info); - if (ret_code) - return ret_code; - qp->virtual_map = info->virtual_map; - - pble_obj_cnt = info->pd->dev->hmc_info->hmc_obj[I40IW_HMC_IW_PBLE].cnt; - - if ((info->virtual_map && (info->sq_pa >= pble_obj_cnt)) || - (info->virtual_map && (info->rq_pa >= pble_obj_cnt))) - return I40IW_ERR_INVALID_PBLE_INDEX; - - qp->llp_stream_handle = (void *)(-1); - qp->qp_type = (info->type) ? info->type : I40IW_QP_TYPE_IWARP; - - qp->hw_sq_size = i40iw_get_encoded_wqe_size(qp->qp_uk.sq_ring.size, - false); - i40iw_debug(qp->dev, I40IW_DEBUG_WQE, "%s: hw_sq_size[%04d] sq_ring.size[%04d]\n", - __func__, qp->hw_sq_size, qp->qp_uk.sq_ring.size); - - switch (qp->pd->abi_ver) { - case 4: - ret_code = i40iw_fragcnt_to_wqesize_rq(qp->qp_uk.max_rq_frag_cnt, - &wqe_size); - if (ret_code) - return ret_code; - break; - case 5: /* fallthrough until next ABI version */ - default: - if (qp->qp_uk.max_rq_frag_cnt > I40IW_MAX_WQ_FRAGMENT_COUNT) - return I40IW_ERR_INVALID_FRAG_COUNT; - wqe_size = I40IW_MAX_WQE_SIZE_RQ; - break; - } - qp->hw_rq_size = i40iw_get_encoded_wqe_size(qp->qp_uk.rq_size * - (wqe_size / I40IW_QP_WQE_MIN_SIZE), false); - i40iw_debug(qp->dev, I40IW_DEBUG_WQE, - "%s: hw_rq_size[%04d] qp_uk.rq_size[%04d] wqe_size[%04d]\n", - __func__, qp->hw_rq_size, qp->qp_uk.rq_size, wqe_size); - qp->sq_tph_val = info->sq_tph_val; - qp->rq_tph_val = info->rq_tph_val; - qp->sq_tph_en = info->sq_tph_en; - qp->rq_tph_en = info->rq_tph_en; - qp->rcv_tph_en = info->rcv_tph_en; - qp->xmit_tph_en = info->xmit_tph_en; - qp->qs_handle = qp->vsi->qos[qp->user_pri].qs_handle; - - return 0; -} - -/** - * i40iw_sc_qp_create - create qp - * @qp: sc qp - * @info: qp create info - * @scratch: u64 saved to be used during cqp completion - * @post_sq: flag for cqp db to ring - */ -static enum i40iw_status_code i40iw_sc_qp_create( - struct i40iw_sc_qp *qp, - struct i40iw_create_qp_info *info, - u64 scratch, - bool post_sq) -{ - struct i40iw_sc_cqp *cqp; - u64 *wqe; - u64 header; - - if ((qp->qp_uk.qp_id < I40IW_MIN_IW_QP_ID) || - (qp->qp_uk.qp_id > I40IW_MAX_IW_QP_ID)) - return I40IW_ERR_INVALID_QP_ID; - - cqp = qp->pd->dev->cqp; - wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch); - if (!wqe) - return I40IW_ERR_RING_FULL; - - set_64bit_val(wqe, 16, qp->hw_host_ctx_pa); - - set_64bit_val(wqe, 40, qp->shadow_area_pa); - - header = qp->qp_uk.qp_id | - LS_64(I40IW_CQP_OP_CREATE_QP, I40IW_CQPSQ_OPCODE) | - LS_64((info->ord_valid ? 1 : 0), I40IW_CQPSQ_QP_ORDVALID) | - LS_64(info->tcp_ctx_valid, I40IW_CQPSQ_QP_TOECTXVALID) | - LS_64(qp->qp_type, I40IW_CQPSQ_QP_QPTYPE) | - LS_64(qp->virtual_map, I40IW_CQPSQ_QP_VQ) | - LS_64(info->cq_num_valid, I40IW_CQPSQ_QP_CQNUMVALID) | - LS_64(info->arp_cache_idx_valid, I40IW_CQPSQ_QP_ARPTABIDXVALID) | - LS_64(info->next_iwarp_state, I40IW_CQPSQ_QP_NEXTIWSTATE) | - LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID); - - i40iw_insert_wqe_hdr(wqe, header); - i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "QP_CREATE WQE", - wqe, I40IW_CQP_WQE_SIZE * 8); - - if (post_sq) - i40iw_sc_cqp_post_sq(cqp); - return 0; -} - -/** - * i40iw_sc_qp_modify - modify qp cqp wqe - * @qp: sc qp - * @info: modify qp info - * @scratch: u64 saved to be used during cqp completion - * @post_sq: flag for cqp db to ring - */ -static enum i40iw_status_code i40iw_sc_qp_modify( - struct i40iw_sc_qp *qp, - struct i40iw_modify_qp_info *info, - u64 scratch, - bool post_sq) -{ - u64 *wqe; - struct i40iw_sc_cqp *cqp; - u64 header; - u8 term_actions = 0; - u8 term_len = 0; - - cqp = qp->pd->dev->cqp; - wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch); - if (!wqe) - return I40IW_ERR_RING_FULL; - if (info->next_iwarp_state == I40IW_QP_STATE_TERMINATE) { - if (info->dont_send_fin) - term_actions += I40IWQP_TERM_SEND_TERM_ONLY; - if (info->dont_send_term) - term_actions += I40IWQP_TERM_SEND_FIN_ONLY; - if ((term_actions == I40IWQP_TERM_SEND_TERM_AND_FIN) || - (term_actions == I40IWQP_TERM_SEND_TERM_ONLY)) - term_len = info->termlen; - } - - set_64bit_val(wqe, - 8, - LS_64(term_len, I40IW_CQPSQ_QP_TERMLEN)); - - set_64bit_val(wqe, 16, qp->hw_host_ctx_pa); - set_64bit_val(wqe, 40, qp->shadow_area_pa); - - header = qp->qp_uk.qp_id | - LS_64(I40IW_CQP_OP_MODIFY_QP, I40IW_CQPSQ_OPCODE) | - LS_64(info->ord_valid, I40IW_CQPSQ_QP_ORDVALID) | - LS_64(info->tcp_ctx_valid, I40IW_CQPSQ_QP_TOECTXVALID) | - LS_64(info->cached_var_valid, I40IW_CQPSQ_QP_CACHEDVARVALID) | - LS_64(qp->virtual_map, I40IW_CQPSQ_QP_VQ) | - LS_64(info->cq_num_valid, I40IW_CQPSQ_QP_CQNUMVALID) | - LS_64(info->force_loopback, I40IW_CQPSQ_QP_FORCELOOPBACK) | - LS_64(qp->qp_type, I40IW_CQPSQ_QP_QPTYPE) | - LS_64(info->remove_hash_idx, I40IW_CQPSQ_QP_REMOVEHASHENTRY) | - LS_64(term_actions, I40IW_CQPSQ_QP_TERMACT) | - LS_64(info->reset_tcp_conn, I40IW_CQPSQ_QP_RESETCON) | - LS_64(info->arp_cache_idx_valid, I40IW_CQPSQ_QP_ARPTABIDXVALID) | - LS_64(info->next_iwarp_state, I40IW_CQPSQ_QP_NEXTIWSTATE) | - LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID); - - i40iw_insert_wqe_hdr(wqe, header); - - i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "QP_MODIFY WQE", - wqe, I40IW_CQP_WQE_SIZE * 8); - - if (post_sq) - i40iw_sc_cqp_post_sq(cqp); - return 0; -} - -/** - * i40iw_sc_qp_destroy - cqp destroy qp - * @qp: sc qp - * @scratch: u64 saved to be used during cqp completion - * @remove_hash_idx: flag if to remove hash idx - * @ignore_mw_bnd: memory window bind flag - * @post_sq: flag for cqp db to ring - */ -static enum i40iw_status_code i40iw_sc_qp_destroy( - struct i40iw_sc_qp *qp, - u64 scratch, - bool remove_hash_idx, - bool ignore_mw_bnd, - bool post_sq) -{ - u64 *wqe; - struct i40iw_sc_cqp *cqp; - u64 header; - - i40iw_qp_rem_qos(qp); - cqp = qp->pd->dev->cqp; - wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch); - if (!wqe) - return I40IW_ERR_RING_FULL; - set_64bit_val(wqe, 16, qp->hw_host_ctx_pa); - set_64bit_val(wqe, 40, qp->shadow_area_pa); - - header = qp->qp_uk.qp_id | - LS_64(I40IW_CQP_OP_DESTROY_QP, I40IW_CQPSQ_OPCODE) | - LS_64(qp->qp_type, I40IW_CQPSQ_QP_QPTYPE) | - LS_64(ignore_mw_bnd, I40IW_CQPSQ_QP_IGNOREMWBOUND) | - LS_64(remove_hash_idx, I40IW_CQPSQ_QP_REMOVEHASHENTRY) | - LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID); - - i40iw_insert_wqe_hdr(wqe, header); - i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "QP_DESTROY WQE", - wqe, I40IW_CQP_WQE_SIZE * 8); - - if (post_sq) - i40iw_sc_cqp_post_sq(cqp); - return 0; -} - -/** - * i40iw_sc_qp_flush_wqes - flush qp's wqe - * @qp: sc qp - * @info: dlush information - * @scratch: u64 saved to be used during cqp completion - * @post_sq: flag for cqp db to ring - */ -static enum i40iw_status_code i40iw_sc_qp_flush_wqes( - struct i40iw_sc_qp *qp, - struct i40iw_qp_flush_info *info, - u64 scratch, - bool post_sq) -{ - u64 temp = 0; - u64 *wqe; - struct i40iw_sc_cqp *cqp; - u64 header; - bool flush_sq = false, flush_rq = false; - - if (info->rq && !qp->flush_rq) - flush_rq = true; - - if (info->sq && !qp->flush_sq) - flush_sq = true; - - qp->flush_sq |= flush_sq; - qp->flush_rq |= flush_rq; - if (!flush_sq && !flush_rq) - return 0; - - cqp = qp->pd->dev->cqp; - wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch); - if (!wqe) - return I40IW_ERR_RING_FULL; - if (info->userflushcode) { - if (flush_rq) { - temp |= LS_64(info->rq_minor_code, I40IW_CQPSQ_FWQE_RQMNERR) | - LS_64(info->rq_major_code, I40IW_CQPSQ_FWQE_RQMJERR); - } - if (flush_sq) { - temp |= LS_64(info->sq_minor_code, I40IW_CQPSQ_FWQE_SQMNERR) | - LS_64(info->sq_major_code, I40IW_CQPSQ_FWQE_SQMJERR); - } - } - set_64bit_val(wqe, 16, temp); - - temp = (info->generate_ae) ? - info->ae_code | LS_64(info->ae_source, I40IW_CQPSQ_FWQE_AESOURCE) : 0; - - set_64bit_val(wqe, 8, temp); - - header = qp->qp_uk.qp_id | - LS_64(I40IW_CQP_OP_FLUSH_WQES, I40IW_CQPSQ_OPCODE) | - LS_64(info->generate_ae, I40IW_CQPSQ_FWQE_GENERATE_AE) | - LS_64(info->userflushcode, I40IW_CQPSQ_FWQE_USERFLCODE) | - LS_64(flush_sq, I40IW_CQPSQ_FWQE_FLUSHSQ) | - LS_64(flush_rq, I40IW_CQPSQ_FWQE_FLUSHRQ) | - LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID); - - i40iw_insert_wqe_hdr(wqe, header); - - i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "QP_FLUSH WQE", - wqe, I40IW_CQP_WQE_SIZE * 8); - - if (post_sq) - i40iw_sc_cqp_post_sq(cqp); - return 0; -} - -/** - * i40iw_sc_gen_ae - generate AE, currently uses flush WQE CQP OP - * @qp: sc qp - * @info: gen ae information - * @scratch: u64 saved to be used during cqp completion - * @post_sq: flag for cqp db to ring - */ -static enum i40iw_status_code i40iw_sc_gen_ae( - struct i40iw_sc_qp *qp, - struct i40iw_gen_ae_info *info, - u64 scratch, - bool post_sq) -{ - u64 temp; - u64 *wqe; - struct i40iw_sc_cqp *cqp; - u64 header; - - cqp = qp->pd->dev->cqp; - wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch); - if (!wqe) - return I40IW_ERR_RING_FULL; - - temp = info->ae_code | - LS_64(info->ae_source, I40IW_CQPSQ_FWQE_AESOURCE); - - set_64bit_val(wqe, 8, temp); - - header = qp->qp_uk.qp_id | - LS_64(I40IW_CQP_OP_GEN_AE, I40IW_CQPSQ_OPCODE) | - LS_64(1, I40IW_CQPSQ_FWQE_GENERATE_AE) | - LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID); - - i40iw_insert_wqe_hdr(wqe, header); - - i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "GEN_AE WQE", - wqe, I40IW_CQP_WQE_SIZE * 8); - - if (post_sq) - i40iw_sc_cqp_post_sq(cqp); - return 0; -} - -/** - * i40iw_sc_qp_upload_context - upload qp's context - * @dev: sc device struct - * @info: upload context info ptr for return - * @scratch: u64 saved to be used during cqp completion - * @post_sq: flag for cqp db to ring - */ -static enum i40iw_status_code i40iw_sc_qp_upload_context( - struct i40iw_sc_dev *dev, - struct i40iw_upload_context_info *info, - u64 scratch, - bool post_sq) -{ - u64 *wqe; - struct i40iw_sc_cqp *cqp; - u64 header; - - cqp = dev->cqp; - wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch); - if (!wqe) - return I40IW_ERR_RING_FULL; - set_64bit_val(wqe, 16, info->buf_pa); - - header = LS_64(info->qp_id, I40IW_CQPSQ_UCTX_QPID) | - LS_64(I40IW_CQP_OP_UPLOAD_CONTEXT, I40IW_CQPSQ_OPCODE) | - LS_64(info->qp_type, I40IW_CQPSQ_UCTX_QPTYPE) | - LS_64(info->raw_format, I40IW_CQPSQ_UCTX_RAWFORMAT) | - LS_64(info->freeze_qp, I40IW_CQPSQ_UCTX_FREEZEQP) | - LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID); - - i40iw_insert_wqe_hdr(wqe, header); - - i40iw_debug_buf(dev, I40IW_DEBUG_WQE, "QP_UPLOAD_CTX WQE", - wqe, I40IW_CQP_WQE_SIZE * 8); - - if (post_sq) - i40iw_sc_cqp_post_sq(cqp); - return 0; -} - -/** - * i40iw_sc_qp_setctx - set qp's context - * @qp: sc qp - * @qp_ctx: context ptr - * @info: ctx info - */ -static enum i40iw_status_code i40iw_sc_qp_setctx( - struct i40iw_sc_qp *qp, - u64 *qp_ctx, - struct i40iw_qp_host_ctx_info *info) -{ - struct i40iwarp_offload_info *iw; - struct i40iw_tcp_offload_info *tcp; - struct i40iw_sc_vsi *vsi; - struct i40iw_sc_dev *dev; - u64 qw0, qw3, qw7 = 0; - - iw = info->iwarp_info; - tcp = info->tcp_info; - vsi = qp->vsi; - dev = qp->dev; - if (info->add_to_qoslist) { - qp->user_pri = info->user_pri; - i40iw_qp_add_qos(qp); - i40iw_debug(qp->dev, I40IW_DEBUG_DCB, "%s qp[%d] UP[%d] qset[%d]\n", - __func__, qp->qp_uk.qp_id, qp->user_pri, qp->qs_handle); - } - qw0 = LS_64(qp->qp_uk.rq_wqe_size, I40IWQPC_RQWQESIZE) | - LS_64(info->err_rq_idx_valid, I40IWQPC_ERR_RQ_IDX_VALID) | - LS_64(qp->rcv_tph_en, I40IWQPC_RCVTPHEN) | - LS_64(qp->xmit_tph_en, I40IWQPC_XMITTPHEN) | - LS_64(qp->rq_tph_en, I40IWQPC_RQTPHEN) | - LS_64(qp->sq_tph_en, I40IWQPC_SQTPHEN); - - set_64bit_val(qp_ctx, 8, qp->sq_pa); - set_64bit_val(qp_ctx, 16, qp->rq_pa); - - qw3 = LS_64(qp->src_mac_addr_idx, I40IWQPC_SRCMACADDRIDX) | - LS_64(qp->hw_rq_size, I40IWQPC_RQSIZE) | - LS_64(qp->hw_sq_size, I40IWQPC_SQSIZE); - - set_64bit_val(qp_ctx, - 128, - LS_64(info->err_rq_idx, I40IWQPC_ERR_RQ_IDX)); - - set_64bit_val(qp_ctx, - 136, - LS_64(info->send_cq_num, I40IWQPC_TXCQNUM) | - LS_64(info->rcv_cq_num, I40IWQPC_RXCQNUM)); - - set_64bit_val(qp_ctx, - 168, - LS_64(info->qp_compl_ctx, I40IWQPC_QPCOMPCTX)); - set_64bit_val(qp_ctx, - 176, - LS_64(qp->sq_tph_val, I40IWQPC_SQTPHVAL) | - LS_64(qp->rq_tph_val, I40IWQPC_RQTPHVAL) | - LS_64(qp->qs_handle, I40IWQPC_QSHANDLE) | - LS_64(vsi->exception_lan_queue, I40IWQPC_EXCEPTION_LAN_QUEUE)); - - if (info->iwarp_info_valid) { - qw0 |= LS_64(iw->ddp_ver, I40IWQPC_DDP_VER) | - LS_64(iw->rdmap_ver, I40IWQPC_RDMAP_VER); - - qw7 |= LS_64(iw->pd_id, I40IWQPC_PDIDX); - set_64bit_val(qp_ctx, - 144, - LS_64(qp->q2_pa, I40IWQPC_Q2ADDR) | - LS_64(vsi->fcn_id, I40IWQPC_STAT_INDEX)); - set_64bit_val(qp_ctx, - 152, - LS_64(iw->last_byte_sent, I40IWQPC_LASTBYTESENT)); - - set_64bit_val(qp_ctx, - 160, - LS_64(iw->ord_size, I40IWQPC_ORDSIZE) | - LS_64(iw->ird_size, I40IWQPC_IRDSIZE) | - LS_64(iw->wr_rdresp_en, I40IWQPC_WRRDRSPOK) | - LS_64(iw->rd_enable, I40IWQPC_RDOK) | - LS_64(iw->snd_mark_en, I40IWQPC_SNDMARKERS) | - LS_64(iw->bind_en, I40IWQPC_BINDEN) | - LS_64(iw->fast_reg_en, I40IWQPC_FASTREGEN) | - LS_64(iw->priv_mode_en, I40IWQPC_PRIVEN) | - LS_64((((vsi->stats_fcn_id_alloc) && - (dev->is_pf) && (vsi->fcn_id >= I40IW_FIRST_NON_PF_STAT)) ? 1 : 0), - I40IWQPC_USESTATSINSTANCE) | - LS_64(1, I40IWQPC_IWARPMODE) | - LS_64(iw->rcv_mark_en, I40IWQPC_RCVMARKERS) | - LS_64(iw->align_hdrs, I40IWQPC_ALIGNHDRS) | - LS_64(iw->rcv_no_mpa_crc, I40IWQPC_RCVNOMPACRC) | - LS_64(iw->rcv_mark_offset, I40IWQPC_RCVMARKOFFSET) | - LS_64(iw->snd_mark_offset, I40IWQPC_SNDMARKOFFSET)); - } - if (info->tcp_info_valid) { - qw0 |= LS_64(tcp->ipv4, I40IWQPC_IPV4) | - LS_64(tcp->no_nagle, I40IWQPC_NONAGLE) | - LS_64(tcp->insert_vlan_tag, I40IWQPC_INSERTVLANTAG) | - LS_64(tcp->time_stamp, I40IWQPC_TIMESTAMP) | - LS_64(tcp->cwnd_inc_limit, I40IWQPC_LIMIT) | - LS_64(tcp->drop_ooo_seg, I40IWQPC_DROPOOOSEG) | - LS_64(tcp->dup_ack_thresh, I40IWQPC_DUPACK_THRESH); - - qw3 |= LS_64(tcp->ttl, I40IWQPC_TTL) | - LS_64(tcp->src_mac_addr_idx, I40IWQPC_SRCMACADDRIDX) | - LS_64(tcp->avoid_stretch_ack, I40IWQPC_AVOIDSTRETCHACK) | - LS_64(tcp->tos, I40IWQPC_TOS) | - LS_64(tcp->src_port, I40IWQPC_SRCPORTNUM) | - LS_64(tcp->dst_port, I40IWQPC_DESTPORTNUM); - - qp->src_mac_addr_idx = tcp->src_mac_addr_idx; - set_64bit_val(qp_ctx, - 32, - LS_64(tcp->dest_ip_addr2, I40IWQPC_DESTIPADDR2) | - LS_64(tcp->dest_ip_addr3, I40IWQPC_DESTIPADDR3)); - - set_64bit_val(qp_ctx, - 40, - LS_64(tcp->dest_ip_addr0, I40IWQPC_DESTIPADDR0) | - LS_64(tcp->dest_ip_addr1, I40IWQPC_DESTIPADDR1)); - - set_64bit_val(qp_ctx, - 48, - LS_64(tcp->snd_mss, I40IWQPC_SNDMSS) | - LS_64(tcp->vlan_tag, I40IWQPC_VLANTAG) | - LS_64(tcp->arp_idx, I40IWQPC_ARPIDX)); - - qw7 |= LS_64(tcp->flow_label, I40IWQPC_FLOWLABEL) | - LS_64(tcp->wscale, I40IWQPC_WSCALE) | - LS_64(tcp->ignore_tcp_opt, I40IWQPC_IGNORE_TCP_OPT) | - LS_64(tcp->ignore_tcp_uns_opt, I40IWQPC_IGNORE_TCP_UNS_OPT) | - LS_64(tcp->tcp_state, I40IWQPC_TCPSTATE) | - LS_64(tcp->rcv_wscale, I40IWQPC_RCVSCALE) | - LS_64(tcp->snd_wscale, I40IWQPC_SNDSCALE); - - set_64bit_val(qp_ctx, - 72, - LS_64(tcp->time_stamp_recent, I40IWQPC_TIMESTAMP_RECENT) | - LS_64(tcp->time_stamp_age, I40IWQPC_TIMESTAMP_AGE)); - set_64bit_val(qp_ctx, - 80, - LS_64(tcp->snd_nxt, I40IWQPC_SNDNXT) | - LS_64(tcp->snd_wnd, I40IWQPC_SNDWND)); - - set_64bit_val(qp_ctx, - 88, - LS_64(tcp->rcv_nxt, I40IWQPC_RCVNXT) | - LS_64(tcp->rcv_wnd, I40IWQPC_RCVWND)); - set_64bit_val(qp_ctx, - 96, - LS_64(tcp->snd_max, I40IWQPC_SNDMAX) | - LS_64(tcp->snd_una, I40IWQPC_SNDUNA)); - set_64bit_val(qp_ctx, - 104, - LS_64(tcp->srtt, I40IWQPC_SRTT) | - LS_64(tcp->rtt_var, I40IWQPC_RTTVAR)); - set_64bit_val(qp_ctx, - 112, - LS_64(tcp->ss_thresh, I40IWQPC_SSTHRESH) | - LS_64(tcp->cwnd, I40IWQPC_CWND)); - set_64bit_val(qp_ctx, - 120, - LS_64(tcp->snd_wl1, I40IWQPC_SNDWL1) | - LS_64(tcp->snd_wl2, I40IWQPC_SNDWL2)); - set_64bit_val(qp_ctx, - 128, - LS_64(tcp->max_snd_window, I40IWQPC_MAXSNDWND) | - LS_64(tcp->rexmit_thresh, I40IWQPC_REXMIT_THRESH)); - set_64bit_val(qp_ctx, - 184, - LS_64(tcp->local_ipaddr3, I40IWQPC_LOCAL_IPADDR3) | - LS_64(tcp->local_ipaddr2, I40IWQPC_LOCAL_IPADDR2)); - set_64bit_val(qp_ctx, - 192, - LS_64(tcp->local_ipaddr1, I40IWQPC_LOCAL_IPADDR1) | - LS_64(tcp->local_ipaddr0, I40IWQPC_LOCAL_IPADDR0)); - } - - set_64bit_val(qp_ctx, 0, qw0); - set_64bit_val(qp_ctx, 24, qw3); - set_64bit_val(qp_ctx, 56, qw7); - - i40iw_debug_buf(qp->dev, I40IW_DEBUG_WQE, "QP_HOST)CTX WQE", - qp_ctx, I40IW_QP_CTX_SIZE); - return 0; -} - -/** - * i40iw_sc_alloc_stag - mr stag alloc - * @dev: sc device struct - * @info: stag info - * @scratch: u64 saved to be used during cqp completion - * @post_sq: flag for cqp db to ring - */ -static enum i40iw_status_code i40iw_sc_alloc_stag( - struct i40iw_sc_dev *dev, - struct i40iw_allocate_stag_info *info, - u64 scratch, - bool post_sq) -{ - u64 *wqe; - struct i40iw_sc_cqp *cqp; - u64 header; - enum i40iw_page_size page_size; - - page_size = (info->page_size == 0x200000) ? I40IW_PAGE_SIZE_2M : I40IW_PAGE_SIZE_4K; - cqp = dev->cqp; - wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch); - if (!wqe) - return I40IW_ERR_RING_FULL; - set_64bit_val(wqe, - 8, - LS_64(info->pd_id, I40IW_CQPSQ_STAG_PDID) | - LS_64(info->total_len, I40IW_CQPSQ_STAG_STAGLEN)); - set_64bit_val(wqe, - 16, - LS_64(info->stag_idx, I40IW_CQPSQ_STAG_IDX)); - set_64bit_val(wqe, - 40, - LS_64(info->hmc_fcn_index, I40IW_CQPSQ_STAG_HMCFNIDX)); - - header = LS_64(I40IW_CQP_OP_ALLOC_STAG, I40IW_CQPSQ_OPCODE) | - LS_64(1, I40IW_CQPSQ_STAG_MR) | - LS_64(info->access_rights, I40IW_CQPSQ_STAG_ARIGHTS) | - LS_64(info->chunk_size, I40IW_CQPSQ_STAG_LPBLSIZE) | - LS_64(page_size, I40IW_CQPSQ_STAG_HPAGESIZE) | - LS_64(info->remote_access, I40IW_CQPSQ_STAG_REMACCENABLED) | - LS_64(info->use_hmc_fcn_index, I40IW_CQPSQ_STAG_USEHMCFNIDX) | - LS_64(info->use_pf_rid, I40IW_CQPSQ_STAG_USEPFRID) | - LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID); - - i40iw_insert_wqe_hdr(wqe, header); - - i40iw_debug_buf(dev, I40IW_DEBUG_WQE, "ALLOC_STAG WQE", - wqe, I40IW_CQP_WQE_SIZE * 8); - - if (post_sq) - i40iw_sc_cqp_post_sq(cqp); - return 0; -} - -/** - * i40iw_sc_mr_reg_non_shared - non-shared mr registration - * @dev: sc device struct - * @info: mr info - * @scratch: u64 saved to be used during cqp completion - * @post_sq: flag for cqp db to ring - */ -static enum i40iw_status_code i40iw_sc_mr_reg_non_shared( - struct i40iw_sc_dev *dev, - struct i40iw_reg_ns_stag_info *info, - u64 scratch, - bool post_sq) -{ - u64 *wqe; - u64 temp; - struct i40iw_sc_cqp *cqp; - u64 header; - u32 pble_obj_cnt; - bool remote_access; - u8 addr_type; - enum i40iw_page_size page_size; - - page_size = (info->page_size == 0x200000) ? I40IW_PAGE_SIZE_2M : I40IW_PAGE_SIZE_4K; - if (info->access_rights & (I40IW_ACCESS_FLAGS_REMOTEREAD_ONLY | - I40IW_ACCESS_FLAGS_REMOTEWRITE_ONLY)) - remote_access = true; - else - remote_access = false; - - pble_obj_cnt = dev->hmc_info->hmc_obj[I40IW_HMC_IW_PBLE].cnt; - - if (info->chunk_size && (info->first_pm_pbl_index >= pble_obj_cnt)) - return I40IW_ERR_INVALID_PBLE_INDEX; - - cqp = dev->cqp; - wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch); - if (!wqe) - return I40IW_ERR_RING_FULL; - - temp = (info->addr_type == I40IW_ADDR_TYPE_VA_BASED) ? (uintptr_t)info->va : info->fbo; - set_64bit_val(wqe, 0, temp); - - set_64bit_val(wqe, - 8, - LS_64(info->total_len, I40IW_CQPSQ_STAG_STAGLEN) | - LS_64(info->pd_id, I40IW_CQPSQ_STAG_PDID)); - - set_64bit_val(wqe, - 16, - LS_64(info->stag_key, I40IW_CQPSQ_STAG_KEY) | - LS_64(info->stag_idx, I40IW_CQPSQ_STAG_IDX)); - if (!info->chunk_size) { - set_64bit_val(wqe, 32, info->reg_addr_pa); - set_64bit_val(wqe, 48, 0); - } else { - set_64bit_val(wqe, 32, 0); - set_64bit_val(wqe, 48, info->first_pm_pbl_index); - } - set_64bit_val(wqe, 40, info->hmc_fcn_index); - set_64bit_val(wqe, 56, 0); - - addr_type = (info->addr_type == I40IW_ADDR_TYPE_VA_BASED) ? 1 : 0; - header = LS_64(I40IW_CQP_OP_REG_MR, I40IW_CQPSQ_OPCODE) | - LS_64(1, I40IW_CQPSQ_STAG_MR) | - LS_64(info->chunk_size, I40IW_CQPSQ_STAG_LPBLSIZE) | - LS_64(page_size, I40IW_CQPSQ_STAG_HPAGESIZE) | - LS_64(info->access_rights, I40IW_CQPSQ_STAG_ARIGHTS) | - LS_64(remote_access, I40IW_CQPSQ_STAG_REMACCENABLED) | - LS_64(addr_type, I40IW_CQPSQ_STAG_VABASEDTO) | - LS_64(info->use_hmc_fcn_index, I40IW_CQPSQ_STAG_USEHMCFNIDX) | - LS_64(info->use_pf_rid, I40IW_CQPSQ_STAG_USEPFRID) | - LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID); - - i40iw_insert_wqe_hdr(wqe, header); - - i40iw_debug_buf(dev, I40IW_DEBUG_WQE, "MR_REG_NS WQE", - wqe, I40IW_CQP_WQE_SIZE * 8); - - if (post_sq) - i40iw_sc_cqp_post_sq(cqp); - return 0; -} - -/** - * i40iw_sc_mr_reg_shared - registered shared memory region - * @dev: sc device struct - * @info: info for shared memory registeration - * @scratch: u64 saved to be used during cqp completion - * @post_sq: flag for cqp db to ring - */ -static enum i40iw_status_code i40iw_sc_mr_reg_shared( - struct i40iw_sc_dev *dev, - struct i40iw_register_shared_stag *info, - u64 scratch, - bool post_sq) -{ - u64 *wqe; - struct i40iw_sc_cqp *cqp; - u64 temp, va64, fbo, header; - u32 va32; - bool remote_access; - u8 addr_type; - - if (info->access_rights & (I40IW_ACCESS_FLAGS_REMOTEREAD_ONLY | - I40IW_ACCESS_FLAGS_REMOTEWRITE_ONLY)) - remote_access = true; - else - remote_access = false; - cqp = dev->cqp; - wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch); - if (!wqe) - return I40IW_ERR_RING_FULL; - va64 = (uintptr_t)(info->va); - va32 = (u32)(va64 & 0x00000000FFFFFFFF); - fbo = (u64)(va32 & (4096 - 1)); - - set_64bit_val(wqe, - 0, - (info->addr_type == I40IW_ADDR_TYPE_VA_BASED ? (uintptr_t)info->va : fbo)); - - set_64bit_val(wqe, - 8, - LS_64(info->pd_id, I40IW_CQPSQ_STAG_PDID)); - temp = LS_64(info->new_stag_key, I40IW_CQPSQ_STAG_KEY) | - LS_64(info->new_stag_idx, I40IW_CQPSQ_STAG_IDX) | - LS_64(info->parent_stag_idx, I40IW_CQPSQ_STAG_PARENTSTAGIDX); - set_64bit_val(wqe, 16, temp); - - addr_type = (info->addr_type == I40IW_ADDR_TYPE_VA_BASED) ? 1 : 0; - header = LS_64(I40IW_CQP_OP_REG_SMR, I40IW_CQPSQ_OPCODE) | - LS_64(1, I40IW_CQPSQ_STAG_MR) | - LS_64(info->access_rights, I40IW_CQPSQ_STAG_ARIGHTS) | - LS_64(remote_access, I40IW_CQPSQ_STAG_REMACCENABLED) | - LS_64(addr_type, I40IW_CQPSQ_STAG_VABASEDTO) | - LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID); - - i40iw_insert_wqe_hdr(wqe, header); - - i40iw_debug_buf(dev, I40IW_DEBUG_WQE, "MR_REG_SHARED WQE", - wqe, I40IW_CQP_WQE_SIZE * 8); - - if (post_sq) - i40iw_sc_cqp_post_sq(cqp); - return 0; -} - -/** - * i40iw_sc_dealloc_stag - deallocate stag - * @dev: sc device struct - * @info: dealloc stag info - * @scratch: u64 saved to be used during cqp completion - * @post_sq: flag for cqp db to ring - */ -static enum i40iw_status_code i40iw_sc_dealloc_stag( - struct i40iw_sc_dev *dev, - struct i40iw_dealloc_stag_info *info, - u64 scratch, - bool post_sq) -{ - u64 header; - u64 *wqe; - struct i40iw_sc_cqp *cqp; - - cqp = dev->cqp; - wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch); - if (!wqe) - return I40IW_ERR_RING_FULL; - set_64bit_val(wqe, - 8, - LS_64(info->pd_id, I40IW_CQPSQ_STAG_PDID)); - set_64bit_val(wqe, - 16, - LS_64(info->stag_idx, I40IW_CQPSQ_STAG_IDX)); - - header = LS_64(I40IW_CQP_OP_DEALLOC_STAG, I40IW_CQPSQ_OPCODE) | - LS_64(info->mr, I40IW_CQPSQ_STAG_MR) | - LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID); - - i40iw_insert_wqe_hdr(wqe, header); - - i40iw_debug_buf(dev, I40IW_DEBUG_WQE, "DEALLOC_STAG WQE", - wqe, I40IW_CQP_WQE_SIZE * 8); - - if (post_sq) - i40iw_sc_cqp_post_sq(cqp); - return 0; -} - -/** - * i40iw_sc_query_stag - query hardware for stag - * @dev: sc device struct - * @scratch: u64 saved to be used during cqp completion - * @stag_index: stag index for query - * @post_sq: flag for cqp db to ring - */ -static enum i40iw_status_code i40iw_sc_query_stag(struct i40iw_sc_dev *dev, - u64 scratch, - u32 stag_index, - bool post_sq) -{ - u64 header; - u64 *wqe; - struct i40iw_sc_cqp *cqp; - - cqp = dev->cqp; - wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch); - if (!wqe) - return I40IW_ERR_RING_FULL; - set_64bit_val(wqe, - 16, - LS_64(stag_index, I40IW_CQPSQ_QUERYSTAG_IDX)); - - header = LS_64(I40IW_CQP_OP_QUERY_STAG, I40IW_CQPSQ_OPCODE) | - LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID); - - i40iw_insert_wqe_hdr(wqe, header); - - i40iw_debug_buf(dev, I40IW_DEBUG_WQE, "QUERY_STAG WQE", - wqe, I40IW_CQP_WQE_SIZE * 8); - - if (post_sq) - i40iw_sc_cqp_post_sq(cqp); - return 0; -} - -/** - * i40iw_sc_mw_alloc - mw allocate - * @dev: sc device struct - * @scratch: u64 saved to be used during cqp completion - * @mw_stag_index:stag index - * @pd_id: pd is for this mw - * @post_sq: flag for cqp db to ring - */ -static enum i40iw_status_code i40iw_sc_mw_alloc( - struct i40iw_sc_dev *dev, - u64 scratch, - u32 mw_stag_index, - u16 pd_id, - bool post_sq) -{ - u64 header; - struct i40iw_sc_cqp *cqp; - u64 *wqe; - - cqp = dev->cqp; - wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch); - if (!wqe) - return I40IW_ERR_RING_FULL; - set_64bit_val(wqe, 8, LS_64(pd_id, I40IW_CQPSQ_STAG_PDID)); - set_64bit_val(wqe, - 16, - LS_64(mw_stag_index, I40IW_CQPSQ_STAG_IDX)); - - header = LS_64(I40IW_CQP_OP_ALLOC_STAG, I40IW_CQPSQ_OPCODE) | - LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID); - - i40iw_insert_wqe_hdr(wqe, header); - - i40iw_debug_buf(dev, I40IW_DEBUG_WQE, "MW_ALLOC WQE", - wqe, I40IW_CQP_WQE_SIZE * 8); - - if (post_sq) - i40iw_sc_cqp_post_sq(cqp); - return 0; -} - -/** - * i40iw_sc_mr_fast_register - Posts RDMA fast register mr WR to iwarp qp - * @qp: sc qp struct - * @info: fast mr info - * @post_sq: flag for cqp db to ring - */ -enum i40iw_status_code i40iw_sc_mr_fast_register( - struct i40iw_sc_qp *qp, - struct i40iw_fast_reg_stag_info *info, - bool post_sq) -{ - u64 temp, header; - u64 *wqe; - u32 wqe_idx; - enum i40iw_page_size page_size; - - page_size = (info->page_size == 0x200000) ? I40IW_PAGE_SIZE_2M : I40IW_PAGE_SIZE_4K; - wqe = i40iw_qp_get_next_send_wqe(&qp->qp_uk, &wqe_idx, I40IW_QP_WQE_MIN_SIZE, - 0, info->wr_id); - if (!wqe) - return I40IW_ERR_QP_TOOMANY_WRS_POSTED; - - i40iw_debug(qp->dev, I40IW_DEBUG_MR, "%s: wr_id[%llxh] wqe_idx[%04d] location[%p]\n", - __func__, info->wr_id, wqe_idx, - &qp->qp_uk.sq_wrtrk_array[wqe_idx].wrid); - temp = (info->addr_type == I40IW_ADDR_TYPE_VA_BASED) ? (uintptr_t)info->va : info->fbo; - set_64bit_val(wqe, 0, temp); - - temp = RS_64(info->first_pm_pbl_index >> 16, I40IWQPSQ_FIRSTPMPBLIDXHI); - set_64bit_val(wqe, - 8, - LS_64(temp, I40IWQPSQ_FIRSTPMPBLIDXHI) | - LS_64(info->reg_addr_pa >> I40IWQPSQ_PBLADDR_SHIFT, I40IWQPSQ_PBLADDR)); - - set_64bit_val(wqe, - 16, - info->total_len | - LS_64(info->first_pm_pbl_index, I40IWQPSQ_FIRSTPMPBLIDXLO)); - - header = LS_64(info->stag_key, I40IWQPSQ_STAGKEY) | - LS_64(info->stag_idx, I40IWQPSQ_STAGINDEX) | - LS_64(I40IWQP_OP_FAST_REGISTER, I40IWQPSQ_OPCODE) | - LS_64(info->chunk_size, I40IWQPSQ_LPBLSIZE) | - LS_64(page_size, I40IWQPSQ_HPAGESIZE) | - LS_64(info->access_rights, I40IWQPSQ_STAGRIGHTS) | - LS_64(info->addr_type, I40IWQPSQ_VABASEDTO) | - LS_64(info->read_fence, I40IWQPSQ_READFENCE) | - LS_64(info->local_fence, I40IWQPSQ_LOCALFENCE) | - LS_64(info->signaled, I40IWQPSQ_SIGCOMPL) | - LS_64(qp->qp_uk.swqe_polarity, I40IWQPSQ_VALID); - - i40iw_insert_wqe_hdr(wqe, header); - - i40iw_debug_buf(qp->dev, I40IW_DEBUG_WQE, "FAST_REG WQE", - wqe, I40IW_QP_WQE_MIN_SIZE); - - if (post_sq) - i40iw_qp_post_wr(&qp->qp_uk); - return 0; -} - -/** - * i40iw_sc_send_lsmm - send last streaming mode message - * @qp: sc qp struct - * @lsmm_buf: buffer with lsmm message - * @size: size of lsmm buffer - * @stag: stag of lsmm buffer - */ -static void i40iw_sc_send_lsmm(struct i40iw_sc_qp *qp, - void *lsmm_buf, - u32 size, - i40iw_stag stag) -{ - u64 *wqe; - u64 header; - struct i40iw_qp_uk *qp_uk; - - qp_uk = &qp->qp_uk; - wqe = qp_uk->sq_base->elem; - - set_64bit_val(wqe, 0, (uintptr_t)lsmm_buf); - - set_64bit_val(wqe, 8, (size | LS_64(stag, I40IWQPSQ_FRAG_STAG))); - - set_64bit_val(wqe, 16, 0); - - header = LS_64(I40IWQP_OP_RDMA_SEND, I40IWQPSQ_OPCODE) | - LS_64(1, I40IWQPSQ_STREAMMODE) | - LS_64(1, I40IWQPSQ_WAITFORRCVPDU) | - LS_64(qp->qp_uk.swqe_polarity, I40IWQPSQ_VALID); - - i40iw_insert_wqe_hdr(wqe, header); - - i40iw_debug_buf(qp->dev, I40IW_DEBUG_QP, "SEND_LSMM WQE", - wqe, I40IW_QP_WQE_MIN_SIZE); -} - -/** - * i40iw_sc_send_lsmm_nostag - for privilege qp - * @qp: sc qp struct - * @lsmm_buf: buffer with lsmm message - * @size: size of lsmm buffer - */ -static void i40iw_sc_send_lsmm_nostag(struct i40iw_sc_qp *qp, - void *lsmm_buf, - u32 size) -{ - u64 *wqe; - u64 header; - struct i40iw_qp_uk *qp_uk; - - qp_uk = &qp->qp_uk; - wqe = qp_uk->sq_base->elem; - - set_64bit_val(wqe, 0, (uintptr_t)lsmm_buf); - - set_64bit_val(wqe, 8, size); - - set_64bit_val(wqe, 16, 0); - - header = LS_64(I40IWQP_OP_RDMA_SEND, I40IWQPSQ_OPCODE) | - LS_64(1, I40IWQPSQ_STREAMMODE) | - LS_64(1, I40IWQPSQ_WAITFORRCVPDU) | - LS_64(qp->qp_uk.swqe_polarity, I40IWQPSQ_VALID); - - i40iw_insert_wqe_hdr(wqe, header); - - i40iw_debug_buf(qp->dev, I40IW_DEBUG_WQE, "SEND_LSMM_NOSTAG WQE", - wqe, I40IW_QP_WQE_MIN_SIZE); -} - -/** - * i40iw_sc_send_rtt - send last read0 or write0 - * @qp: sc qp struct - * @read: Do read0 or write0 - */ -static void i40iw_sc_send_rtt(struct i40iw_sc_qp *qp, bool read) -{ - u64 *wqe; - u64 header; - struct i40iw_qp_uk *qp_uk; - - qp_uk = &qp->qp_uk; - wqe = qp_uk->sq_base->elem; - - set_64bit_val(wqe, 0, 0); - set_64bit_val(wqe, 8, 0); - set_64bit_val(wqe, 16, 0); - if (read) { - header = LS_64(0x1234, I40IWQPSQ_REMSTAG) | - LS_64(I40IWQP_OP_RDMA_READ, I40IWQPSQ_OPCODE) | - LS_64(qp->qp_uk.swqe_polarity, I40IWQPSQ_VALID); - set_64bit_val(wqe, 8, ((u64)0xabcd << 32)); - } else { - header = LS_64(I40IWQP_OP_RDMA_WRITE, I40IWQPSQ_OPCODE) | - LS_64(qp->qp_uk.swqe_polarity, I40IWQPSQ_VALID); - } - - i40iw_insert_wqe_hdr(wqe, header); - - i40iw_debug_buf(qp->dev, I40IW_DEBUG_WQE, "RTR WQE", - wqe, I40IW_QP_WQE_MIN_SIZE); -} - -/** - * i40iw_sc_post_wqe0 - send wqe with opcode - * @qp: sc qp struct - * @opcode: opcode to use for wqe0 - */ -static enum i40iw_status_code i40iw_sc_post_wqe0(struct i40iw_sc_qp *qp, u8 opcode) -{ - u64 *wqe; - u64 header; - struct i40iw_qp_uk *qp_uk; - - qp_uk = &qp->qp_uk; - wqe = qp_uk->sq_base->elem; - - if (!wqe) - return I40IW_ERR_QP_TOOMANY_WRS_POSTED; - switch (opcode) { - case I40IWQP_OP_NOP: - set_64bit_val(wqe, 0, 0); - set_64bit_val(wqe, 8, 0); - set_64bit_val(wqe, 16, 0); - header = LS_64(I40IWQP_OP_NOP, I40IWQPSQ_OPCODE) | - LS_64(qp->qp_uk.swqe_polarity, I40IWQPSQ_VALID); - - i40iw_insert_wqe_hdr(wqe, header); - break; - case I40IWQP_OP_RDMA_SEND: - set_64bit_val(wqe, 0, 0); - set_64bit_val(wqe, 8, 0); - set_64bit_val(wqe, 16, 0); - header = LS_64(I40IWQP_OP_RDMA_SEND, I40IWQPSQ_OPCODE) | - LS_64(qp->qp_uk.swqe_polarity, I40IWQPSQ_VALID) | - LS_64(1, I40IWQPSQ_STREAMMODE) | - LS_64(1, I40IWQPSQ_WAITFORRCVPDU); - - i40iw_insert_wqe_hdr(wqe, header); - break; - default: - i40iw_debug(qp->dev, I40IW_DEBUG_QP, "%s: Invalid WQE zero opcode\n", - __func__); - break; - } - return 0; -} - -/** - * i40iw_sc_init_iw_hmc() - queries fpm values using cqp and populates hmc_info - * @dev : ptr to i40iw_dev struct - * @hmc_fn_id: hmc function id - */ -enum i40iw_status_code i40iw_sc_init_iw_hmc(struct i40iw_sc_dev *dev, u8 hmc_fn_id) -{ - struct i40iw_hmc_info *hmc_info; - struct i40iw_dma_mem query_fpm_mem; - struct i40iw_virt_mem virt_mem; - struct i40iw_vfdev *vf_dev = NULL; - u32 mem_size; - enum i40iw_status_code ret_code = 0; - bool poll_registers = true; - u16 iw_vf_idx; - u8 wait_type; - - if (hmc_fn_id >= I40IW_MAX_VF_FPM_ID || - (dev->hmc_fn_id != hmc_fn_id && hmc_fn_id < I40IW_FIRST_VF_FPM_ID)) - return I40IW_ERR_INVALID_HMCFN_ID; - - i40iw_debug(dev, I40IW_DEBUG_HMC, "hmc_fn_id %u, dev->hmc_fn_id %u\n", hmc_fn_id, - dev->hmc_fn_id); - if (hmc_fn_id == dev->hmc_fn_id) { - hmc_info = dev->hmc_info; - query_fpm_mem.pa = dev->fpm_query_buf_pa; - query_fpm_mem.va = dev->fpm_query_buf; - } else { - vf_dev = i40iw_vfdev_from_fpm(dev, hmc_fn_id); - if (!vf_dev) - return I40IW_ERR_INVALID_VF_ID; - - hmc_info = &vf_dev->hmc_info; - iw_vf_idx = vf_dev->iw_vf_idx; - i40iw_debug(dev, I40IW_DEBUG_HMC, "vf_dev %p, hmc_info %p, hmc_obj %p\n", vf_dev, - hmc_info, hmc_info->hmc_obj); - if (!vf_dev->fpm_query_buf) { - if (!dev->vf_fpm_query_buf[iw_vf_idx].va) { - ret_code = i40iw_alloc_query_fpm_buf(dev, - &dev->vf_fpm_query_buf[iw_vf_idx]); - if (ret_code) - return ret_code; - } - vf_dev->fpm_query_buf = dev->vf_fpm_query_buf[iw_vf_idx].va; - vf_dev->fpm_query_buf_pa = dev->vf_fpm_query_buf[iw_vf_idx].pa; - } - query_fpm_mem.pa = vf_dev->fpm_query_buf_pa; - query_fpm_mem.va = vf_dev->fpm_query_buf; - /** - * It is HARDWARE specific: - * this call is done by PF for VF and - * i40iw_sc_query_fpm_values needs ccq poll - * because PF ccq is already created. - */ - poll_registers = false; - } - - hmc_info->hmc_fn_id = hmc_fn_id; - - if (hmc_fn_id != dev->hmc_fn_id) { - ret_code = - i40iw_cqp_query_fpm_values_cmd(dev, &query_fpm_mem, hmc_fn_id); - } else { - wait_type = poll_registers ? (u8)I40IW_CQP_WAIT_POLL_REGS : - (u8)I40IW_CQP_WAIT_POLL_CQ; - - ret_code = i40iw_sc_query_fpm_values( - dev->cqp, - 0, - hmc_info->hmc_fn_id, - &query_fpm_mem, - true, - wait_type); - } - if (ret_code) - return ret_code; - - /* parse the fpm_query_buf and fill hmc obj info */ - ret_code = - i40iw_sc_parse_fpm_query_buf((u64 *)query_fpm_mem.va, - hmc_info, - &dev->hmc_fpm_misc); - if (ret_code) - return ret_code; - i40iw_debug_buf(dev, I40IW_DEBUG_HMC, "QUERY FPM BUFFER", - query_fpm_mem.va, I40IW_QUERY_FPM_BUF_SIZE); - - if (hmc_fn_id != dev->hmc_fn_id) { - i40iw_cqp_commit_fpm_values_cmd(dev, &query_fpm_mem, hmc_fn_id); - - /* parse the fpm_commit_buf and fill hmc obj info */ - i40iw_sc_parse_fpm_commit_buf((u64 *)query_fpm_mem.va, hmc_info->hmc_obj, &hmc_info->sd_table.sd_cnt); - mem_size = sizeof(struct i40iw_hmc_sd_entry) * - (hmc_info->sd_table.sd_cnt + hmc_info->first_sd_index); - ret_code = i40iw_allocate_virt_mem(dev->hw, &virt_mem, mem_size); - if (ret_code) - return ret_code; - hmc_info->sd_table.sd_entry = virt_mem.va; - } - - return ret_code; -} - -/** - * i40iw_sc_configure_iw_fpm() - commits hmc obj cnt values using cqp command and - * populates fpm base address in hmc_info - * @dev : ptr to i40iw_dev struct - * @hmc_fn_id: hmc function id - */ -static enum i40iw_status_code i40iw_sc_configure_iw_fpm(struct i40iw_sc_dev *dev, - u8 hmc_fn_id) -{ - struct i40iw_hmc_info *hmc_info; - struct i40iw_hmc_obj_info *obj_info; - u64 *buf; - struct i40iw_dma_mem commit_fpm_mem; - u32 i, j; - enum i40iw_status_code ret_code = 0; - bool poll_registers = true; - u8 wait_type; - - if (hmc_fn_id >= I40IW_MAX_VF_FPM_ID || - (dev->hmc_fn_id != hmc_fn_id && hmc_fn_id < I40IW_FIRST_VF_FPM_ID)) - return I40IW_ERR_INVALID_HMCFN_ID; - - if (hmc_fn_id == dev->hmc_fn_id) { - hmc_info = dev->hmc_info; - } else { - hmc_info = i40iw_vf_hmcinfo_from_fpm(dev, hmc_fn_id); - poll_registers = false; - } - if (!hmc_info) - return I40IW_ERR_BAD_PTR; - - obj_info = hmc_info->hmc_obj; - buf = dev->fpm_commit_buf; - - /* copy cnt values in commit buf */ - for (i = I40IW_HMC_IW_QP, j = 0; i <= I40IW_HMC_IW_PBLE; - i++, j += 8) - set_64bit_val(buf, j, (u64)obj_info[i].cnt); - - set_64bit_val(buf, 40, 0); /* APBVT rsvd */ - - commit_fpm_mem.pa = dev->fpm_commit_buf_pa; - commit_fpm_mem.va = dev->fpm_commit_buf; - wait_type = poll_registers ? (u8)I40IW_CQP_WAIT_POLL_REGS : - (u8)I40IW_CQP_WAIT_POLL_CQ; - ret_code = i40iw_sc_commit_fpm_values( - dev->cqp, - 0, - hmc_info->hmc_fn_id, - &commit_fpm_mem, - true, - wait_type); - - /* parse the fpm_commit_buf and fill hmc obj info */ - if (!ret_code) - ret_code = i40iw_sc_parse_fpm_commit_buf(dev->fpm_commit_buf, - hmc_info->hmc_obj, - &hmc_info->sd_table.sd_cnt); - - i40iw_debug_buf(dev, I40IW_DEBUG_HMC, "COMMIT FPM BUFFER", - commit_fpm_mem.va, I40IW_COMMIT_FPM_BUF_SIZE); - - return ret_code; -} - -/** - * cqp_sds_wqe_fill - fill cqp wqe doe sd - * @cqp: struct for cqp hw - * @info: sd info for wqe - * @scratch: u64 saved to be used during cqp completion - */ -static enum i40iw_status_code cqp_sds_wqe_fill(struct i40iw_sc_cqp *cqp, - struct i40iw_update_sds_info *info, - u64 scratch) -{ - u64 data; - u64 header; - u64 *wqe; - int mem_entries, wqe_entries; - struct i40iw_dma_mem *sdbuf = &cqp->sdbuf; - u64 offset; - u32 wqe_idx; - - wqe = i40iw_sc_cqp_get_next_send_wqe_idx(cqp, scratch, &wqe_idx); - if (!wqe) - return I40IW_ERR_RING_FULL; - - I40IW_CQP_INIT_WQE(wqe); - wqe_entries = (info->cnt > 3) ? 3 : info->cnt; - mem_entries = info->cnt - wqe_entries; - - header = LS_64(I40IW_CQP_OP_UPDATE_PE_SDS, I40IW_CQPSQ_OPCODE) | - LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID) | - LS_64(mem_entries, I40IW_CQPSQ_UPESD_ENTRY_COUNT); - - if (mem_entries) { - offset = wqe_idx * I40IW_UPDATE_SD_BUF_SIZE; - memcpy((char *)sdbuf->va + offset, &info->entry[3], - mem_entries << 4); - data = (u64)sdbuf->pa + offset; - } else { - data = 0; - } - data |= LS_64(info->hmc_fn_id, I40IW_CQPSQ_UPESD_HMCFNID); - - set_64bit_val(wqe, 16, data); - - switch (wqe_entries) { - case 3: - set_64bit_val(wqe, 48, - (LS_64(info->entry[2].cmd, I40IW_CQPSQ_UPESD_SDCMD) | - LS_64(1, I40IW_CQPSQ_UPESD_ENTRY_VALID))); - - set_64bit_val(wqe, 56, info->entry[2].data); - fallthrough; - case 2: - set_64bit_val(wqe, 32, - (LS_64(info->entry[1].cmd, I40IW_CQPSQ_UPESD_SDCMD) | - LS_64(1, I40IW_CQPSQ_UPESD_ENTRY_VALID))); - - set_64bit_val(wqe, 40, info->entry[1].data); - fallthrough; - case 1: - set_64bit_val(wqe, 0, - LS_64(info->entry[0].cmd, I40IW_CQPSQ_UPESD_SDCMD)); - - set_64bit_val(wqe, 8, info->entry[0].data); - break; - default: - break; - } - - i40iw_insert_wqe_hdr(wqe, header); - - i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "UPDATE_PE_SDS WQE", - wqe, I40IW_CQP_WQE_SIZE * 8); - return 0; -} - -/** - * i40iw_update_pe_sds - cqp wqe for sd - * @dev: ptr to i40iw_dev struct - * @info: sd info for sd's - * @scratch: u64 saved to be used during cqp completion - */ -static enum i40iw_status_code i40iw_update_pe_sds(struct i40iw_sc_dev *dev, - struct i40iw_update_sds_info *info, - u64 scratch) -{ - struct i40iw_sc_cqp *cqp = dev->cqp; - enum i40iw_status_code ret_code; - - ret_code = cqp_sds_wqe_fill(cqp, info, scratch); - if (!ret_code) - i40iw_sc_cqp_post_sq(cqp); - - return ret_code; -} - -/** - * i40iw_update_sds_noccq - update sd before ccq created - * @dev: sc device struct - * @info: sd info for sd's - */ -enum i40iw_status_code i40iw_update_sds_noccq(struct i40iw_sc_dev *dev, - struct i40iw_update_sds_info *info) -{ - u32 error, val, tail; - struct i40iw_sc_cqp *cqp = dev->cqp; - enum i40iw_status_code ret_code; - - ret_code = cqp_sds_wqe_fill(cqp, info, 0); - if (ret_code) - return ret_code; - i40iw_get_cqp_reg_info(cqp, &val, &tail, &error); - if (error) - return I40IW_ERR_CQP_COMPL_ERROR; - - i40iw_sc_cqp_post_sq(cqp); - ret_code = i40iw_cqp_poll_registers(cqp, tail, I40IW_DONE_COUNT); - - return ret_code; -} - -/** - * i40iw_sc_suspend_qp - suspend qp for param change - * @cqp: struct for cqp hw - * @qp: sc qp struct - * @scratch: u64 saved to be used during cqp completion - */ -enum i40iw_status_code i40iw_sc_suspend_qp(struct i40iw_sc_cqp *cqp, - struct i40iw_sc_qp *qp, - u64 scratch) -{ - u64 header; - u64 *wqe; - - wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch); - if (!wqe) - return I40IW_ERR_RING_FULL; - header = LS_64(qp->qp_uk.qp_id, I40IW_CQPSQ_SUSPENDQP_QPID) | - LS_64(I40IW_CQP_OP_SUSPEND_QP, I40IW_CQPSQ_OPCODE) | - LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID); - - i40iw_insert_wqe_hdr(wqe, header); - - i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "SUSPEND_QP WQE", - wqe, I40IW_CQP_WQE_SIZE * 8); - - i40iw_sc_cqp_post_sq(cqp); - return 0; -} - -/** - * i40iw_sc_resume_qp - resume qp after suspend - * @cqp: struct for cqp hw - * @qp: sc qp struct - * @scratch: u64 saved to be used during cqp completion - */ -enum i40iw_status_code i40iw_sc_resume_qp(struct i40iw_sc_cqp *cqp, - struct i40iw_sc_qp *qp, - u64 scratch) -{ - u64 header; - u64 *wqe; - - wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch); - if (!wqe) - return I40IW_ERR_RING_FULL; - set_64bit_val(wqe, - 16, - LS_64(qp->qs_handle, I40IW_CQPSQ_RESUMEQP_QSHANDLE)); - - header = LS_64(qp->qp_uk.qp_id, I40IW_CQPSQ_RESUMEQP_QPID) | - LS_64(I40IW_CQP_OP_RESUME_QP, I40IW_CQPSQ_OPCODE) | - LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID); - - i40iw_insert_wqe_hdr(wqe, header); - - i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "RESUME_QP WQE", - wqe, I40IW_CQP_WQE_SIZE * 8); - - i40iw_sc_cqp_post_sq(cqp); - return 0; -} - -/** - * i40iw_sc_static_hmc_pages_allocated - cqp wqe to allocate hmc pages - * @cqp: struct for cqp hw - * @scratch: u64 saved to be used during cqp completion - * @hmc_fn_id: hmc function id - * @post_sq: flag for cqp db to ring - * @poll_registers: flag to poll register for cqp completion - */ -enum i40iw_status_code i40iw_sc_static_hmc_pages_allocated( - struct i40iw_sc_cqp *cqp, - u64 scratch, - u8 hmc_fn_id, - bool post_sq, - bool poll_registers) -{ - u64 header; - u64 *wqe; - u32 tail, val, error; - enum i40iw_status_code ret_code = 0; - - wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch); - if (!wqe) - return I40IW_ERR_RING_FULL; - set_64bit_val(wqe, - 16, - LS_64(hmc_fn_id, I40IW_SHMC_PAGE_ALLOCATED_HMC_FN_ID)); - - header = LS_64(I40IW_CQP_OP_SHMC_PAGES_ALLOCATED, I40IW_CQPSQ_OPCODE) | - LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID); - - i40iw_insert_wqe_hdr(wqe, header); - - i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "SHMC_PAGES_ALLOCATED WQE", - wqe, I40IW_CQP_WQE_SIZE * 8); - i40iw_get_cqp_reg_info(cqp, &val, &tail, &error); - if (error) { - ret_code = I40IW_ERR_CQP_COMPL_ERROR; - return ret_code; - } - if (post_sq) { - i40iw_sc_cqp_post_sq(cqp); - if (poll_registers) - /* check for cqp sq tail update */ - ret_code = i40iw_cqp_poll_registers(cqp, tail, 1000); - else - ret_code = i40iw_sc_poll_for_cqp_op_done(cqp, - I40IW_CQP_OP_SHMC_PAGES_ALLOCATED, - NULL); - } - - return ret_code; -} - -/** - * i40iw_ring_full - check if cqp ring is full - * @cqp: struct for cqp hw - */ -static bool i40iw_ring_full(struct i40iw_sc_cqp *cqp) -{ - return I40IW_RING_FULL_ERR(cqp->sq_ring); -} - -/** - * i40iw_est_sd - returns approximate number of SDs for HMC - * @dev: sc device struct - * @hmc_info: hmc structure, size and count for HMC objects - */ -static u64 i40iw_est_sd(struct i40iw_sc_dev *dev, struct i40iw_hmc_info *hmc_info) -{ - int i; - u64 size = 0; - u64 sd; - - for (i = I40IW_HMC_IW_QP; i < I40IW_HMC_IW_PBLE; i++) - size += hmc_info->hmc_obj[i].cnt * hmc_info->hmc_obj[i].size; - - if (dev->is_pf) - size += hmc_info->hmc_obj[I40IW_HMC_IW_PBLE].cnt * hmc_info->hmc_obj[I40IW_HMC_IW_PBLE].size; - - if (size & 0x1FFFFF) - sd = (size >> 21) + 1; /* add 1 for remainder */ - else - sd = size >> 21; - - if (!dev->is_pf) { - /* 2MB alignment for VF PBLE HMC */ - size = hmc_info->hmc_obj[I40IW_HMC_IW_PBLE].cnt * hmc_info->hmc_obj[I40IW_HMC_IW_PBLE].size; - if (size & 0x1FFFFF) - sd += (size >> 21) + 1; /* add 1 for remainder */ - else - sd += size >> 21; - } - - return sd; -} - -/** - * i40iw_config_fpm_values - configure HMC objects - * @dev: sc device struct - * @qp_count: desired qp count - */ -enum i40iw_status_code i40iw_config_fpm_values(struct i40iw_sc_dev *dev, u32 qp_count) -{ - struct i40iw_virt_mem virt_mem; - u32 i, mem_size; - u32 qpwantedoriginal, qpwanted, mrwanted, pblewanted; - u64 sd_needed; - u32 loop_count = 0; - - struct i40iw_hmc_info *hmc_info; - struct i40iw_hmc_fpm_misc *hmc_fpm_misc; - enum i40iw_status_code ret_code = 0; - - hmc_info = dev->hmc_info; - hmc_fpm_misc = &dev->hmc_fpm_misc; - - ret_code = i40iw_sc_init_iw_hmc(dev, dev->hmc_fn_id); - if (ret_code) { - i40iw_debug(dev, I40IW_DEBUG_HMC, - "i40iw_sc_init_iw_hmc returned error_code = %d\n", - ret_code); - return ret_code; - } - - for (i = I40IW_HMC_IW_QP; i < I40IW_HMC_IW_MAX; i++) - hmc_info->hmc_obj[i].cnt = hmc_info->hmc_obj[i].max_cnt; - sd_needed = i40iw_est_sd(dev, hmc_info); - i40iw_debug(dev, I40IW_DEBUG_HMC, - "%s: FW initial max sd_count[%08lld] first_sd_index[%04d]\n", - __func__, sd_needed, hmc_info->first_sd_index); - i40iw_debug(dev, I40IW_DEBUG_HMC, - "%s: sd count %d where max sd is %d\n", - __func__, hmc_info->sd_table.sd_cnt, - hmc_fpm_misc->max_sds); - - qpwanted = min(qp_count, hmc_info->hmc_obj[I40IW_HMC_IW_QP].max_cnt); - qpwantedoriginal = qpwanted; - mrwanted = hmc_info->hmc_obj[I40IW_HMC_IW_MR].max_cnt; - pblewanted = hmc_info->hmc_obj[I40IW_HMC_IW_PBLE].max_cnt; - - i40iw_debug(dev, I40IW_DEBUG_HMC, - "req_qp=%d max_sd=%d, max_qp = %d, max_cq=%d, max_mr=%d, max_pble=%d\n", - qp_count, hmc_fpm_misc->max_sds, - hmc_info->hmc_obj[I40IW_HMC_IW_QP].max_cnt, - hmc_info->hmc_obj[I40IW_HMC_IW_CQ].max_cnt, - hmc_info->hmc_obj[I40IW_HMC_IW_MR].max_cnt, - hmc_info->hmc_obj[I40IW_HMC_IW_PBLE].max_cnt); - - do { - ++loop_count; - hmc_info->hmc_obj[I40IW_HMC_IW_QP].cnt = qpwanted; - hmc_info->hmc_obj[I40IW_HMC_IW_CQ].cnt = - min(2 * qpwanted, hmc_info->hmc_obj[I40IW_HMC_IW_CQ].cnt); - hmc_info->hmc_obj[I40IW_HMC_IW_SRQ].cnt = 0x00; /* Reserved */ - hmc_info->hmc_obj[I40IW_HMC_IW_HTE].cnt = - qpwanted * hmc_fpm_misc->ht_multiplier; - hmc_info->hmc_obj[I40IW_HMC_IW_ARP].cnt = - hmc_info->hmc_obj[I40IW_HMC_IW_ARP].max_cnt; - hmc_info->hmc_obj[I40IW_HMC_IW_APBVT_ENTRY].cnt = 1; - hmc_info->hmc_obj[I40IW_HMC_IW_MR].cnt = mrwanted; - - hmc_info->hmc_obj[I40IW_HMC_IW_XF].cnt = - roundup_pow_of_two(I40IW_MAX_WQ_ENTRIES * qpwanted); - hmc_info->hmc_obj[I40IW_HMC_IW_Q1].cnt = - roundup_pow_of_two(2 * I40IW_MAX_IRD_SIZE * qpwanted); - hmc_info->hmc_obj[I40IW_HMC_IW_XFFL].cnt = - hmc_info->hmc_obj[I40IW_HMC_IW_XF].cnt / hmc_fpm_misc->xf_block_size; - hmc_info->hmc_obj[I40IW_HMC_IW_Q1FL].cnt = - hmc_info->hmc_obj[I40IW_HMC_IW_Q1].cnt / hmc_fpm_misc->q1_block_size; - hmc_info->hmc_obj[I40IW_HMC_IW_TIMER].cnt = - ((qpwanted) / 512 + 1) * hmc_fpm_misc->timer_bucket; - hmc_info->hmc_obj[I40IW_HMC_IW_FSIMC].cnt = 0x00; - hmc_info->hmc_obj[I40IW_HMC_IW_FSIAV].cnt = 0x00; - hmc_info->hmc_obj[I40IW_HMC_IW_PBLE].cnt = pblewanted; - - /* How much memory is needed for all the objects. */ - sd_needed = i40iw_est_sd(dev, hmc_info); - if ((loop_count > 1000) || - ((!(loop_count % 10)) && - (qpwanted > qpwantedoriginal * 2 / 3))) { - if (qpwanted > FPM_MULTIPLIER) - qpwanted = roundup_pow_of_two(qpwanted - - FPM_MULTIPLIER); - qpwanted >>= 1; - } - if (mrwanted > FPM_MULTIPLIER * 10) - mrwanted -= FPM_MULTIPLIER * 10; - if (pblewanted > FPM_MULTIPLIER * 1000) - pblewanted -= FPM_MULTIPLIER * 1000; - } while (sd_needed > hmc_fpm_misc->max_sds && loop_count < 2000); - - i40iw_debug(dev, I40IW_DEBUG_HMC, - "loop_cnt=%d, sd_needed=%lld, qpcnt = %d, cqcnt=%d, mrcnt=%d, pblecnt=%d\n", - loop_count, sd_needed, - hmc_info->hmc_obj[I40IW_HMC_IW_QP].cnt, - hmc_info->hmc_obj[I40IW_HMC_IW_CQ].cnt, - hmc_info->hmc_obj[I40IW_HMC_IW_MR].cnt, - hmc_info->hmc_obj[I40IW_HMC_IW_PBLE].cnt); - - ret_code = i40iw_sc_configure_iw_fpm(dev, dev->hmc_fn_id); - if (ret_code) { - i40iw_debug(dev, I40IW_DEBUG_HMC, - "configure_iw_fpm returned error_code[x%08X]\n", - i40iw_rd32(dev->hw, dev->is_pf ? I40E_PFPE_CQPERRCODES : I40E_VFPE_CQPERRCODES1)); - return ret_code; - } - - mem_size = sizeof(struct i40iw_hmc_sd_entry) * - (hmc_info->sd_table.sd_cnt + hmc_info->first_sd_index + 1); - ret_code = i40iw_allocate_virt_mem(dev->hw, &virt_mem, mem_size); - if (ret_code) { - i40iw_debug(dev, I40IW_DEBUG_HMC, - "%s: failed to allocate memory for sd_entry buffer\n", - __func__); - return ret_code; - } - hmc_info->sd_table.sd_entry = virt_mem.va; - - return ret_code; -} - -/** - * i40iw_exec_cqp_cmd - execute cqp cmd when wqe are available - * @dev: rdma device - * @pcmdinfo: cqp command info - */ -static enum i40iw_status_code i40iw_exec_cqp_cmd(struct i40iw_sc_dev *dev, - struct cqp_commands_info *pcmdinfo) -{ - enum i40iw_status_code status; - struct i40iw_dma_mem values_mem; - - dev->cqp_cmd_stats[pcmdinfo->cqp_cmd]++; - switch (pcmdinfo->cqp_cmd) { - case OP_DELETE_LOCAL_MAC_IPADDR_ENTRY: - status = i40iw_sc_del_local_mac_ipaddr_entry( - pcmdinfo->in.u.del_local_mac_ipaddr_entry.cqp, - pcmdinfo->in.u.del_local_mac_ipaddr_entry.scratch, - pcmdinfo->in.u.del_local_mac_ipaddr_entry.entry_idx, - pcmdinfo->in.u.del_local_mac_ipaddr_entry.ignore_ref_count, - pcmdinfo->post_sq); - break; - case OP_CEQ_DESTROY: - status = i40iw_sc_ceq_destroy(pcmdinfo->in.u.ceq_destroy.ceq, - pcmdinfo->in.u.ceq_destroy.scratch, - pcmdinfo->post_sq); - break; - case OP_AEQ_DESTROY: - status = i40iw_sc_aeq_destroy(pcmdinfo->in.u.aeq_destroy.aeq, - pcmdinfo->in.u.aeq_destroy.scratch, - pcmdinfo->post_sq); - - break; - case OP_DELETE_ARP_CACHE_ENTRY: - status = i40iw_sc_del_arp_cache_entry( - pcmdinfo->in.u.del_arp_cache_entry.cqp, - pcmdinfo->in.u.del_arp_cache_entry.scratch, - pcmdinfo->in.u.del_arp_cache_entry.arp_index, - pcmdinfo->post_sq); - break; - case OP_MANAGE_APBVT_ENTRY: - status = i40iw_sc_manage_apbvt_entry( - pcmdinfo->in.u.manage_apbvt_entry.cqp, - &pcmdinfo->in.u.manage_apbvt_entry.info, - pcmdinfo->in.u.manage_apbvt_entry.scratch, - pcmdinfo->post_sq); - break; - case OP_CEQ_CREATE: - status = i40iw_sc_ceq_create(pcmdinfo->in.u.ceq_create.ceq, - pcmdinfo->in.u.ceq_create.scratch, - pcmdinfo->post_sq); - break; - case OP_AEQ_CREATE: - status = i40iw_sc_aeq_create(pcmdinfo->in.u.aeq_create.aeq, - pcmdinfo->in.u.aeq_create.scratch, - pcmdinfo->post_sq); - break; - case OP_ALLOC_LOCAL_MAC_IPADDR_ENTRY: - status = i40iw_sc_alloc_local_mac_ipaddr_entry( - pcmdinfo->in.u.alloc_local_mac_ipaddr_entry.cqp, - pcmdinfo->in.u.alloc_local_mac_ipaddr_entry.scratch, - pcmdinfo->post_sq); - break; - case OP_ADD_LOCAL_MAC_IPADDR_ENTRY: - status = i40iw_sc_add_local_mac_ipaddr_entry( - pcmdinfo->in.u.add_local_mac_ipaddr_entry.cqp, - &pcmdinfo->in.u.add_local_mac_ipaddr_entry.info, - pcmdinfo->in.u.add_local_mac_ipaddr_entry.scratch, - pcmdinfo->post_sq); - break; - case OP_MANAGE_QHASH_TABLE_ENTRY: - status = i40iw_sc_manage_qhash_table_entry( - pcmdinfo->in.u.manage_qhash_table_entry.cqp, - &pcmdinfo->in.u.manage_qhash_table_entry.info, - pcmdinfo->in.u.manage_qhash_table_entry.scratch, - pcmdinfo->post_sq); - - break; - case OP_QP_MODIFY: - status = i40iw_sc_qp_modify( - pcmdinfo->in.u.qp_modify.qp, - &pcmdinfo->in.u.qp_modify.info, - pcmdinfo->in.u.qp_modify.scratch, - pcmdinfo->post_sq); - - break; - case OP_QP_UPLOAD_CONTEXT: - status = i40iw_sc_qp_upload_context( - pcmdinfo->in.u.qp_upload_context.dev, - &pcmdinfo->in.u.qp_upload_context.info, - pcmdinfo->in.u.qp_upload_context.scratch, - pcmdinfo->post_sq); - - break; - case OP_CQ_CREATE: - status = i40iw_sc_cq_create( - pcmdinfo->in.u.cq_create.cq, - pcmdinfo->in.u.cq_create.scratch, - pcmdinfo->in.u.cq_create.check_overflow, - pcmdinfo->post_sq); - break; - case OP_CQ_DESTROY: - status = i40iw_sc_cq_destroy( - pcmdinfo->in.u.cq_destroy.cq, - pcmdinfo->in.u.cq_destroy.scratch, - pcmdinfo->post_sq); - - break; - case OP_QP_CREATE: - status = i40iw_sc_qp_create( - pcmdinfo->in.u.qp_create.qp, - &pcmdinfo->in.u.qp_create.info, - pcmdinfo->in.u.qp_create.scratch, - pcmdinfo->post_sq); - break; - case OP_QP_DESTROY: - status = i40iw_sc_qp_destroy( - pcmdinfo->in.u.qp_destroy.qp, - pcmdinfo->in.u.qp_destroy.scratch, - pcmdinfo->in.u.qp_destroy.remove_hash_idx, - pcmdinfo->in.u.qp_destroy. - ignore_mw_bnd, - pcmdinfo->post_sq); - - break; - case OP_ALLOC_STAG: - status = i40iw_sc_alloc_stag( - pcmdinfo->in.u.alloc_stag.dev, - &pcmdinfo->in.u.alloc_stag.info, - pcmdinfo->in.u.alloc_stag.scratch, - pcmdinfo->post_sq); - break; - case OP_MR_REG_NON_SHARED: - status = i40iw_sc_mr_reg_non_shared( - pcmdinfo->in.u.mr_reg_non_shared.dev, - &pcmdinfo->in.u.mr_reg_non_shared.info, - pcmdinfo->in.u.mr_reg_non_shared.scratch, - pcmdinfo->post_sq); - - break; - case OP_DEALLOC_STAG: - status = i40iw_sc_dealloc_stag( - pcmdinfo->in.u.dealloc_stag.dev, - &pcmdinfo->in.u.dealloc_stag.info, - pcmdinfo->in.u.dealloc_stag.scratch, - pcmdinfo->post_sq); - - break; - case OP_MW_ALLOC: - status = i40iw_sc_mw_alloc( - pcmdinfo->in.u.mw_alloc.dev, - pcmdinfo->in.u.mw_alloc.scratch, - pcmdinfo->in.u.mw_alloc.mw_stag_index, - pcmdinfo->in.u.mw_alloc.pd_id, - pcmdinfo->post_sq); - - break; - case OP_QP_FLUSH_WQES: - status = i40iw_sc_qp_flush_wqes( - pcmdinfo->in.u.qp_flush_wqes.qp, - &pcmdinfo->in.u.qp_flush_wqes.info, - pcmdinfo->in.u.qp_flush_wqes. - scratch, pcmdinfo->post_sq); - break; - case OP_GEN_AE: - status = i40iw_sc_gen_ae( - pcmdinfo->in.u.gen_ae.qp, - &pcmdinfo->in.u.gen_ae.info, - pcmdinfo->in.u.gen_ae.scratch, - pcmdinfo->post_sq); - break; - case OP_ADD_ARP_CACHE_ENTRY: - status = i40iw_sc_add_arp_cache_entry( - pcmdinfo->in.u.add_arp_cache_entry.cqp, - &pcmdinfo->in.u.add_arp_cache_entry.info, - pcmdinfo->in.u.add_arp_cache_entry.scratch, - pcmdinfo->post_sq); - break; - case OP_UPDATE_PE_SDS: - /* case I40IW_CQP_OP_UPDATE_PE_SDS */ - status = i40iw_update_pe_sds( - pcmdinfo->in.u.update_pe_sds.dev, - &pcmdinfo->in.u.update_pe_sds.info, - pcmdinfo->in.u.update_pe_sds. - scratch); - - break; - case OP_MANAGE_HMC_PM_FUNC_TABLE: - status = i40iw_sc_manage_hmc_pm_func_table( - pcmdinfo->in.u.manage_hmc_pm.dev->cqp, - pcmdinfo->in.u.manage_hmc_pm.scratch, - (u8)pcmdinfo->in.u.manage_hmc_pm.info.vf_id, - pcmdinfo->in.u.manage_hmc_pm.info.free_fcn, - true); - break; - case OP_SUSPEND: - status = i40iw_sc_suspend_qp( - pcmdinfo->in.u.suspend_resume.cqp, - pcmdinfo->in.u.suspend_resume.qp, - pcmdinfo->in.u.suspend_resume.scratch); - break; - case OP_RESUME: - status = i40iw_sc_resume_qp( - pcmdinfo->in.u.suspend_resume.cqp, - pcmdinfo->in.u.suspend_resume.qp, - pcmdinfo->in.u.suspend_resume.scratch); - break; - case OP_MANAGE_VF_PBLE_BP: - status = i40iw_manage_vf_pble_bp( - pcmdinfo->in.u.manage_vf_pble_bp.cqp, - &pcmdinfo->in.u.manage_vf_pble_bp.info, - pcmdinfo->in.u.manage_vf_pble_bp.scratch, true); - break; - case OP_QUERY_FPM_VALUES: - values_mem.pa = pcmdinfo->in.u.query_fpm_values.fpm_values_pa; - values_mem.va = pcmdinfo->in.u.query_fpm_values.fpm_values_va; - status = i40iw_sc_query_fpm_values( - pcmdinfo->in.u.query_fpm_values.cqp, - pcmdinfo->in.u.query_fpm_values.scratch, - pcmdinfo->in.u.query_fpm_values.hmc_fn_id, - &values_mem, true, I40IW_CQP_WAIT_EVENT); - break; - case OP_COMMIT_FPM_VALUES: - values_mem.pa = pcmdinfo->in.u.commit_fpm_values.fpm_values_pa; - values_mem.va = pcmdinfo->in.u.commit_fpm_values.fpm_values_va; - status = i40iw_sc_commit_fpm_values( - pcmdinfo->in.u.commit_fpm_values.cqp, - pcmdinfo->in.u.commit_fpm_values.scratch, - pcmdinfo->in.u.commit_fpm_values.hmc_fn_id, - &values_mem, - true, - I40IW_CQP_WAIT_EVENT); - break; - case OP_QUERY_RDMA_FEATURES: - values_mem.pa = pcmdinfo->in.u.query_rdma_features.cap_pa; - values_mem.va = pcmdinfo->in.u.query_rdma_features.cap_va; - status = i40iw_sc_query_rdma_features( - pcmdinfo->in.u.query_rdma_features.cqp, &values_mem, - pcmdinfo->in.u.query_rdma_features.scratch); - break; - default: - status = I40IW_NOT_SUPPORTED; - break; - } - - return status; -} - -/** - * i40iw_process_cqp_cmd - process all cqp commands - * @dev: sc device struct - * @pcmdinfo: cqp command info - */ -enum i40iw_status_code i40iw_process_cqp_cmd(struct i40iw_sc_dev *dev, - struct cqp_commands_info *pcmdinfo) -{ - enum i40iw_status_code status = 0; - unsigned long flags; - - spin_lock_irqsave(&dev->cqp_lock, flags); - if (list_empty(&dev->cqp_cmd_head) && !i40iw_ring_full(dev->cqp)) - status = i40iw_exec_cqp_cmd(dev, pcmdinfo); - else - list_add_tail(&pcmdinfo->cqp_cmd_entry, &dev->cqp_cmd_head); - spin_unlock_irqrestore(&dev->cqp_lock, flags); - return status; -} - -/** - * i40iw_process_bh - called from tasklet for cqp list - * @dev: sc device struct - */ -enum i40iw_status_code i40iw_process_bh(struct i40iw_sc_dev *dev) -{ - enum i40iw_status_code status = 0; - struct cqp_commands_info *pcmdinfo; - unsigned long flags; - - spin_lock_irqsave(&dev->cqp_lock, flags); - while (!list_empty(&dev->cqp_cmd_head) && !i40iw_ring_full(dev->cqp)) { - pcmdinfo = (struct cqp_commands_info *)i40iw_remove_head(&dev->cqp_cmd_head); - - status = i40iw_exec_cqp_cmd(dev, pcmdinfo); - if (status) - break; - } - spin_unlock_irqrestore(&dev->cqp_lock, flags); - return status; -} - -/** - * i40iw_iwarp_opcode - determine if incoming is rdma layer - * @info: aeq info for the packet - * @pkt: packet for error - */ -static u32 i40iw_iwarp_opcode(struct i40iw_aeqe_info *info, u8 *pkt) -{ - __be16 *mpa; - u32 opcode = 0xffffffff; - - if (info->q2_data_written) { - mpa = (__be16 *)pkt; - opcode = ntohs(mpa[1]) & 0xf; - } - return opcode; -} - -/** - * i40iw_locate_mpa - return pointer to mpa in the pkt - * @pkt: packet with data - */ -static u8 *i40iw_locate_mpa(u8 *pkt) -{ - /* skip over ethernet header */ - pkt += I40IW_MAC_HLEN; - - /* Skip over IP and TCP headers */ - pkt += 4 * (pkt[0] & 0x0f); - pkt += 4 * ((pkt[12] >> 4) & 0x0f); - return pkt; -} - -/** - * i40iw_setup_termhdr - termhdr for terminate pkt - * @qp: sc qp ptr for pkt - * @hdr: term hdr - * @opcode: flush opcode for termhdr - * @layer_etype: error layer + error type - * @err: error cod ein the header - */ -static void i40iw_setup_termhdr(struct i40iw_sc_qp *qp, - struct i40iw_terminate_hdr *hdr, - enum i40iw_flush_opcode opcode, - u8 layer_etype, - u8 err) -{ - qp->flush_code = opcode; - hdr->layer_etype = layer_etype; - hdr->error_code = err; -} - -/** - * i40iw_bld_terminate_hdr - build terminate message header - * @qp: qp associated with received terminate AE - * @info: the struct contiaing AE information - */ -static int i40iw_bld_terminate_hdr(struct i40iw_sc_qp *qp, - struct i40iw_aeqe_info *info) -{ - u8 *pkt = qp->q2_buf + Q2_BAD_FRAME_OFFSET; - u16 ddp_seg_len; - int copy_len = 0; - u8 is_tagged = 0; - u32 opcode; - struct i40iw_terminate_hdr *termhdr; - - termhdr = (struct i40iw_terminate_hdr *)qp->q2_buf; - memset(termhdr, 0, Q2_BAD_FRAME_OFFSET); - - if (info->q2_data_written) { - /* Use data from offending packet to fill in ddp & rdma hdrs */ - pkt = i40iw_locate_mpa(pkt); - ddp_seg_len = ntohs(*(__be16 *)pkt); - if (ddp_seg_len) { - copy_len = 2; - termhdr->hdrct = DDP_LEN_FLAG; - if (pkt[2] & 0x80) { - is_tagged = 1; - if (ddp_seg_len >= TERM_DDP_LEN_TAGGED) { - copy_len += TERM_DDP_LEN_TAGGED; - termhdr->hdrct |= DDP_HDR_FLAG; - } - } else { - if (ddp_seg_len >= TERM_DDP_LEN_UNTAGGED) { - copy_len += TERM_DDP_LEN_UNTAGGED; - termhdr->hdrct |= DDP_HDR_FLAG; - } - - if (ddp_seg_len >= (TERM_DDP_LEN_UNTAGGED + TERM_RDMA_LEN)) { - if ((pkt[3] & RDMA_OPCODE_MASK) == RDMA_READ_REQ_OPCODE) { - copy_len += TERM_RDMA_LEN; - termhdr->hdrct |= RDMA_HDR_FLAG; - } - } - } - } - } - - opcode = i40iw_iwarp_opcode(info, pkt); - - switch (info->ae_id) { - case I40IW_AE_AMP_UNALLOCATED_STAG: - qp->eventtype = TERM_EVENT_QP_ACCESS_ERR; - if (opcode == I40IW_OP_TYPE_RDMA_WRITE) - i40iw_setup_termhdr(qp, termhdr, FLUSH_PROT_ERR, - (LAYER_DDP << 4) | DDP_TAGGED_BUFFER, DDP_TAGGED_INV_STAG); - else - i40iw_setup_termhdr(qp, termhdr, FLUSH_REM_ACCESS_ERR, - (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT, RDMAP_INV_STAG); - break; - case I40IW_AE_AMP_BOUNDS_VIOLATION: - qp->eventtype = TERM_EVENT_QP_ACCESS_ERR; - if (info->q2_data_written) - i40iw_setup_termhdr(qp, termhdr, FLUSH_PROT_ERR, - (LAYER_DDP << 4) | DDP_TAGGED_BUFFER, DDP_TAGGED_BOUNDS); - else - i40iw_setup_termhdr(qp, termhdr, FLUSH_REM_ACCESS_ERR, - (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT, RDMAP_INV_BOUNDS); - break; - case I40IW_AE_AMP_BAD_PD: - switch (opcode) { - case I40IW_OP_TYPE_RDMA_WRITE: - i40iw_setup_termhdr(qp, termhdr, FLUSH_PROT_ERR, - (LAYER_DDP << 4) | DDP_TAGGED_BUFFER, DDP_TAGGED_UNASSOC_STAG); - break; - case I40IW_OP_TYPE_SEND_INV: - case I40IW_OP_TYPE_SEND_SOL_INV: - i40iw_setup_termhdr(qp, termhdr, FLUSH_REM_ACCESS_ERR, - (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT, RDMAP_CANT_INV_STAG); - break; - default: - i40iw_setup_termhdr(qp, termhdr, FLUSH_REM_ACCESS_ERR, - (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT, RDMAP_UNASSOC_STAG); - } - break; - case I40IW_AE_AMP_INVALID_STAG: - qp->eventtype = TERM_EVENT_QP_ACCESS_ERR; - i40iw_setup_termhdr(qp, termhdr, FLUSH_REM_ACCESS_ERR, - (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT, RDMAP_INV_STAG); - break; - case I40IW_AE_AMP_BAD_QP: - i40iw_setup_termhdr(qp, termhdr, FLUSH_LOC_QP_OP_ERR, - (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER, DDP_UNTAGGED_INV_QN); - break; - case I40IW_AE_AMP_BAD_STAG_KEY: - case I40IW_AE_AMP_BAD_STAG_INDEX: - qp->eventtype = TERM_EVENT_QP_ACCESS_ERR; - switch (opcode) { - case I40IW_OP_TYPE_SEND_INV: - case I40IW_OP_TYPE_SEND_SOL_INV: - i40iw_setup_termhdr(qp, termhdr, FLUSH_REM_OP_ERR, - (LAYER_RDMA << 4) | RDMAP_REMOTE_OP, RDMAP_CANT_INV_STAG); - break; - default: - i40iw_setup_termhdr(qp, termhdr, FLUSH_REM_ACCESS_ERR, - (LAYER_RDMA << 4) | RDMAP_REMOTE_OP, RDMAP_INV_STAG); - } - break; - case I40IW_AE_AMP_RIGHTS_VIOLATION: - case I40IW_AE_AMP_INVALIDATE_NO_REMOTE_ACCESS_RIGHTS: - case I40IW_AE_PRIV_OPERATION_DENIED: - qp->eventtype = TERM_EVENT_QP_ACCESS_ERR; - i40iw_setup_termhdr(qp, termhdr, FLUSH_REM_ACCESS_ERR, - (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT, RDMAP_ACCESS); - break; - case I40IW_AE_AMP_TO_WRAP: - qp->eventtype = TERM_EVENT_QP_ACCESS_ERR; - i40iw_setup_termhdr(qp, termhdr, FLUSH_REM_ACCESS_ERR, - (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT, RDMAP_TO_WRAP); - break; - case I40IW_AE_LLP_RECEIVED_MPA_CRC_ERROR: - i40iw_setup_termhdr(qp, termhdr, FLUSH_GENERAL_ERR, - (LAYER_MPA << 4) | DDP_LLP, MPA_CRC); - break; - case I40IW_AE_LLP_SEGMENT_TOO_LARGE: - case I40IW_AE_LLP_SEGMENT_TOO_SMALL: - i40iw_setup_termhdr(qp, termhdr, FLUSH_LOC_LEN_ERR, - (LAYER_DDP << 4) | DDP_CATASTROPHIC, DDP_CATASTROPHIC_LOCAL); - break; - case I40IW_AE_LCE_QP_CATASTROPHIC: - case I40IW_AE_DDP_NO_L_BIT: - i40iw_setup_termhdr(qp, termhdr, FLUSH_FATAL_ERR, - (LAYER_DDP << 4) | DDP_CATASTROPHIC, DDP_CATASTROPHIC_LOCAL); - break; - case I40IW_AE_DDP_INVALID_MSN_GAP_IN_MSN: - i40iw_setup_termhdr(qp, termhdr, FLUSH_GENERAL_ERR, - (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER, DDP_UNTAGGED_INV_MSN_RANGE); - break; - case I40IW_AE_DDP_UBE_DDP_MESSAGE_TOO_LONG_FOR_AVAILABLE_BUFFER: - qp->eventtype = TERM_EVENT_QP_ACCESS_ERR; - i40iw_setup_termhdr(qp, termhdr, FLUSH_LOC_LEN_ERR, - (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER, DDP_UNTAGGED_INV_TOO_LONG); - break; - case I40IW_AE_DDP_UBE_INVALID_DDP_VERSION: - if (is_tagged) - i40iw_setup_termhdr(qp, termhdr, FLUSH_GENERAL_ERR, - (LAYER_DDP << 4) | DDP_TAGGED_BUFFER, DDP_TAGGED_INV_DDP_VER); - else - i40iw_setup_termhdr(qp, termhdr, FLUSH_GENERAL_ERR, - (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER, DDP_UNTAGGED_INV_DDP_VER); - break; - case I40IW_AE_DDP_UBE_INVALID_MO: - i40iw_setup_termhdr(qp, termhdr, FLUSH_GENERAL_ERR, - (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER, DDP_UNTAGGED_INV_MO); - break; - case I40IW_AE_DDP_UBE_INVALID_MSN_NO_BUFFER_AVAILABLE: - i40iw_setup_termhdr(qp, termhdr, FLUSH_REM_OP_ERR, - (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER, DDP_UNTAGGED_INV_MSN_NO_BUF); - break; - case I40IW_AE_DDP_UBE_INVALID_QN: - i40iw_setup_termhdr(qp, termhdr, FLUSH_GENERAL_ERR, - (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER, DDP_UNTAGGED_INV_QN); - break; - case I40IW_AE_RDMAP_ROE_INVALID_RDMAP_VERSION: - i40iw_setup_termhdr(qp, termhdr, FLUSH_GENERAL_ERR, - (LAYER_RDMA << 4) | RDMAP_REMOTE_OP, RDMAP_INV_RDMAP_VER); - break; - case I40IW_AE_RDMAP_ROE_UNEXPECTED_OPCODE: - i40iw_setup_termhdr(qp, termhdr, FLUSH_LOC_QP_OP_ERR, - (LAYER_RDMA << 4) | RDMAP_REMOTE_OP, RDMAP_UNEXPECTED_OP); - break; - default: - i40iw_setup_termhdr(qp, termhdr, FLUSH_FATAL_ERR, - (LAYER_RDMA << 4) | RDMAP_REMOTE_OP, RDMAP_UNSPECIFIED); - break; - } - - if (copy_len) - memcpy(termhdr + 1, pkt, copy_len); - - return sizeof(struct i40iw_terminate_hdr) + copy_len; -} - -/** - * i40iw_terminate_send_fin() - Send fin for terminate message - * @qp: qp associated with received terminate AE - */ -void i40iw_terminate_send_fin(struct i40iw_sc_qp *qp) -{ - /* Send the fin only */ - i40iw_term_modify_qp(qp, - I40IW_QP_STATE_TERMINATE, - I40IWQP_TERM_SEND_FIN_ONLY, - 0); -} - -/** - * i40iw_terminate_connection() - Bad AE and send terminate to remote QP - * @qp: qp associated with received terminate AE - * @info: the struct contiaing AE information - */ -void i40iw_terminate_connection(struct i40iw_sc_qp *qp, struct i40iw_aeqe_info *info) -{ - u8 termlen = 0; - - if (qp->term_flags & I40IW_TERM_SENT) - return; /* Sanity check */ - - /* Eventtype can change from bld_terminate_hdr */ - qp->eventtype = TERM_EVENT_QP_FATAL; - termlen = i40iw_bld_terminate_hdr(qp, info); - i40iw_terminate_start_timer(qp); - qp->term_flags |= I40IW_TERM_SENT; - i40iw_term_modify_qp(qp, I40IW_QP_STATE_TERMINATE, - I40IWQP_TERM_SEND_TERM_ONLY, termlen); -} - -/** - * i40iw_terminate_received - handle terminate received AE - * @qp: qp associated with received terminate AE - * @info: the struct contiaing AE information - */ -void i40iw_terminate_received(struct i40iw_sc_qp *qp, struct i40iw_aeqe_info *info) -{ - u8 *pkt = qp->q2_buf + Q2_BAD_FRAME_OFFSET; - __be32 *mpa; - u8 ddp_ctl; - u8 rdma_ctl; - u16 aeq_id = 0; - struct i40iw_terminate_hdr *termhdr; - - mpa = (__be32 *)i40iw_locate_mpa(pkt); - if (info->q2_data_written) { - /* did not validate the frame - do it now */ - ddp_ctl = (ntohl(mpa[0]) >> 8) & 0xff; - rdma_ctl = ntohl(mpa[0]) & 0xff; - if ((ddp_ctl & 0xc0) != 0x40) - aeq_id = I40IW_AE_LCE_QP_CATASTROPHIC; - else if ((ddp_ctl & 0x03) != 1) - aeq_id = I40IW_AE_DDP_UBE_INVALID_DDP_VERSION; - else if (ntohl(mpa[2]) != 2) - aeq_id = I40IW_AE_DDP_UBE_INVALID_QN; - else if (ntohl(mpa[3]) != 1) - aeq_id = I40IW_AE_DDP_INVALID_MSN_GAP_IN_MSN; - else if (ntohl(mpa[4]) != 0) - aeq_id = I40IW_AE_DDP_UBE_INVALID_MO; - else if ((rdma_ctl & 0xc0) != 0x40) - aeq_id = I40IW_AE_RDMAP_ROE_INVALID_RDMAP_VERSION; - - info->ae_id = aeq_id; - if (info->ae_id) { - /* Bad terminate recvd - send back a terminate */ - i40iw_terminate_connection(qp, info); - return; - } - } - - qp->term_flags |= I40IW_TERM_RCVD; - qp->eventtype = TERM_EVENT_QP_FATAL; - termhdr = (struct i40iw_terminate_hdr *)&mpa[5]; - if (termhdr->layer_etype == RDMAP_REMOTE_PROT || - termhdr->layer_etype == RDMAP_REMOTE_OP) { - i40iw_terminate_done(qp, 0); - } else { - i40iw_terminate_start_timer(qp); - i40iw_terminate_send_fin(qp); - } -} - -/** - * i40iw_sc_vsi_init - Initialize virtual device - * @vsi: pointer to the vsi structure - * @info: parameters to initialize vsi - **/ -void i40iw_sc_vsi_init(struct i40iw_sc_vsi *vsi, struct i40iw_vsi_init_info *info) -{ - int i; - - vsi->dev = info->dev; - vsi->back_vsi = info->back_vsi; - vsi->mtu = info->params->mtu; - vsi->exception_lan_queue = info->exception_lan_queue; - i40iw_fill_qos_list(info->params->qs_handle_list); - - for (i = 0; i < I40IW_MAX_USER_PRIORITY; i++) { - vsi->qos[i].qs_handle = info->params->qs_handle_list[i]; - i40iw_debug(vsi->dev, I40IW_DEBUG_DCB, "qset[%d]: %d\n", i, - vsi->qos[i].qs_handle); - spin_lock_init(&vsi->qos[i].lock); - INIT_LIST_HEAD(&vsi->qos[i].qplist); - } -} - -/** - * i40iw_hw_stats_init - Initiliaze HW stats table - * @stats: pestat struct - * @fcn_idx: PCI fn id - * @is_pf: Is it a PF? - * - * Populate the HW stats table with register offset addr for each - * stats. And start the perioidic stats timer. - */ -void i40iw_hw_stats_init(struct i40iw_vsi_pestat *stats, u8 fcn_idx, bool is_pf) -{ - u32 stats_reg_offset; - u32 stats_index; - struct i40iw_dev_hw_stats_offsets *stats_table = - &stats->hw_stats_offsets; - struct i40iw_dev_hw_stats *last_rd_stats = &stats->last_read_hw_stats; - - if (is_pf) { - stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_IP4RXDISCARD] = - I40E_GLPES_PFIP4RXDISCARD(fcn_idx); - stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_IP4RXTRUNC] = - I40E_GLPES_PFIP4RXTRUNC(fcn_idx); - stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_IP4TXNOROUTE] = - I40E_GLPES_PFIP4TXNOROUTE(fcn_idx); - stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_IP6RXDISCARD] = - I40E_GLPES_PFIP6RXDISCARD(fcn_idx); - stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_IP6RXTRUNC] = - I40E_GLPES_PFIP6RXTRUNC(fcn_idx); - stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_IP6TXNOROUTE] = - I40E_GLPES_PFIP6TXNOROUTE(fcn_idx); - stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_TCPRTXSEG] = - I40E_GLPES_PFTCPRTXSEG(fcn_idx); - stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_TCPRXOPTERR] = - I40E_GLPES_PFTCPRXOPTERR(fcn_idx); - stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_TCPRXPROTOERR] = - I40E_GLPES_PFTCPRXPROTOERR(fcn_idx); - - stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP4RXOCTS] = - I40E_GLPES_PFIP4RXOCTSLO(fcn_idx); - stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP4RXPKTS] = - I40E_GLPES_PFIP4RXPKTSLO(fcn_idx); - stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP4RXFRAGS] = - I40E_GLPES_PFIP4RXFRAGSLO(fcn_idx); - stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP4RXMCPKTS] = - I40E_GLPES_PFIP4RXMCPKTSLO(fcn_idx); - stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP4TXOCTS] = - I40E_GLPES_PFIP4TXOCTSLO(fcn_idx); - stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP4TXPKTS] = - I40E_GLPES_PFIP4TXPKTSLO(fcn_idx); - stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP4TXFRAGS] = - I40E_GLPES_PFIP4TXFRAGSLO(fcn_idx); - stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP4TXMCPKTS] = - I40E_GLPES_PFIP4TXMCPKTSLO(fcn_idx); - stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP6RXOCTS] = - I40E_GLPES_PFIP6RXOCTSLO(fcn_idx); - stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP6RXPKTS] = - I40E_GLPES_PFIP6RXPKTSLO(fcn_idx); - stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP6RXFRAGS] = - I40E_GLPES_PFIP6RXFRAGSLO(fcn_idx); - stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP6RXMCPKTS] = - I40E_GLPES_PFIP6RXMCPKTSLO(fcn_idx); - stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP6TXOCTS] = - I40E_GLPES_PFIP6TXOCTSLO(fcn_idx); - stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP6TXPKTS] = - I40E_GLPES_PFIP6TXPKTSLO(fcn_idx); - stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP6TXPKTS] = - I40E_GLPES_PFIP6TXPKTSLO(fcn_idx); - stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP6TXFRAGS] = - I40E_GLPES_PFIP6TXFRAGSLO(fcn_idx); - stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_TCPRXSEGS] = - I40E_GLPES_PFTCPRXSEGSLO(fcn_idx); - stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_TCPTXSEG] = - I40E_GLPES_PFTCPTXSEGLO(fcn_idx); - stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_RDMARXRDS] = - I40E_GLPES_PFRDMARXRDSLO(fcn_idx); - stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_RDMARXSNDS] = - I40E_GLPES_PFRDMARXSNDSLO(fcn_idx); - stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_RDMARXWRS] = - I40E_GLPES_PFRDMARXWRSLO(fcn_idx); - stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_RDMATXRDS] = - I40E_GLPES_PFRDMATXRDSLO(fcn_idx); - stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_RDMATXSNDS] = - I40E_GLPES_PFRDMATXSNDSLO(fcn_idx); - stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_RDMATXWRS] = - I40E_GLPES_PFRDMATXWRSLO(fcn_idx); - stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_RDMAVBND] = - I40E_GLPES_PFRDMAVBNDLO(fcn_idx); - stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_RDMAVINV] = - I40E_GLPES_PFRDMAVINVLO(fcn_idx); - } else { - stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_IP4RXDISCARD] = - I40E_GLPES_VFIP4RXDISCARD(fcn_idx); - stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_IP4RXTRUNC] = - I40E_GLPES_VFIP4RXTRUNC(fcn_idx); - stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_IP4TXNOROUTE] = - I40E_GLPES_VFIP4TXNOROUTE(fcn_idx); - stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_IP6RXDISCARD] = - I40E_GLPES_VFIP6RXDISCARD(fcn_idx); - stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_IP6RXTRUNC] = - I40E_GLPES_VFIP6RXTRUNC(fcn_idx); - stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_IP6TXNOROUTE] = - I40E_GLPES_VFIP6TXNOROUTE(fcn_idx); - stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_TCPRTXSEG] = - I40E_GLPES_VFTCPRTXSEG(fcn_idx); - stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_TCPRXOPTERR] = - I40E_GLPES_VFTCPRXOPTERR(fcn_idx); - stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_TCPRXPROTOERR] = - I40E_GLPES_VFTCPRXPROTOERR(fcn_idx); - - stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP4RXOCTS] = - I40E_GLPES_VFIP4RXOCTSLO(fcn_idx); - stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP4RXPKTS] = - I40E_GLPES_VFIP4RXPKTSLO(fcn_idx); - stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP4RXFRAGS] = - I40E_GLPES_VFIP4RXFRAGSLO(fcn_idx); - stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP4RXMCPKTS] = - I40E_GLPES_VFIP4RXMCPKTSLO(fcn_idx); - stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP4TXOCTS] = - I40E_GLPES_VFIP4TXOCTSLO(fcn_idx); - stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP4TXPKTS] = - I40E_GLPES_VFIP4TXPKTSLO(fcn_idx); - stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP4TXFRAGS] = - I40E_GLPES_VFIP4TXFRAGSLO(fcn_idx); - stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP4TXMCPKTS] = - I40E_GLPES_VFIP4TXMCPKTSLO(fcn_idx); - stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP6RXOCTS] = - I40E_GLPES_VFIP6RXOCTSLO(fcn_idx); - stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP6RXPKTS] = - I40E_GLPES_VFIP6RXPKTSLO(fcn_idx); - stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP6RXFRAGS] = - I40E_GLPES_VFIP6RXFRAGSLO(fcn_idx); - stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP6RXMCPKTS] = - I40E_GLPES_VFIP6RXMCPKTSLO(fcn_idx); - stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP6TXOCTS] = - I40E_GLPES_VFIP6TXOCTSLO(fcn_idx); - stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP6TXPKTS] = - I40E_GLPES_VFIP6TXPKTSLO(fcn_idx); - stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP6TXPKTS] = - I40E_GLPES_VFIP6TXPKTSLO(fcn_idx); - stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP6TXFRAGS] = - I40E_GLPES_VFIP6TXFRAGSLO(fcn_idx); - stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_TCPRXSEGS] = - I40E_GLPES_VFTCPRXSEGSLO(fcn_idx); - stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_TCPTXSEG] = - I40E_GLPES_VFTCPTXSEGLO(fcn_idx); - stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_RDMARXRDS] = - I40E_GLPES_VFRDMARXRDSLO(fcn_idx); - stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_RDMARXSNDS] = - I40E_GLPES_VFRDMARXSNDSLO(fcn_idx); - stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_RDMARXWRS] = - I40E_GLPES_VFRDMARXWRSLO(fcn_idx); - stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_RDMATXRDS] = - I40E_GLPES_VFRDMATXRDSLO(fcn_idx); - stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_RDMATXSNDS] = - I40E_GLPES_VFRDMATXSNDSLO(fcn_idx); - stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_RDMATXWRS] = - I40E_GLPES_VFRDMATXWRSLO(fcn_idx); - stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_RDMAVBND] = - I40E_GLPES_VFRDMAVBNDLO(fcn_idx); - stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_RDMAVINV] = - I40E_GLPES_VFRDMAVINVLO(fcn_idx); - } - - for (stats_index = 0; stats_index < I40IW_HW_STAT_INDEX_MAX_64; - stats_index++) { - stats_reg_offset = stats_table->stats_offset_64[stats_index]; - last_rd_stats->stats_value_64[stats_index] = - readq(stats->hw->hw_addr + stats_reg_offset); - } - - for (stats_index = 0; stats_index < I40IW_HW_STAT_INDEX_MAX_32; - stats_index++) { - stats_reg_offset = stats_table->stats_offset_32[stats_index]; - last_rd_stats->stats_value_32[stats_index] = - i40iw_rd32(stats->hw, stats_reg_offset); - } -} - -/** - * i40iw_hw_stats_read_32 - Read 32-bit HW stats counters and accommodates for roll-overs. - * @stats: pestat struct - * @index: index in HW stats table which contains offset reg-addr - * @value: hw stats value - */ -void i40iw_hw_stats_read_32(struct i40iw_vsi_pestat *stats, - enum i40iw_hw_stats_index_32b index, - u64 *value) -{ - struct i40iw_dev_hw_stats_offsets *stats_table = - &stats->hw_stats_offsets; - struct i40iw_dev_hw_stats *last_rd_stats = &stats->last_read_hw_stats; - struct i40iw_dev_hw_stats *hw_stats = &stats->hw_stats; - u64 new_stats_value = 0; - u32 stats_reg_offset = stats_table->stats_offset_32[index]; - - new_stats_value = i40iw_rd32(stats->hw, stats_reg_offset); - /*roll-over case */ - if (new_stats_value < last_rd_stats->stats_value_32[index]) - hw_stats->stats_value_32[index] += new_stats_value; - else - hw_stats->stats_value_32[index] += - new_stats_value - last_rd_stats->stats_value_32[index]; - last_rd_stats->stats_value_32[index] = new_stats_value; - *value = hw_stats->stats_value_32[index]; -} - -/** - * i40iw_hw_stats_read_64 - Read HW stats counters (greater than 32-bit) and accommodates for roll-overs. - * @stats: pestat struct - * @index: index in HW stats table which contains offset reg-addr - * @value: hw stats value - */ -void i40iw_hw_stats_read_64(struct i40iw_vsi_pestat *stats, - enum i40iw_hw_stats_index_64b index, - u64 *value) -{ - struct i40iw_dev_hw_stats_offsets *stats_table = - &stats->hw_stats_offsets; - struct i40iw_dev_hw_stats *last_rd_stats = &stats->last_read_hw_stats; - struct i40iw_dev_hw_stats *hw_stats = &stats->hw_stats; - u64 new_stats_value = 0; - u32 stats_reg_offset = stats_table->stats_offset_64[index]; - - new_stats_value = readq(stats->hw->hw_addr + stats_reg_offset); - /*roll-over case */ - if (new_stats_value < last_rd_stats->stats_value_64[index]) - hw_stats->stats_value_64[index] += new_stats_value; - else - hw_stats->stats_value_64[index] += - new_stats_value - last_rd_stats->stats_value_64[index]; - last_rd_stats->stats_value_64[index] = new_stats_value; - *value = hw_stats->stats_value_64[index]; -} - -/** - * i40iw_hw_stats_read_all - read all HW stat counters - * @stats: pestat struct - * @stats_values: hw stats structure - * - * Read all the HW stat counters and populates hw_stats structure - * of passed-in vsi's pestat as well as copy created in stat_values. - */ -void i40iw_hw_stats_read_all(struct i40iw_vsi_pestat *stats, - struct i40iw_dev_hw_stats *stats_values) -{ - u32 stats_index; - unsigned long flags; - - spin_lock_irqsave(&stats->lock, flags); - - for (stats_index = 0; stats_index < I40IW_HW_STAT_INDEX_MAX_32; - stats_index++) - i40iw_hw_stats_read_32(stats, stats_index, - &stats_values->stats_value_32[stats_index]); - for (stats_index = 0; stats_index < I40IW_HW_STAT_INDEX_MAX_64; - stats_index++) - i40iw_hw_stats_read_64(stats, stats_index, - &stats_values->stats_value_64[stats_index]); - spin_unlock_irqrestore(&stats->lock, flags); -} - -/** - * i40iw_hw_stats_refresh_all - Update all HW stats structs - * @stats: pestat struct - * - * Read all the HW stats counters to refresh values in hw_stats structure - * of passed-in dev's pestat - */ -void i40iw_hw_stats_refresh_all(struct i40iw_vsi_pestat *stats) -{ - u64 stats_value; - u32 stats_index; - unsigned long flags; - - spin_lock_irqsave(&stats->lock, flags); - - for (stats_index = 0; stats_index < I40IW_HW_STAT_INDEX_MAX_32; - stats_index++) - i40iw_hw_stats_read_32(stats, stats_index, &stats_value); - for (stats_index = 0; stats_index < I40IW_HW_STAT_INDEX_MAX_64; - stats_index++) - i40iw_hw_stats_read_64(stats, stats_index, &stats_value); - spin_unlock_irqrestore(&stats->lock, flags); -} - -/** - * i40iw_get_fcn_id - Return the function id - * @dev: pointer to the device - */ -static u8 i40iw_get_fcn_id(struct i40iw_sc_dev *dev) -{ - u8 fcn_id = I40IW_INVALID_FCN_ID; - u8 i; - - for (i = I40IW_FIRST_NON_PF_STAT; i < I40IW_MAX_STATS_COUNT; i++) - if (!dev->fcn_id_array[i]) { - fcn_id = i; - dev->fcn_id_array[i] = true; - break; - } - return fcn_id; -} - -/** - * i40iw_vsi_stats_init - Initialize the vsi statistics - * @vsi: pointer to the vsi structure - * @info: The info structure used for initialization - */ -enum i40iw_status_code i40iw_vsi_stats_init(struct i40iw_sc_vsi *vsi, struct i40iw_vsi_stats_info *info) -{ - u8 fcn_id = info->fcn_id; - - if (info->alloc_fcn_id) - fcn_id = i40iw_get_fcn_id(vsi->dev); - - if (fcn_id == I40IW_INVALID_FCN_ID) - return I40IW_ERR_NOT_READY; - - vsi->pestat = info->pestat; - vsi->pestat->hw = vsi->dev->hw; - vsi->pestat->vsi = vsi; - - if (info->stats_initialize) { - i40iw_hw_stats_init(vsi->pestat, fcn_id, true); - spin_lock_init(&vsi->pestat->lock); - i40iw_hw_stats_start_timer(vsi); - } - vsi->stats_fcn_id_alloc = info->alloc_fcn_id; - vsi->fcn_id = fcn_id; - return I40IW_SUCCESS; -} - -/** - * i40iw_vsi_stats_free - Free the vsi stats - * @vsi: pointer to the vsi structure - */ -void i40iw_vsi_stats_free(struct i40iw_sc_vsi *vsi) -{ - u8 fcn_id = vsi->fcn_id; - - if (vsi->stats_fcn_id_alloc && fcn_id < I40IW_MAX_STATS_COUNT) - vsi->dev->fcn_id_array[fcn_id] = false; - i40iw_hw_stats_stop_timer(vsi); -} - -static const struct i40iw_cqp_ops iw_cqp_ops = { - .cqp_init = i40iw_sc_cqp_init, - .cqp_create = i40iw_sc_cqp_create, - .cqp_post_sq = i40iw_sc_cqp_post_sq, - .cqp_get_next_send_wqe = i40iw_sc_cqp_get_next_send_wqe, - .cqp_destroy = i40iw_sc_cqp_destroy, - .poll_for_cqp_op_done = i40iw_sc_poll_for_cqp_op_done -}; - -static const struct i40iw_ccq_ops iw_ccq_ops = { - .ccq_init = i40iw_sc_ccq_init, - .ccq_create = i40iw_sc_ccq_create, - .ccq_destroy = i40iw_sc_ccq_destroy, - .ccq_create_done = i40iw_sc_ccq_create_done, - .ccq_get_cqe_info = i40iw_sc_ccq_get_cqe_info, - .ccq_arm = i40iw_sc_ccq_arm -}; - -static const struct i40iw_ceq_ops iw_ceq_ops = { - .ceq_init = i40iw_sc_ceq_init, - .ceq_create = i40iw_sc_ceq_create, - .cceq_create_done = i40iw_sc_cceq_create_done, - .cceq_destroy_done = i40iw_sc_cceq_destroy_done, - .cceq_create = i40iw_sc_cceq_create, - .ceq_destroy = i40iw_sc_ceq_destroy, - .process_ceq = i40iw_sc_process_ceq -}; - -static const struct i40iw_aeq_ops iw_aeq_ops = { - .aeq_init = i40iw_sc_aeq_init, - .aeq_create = i40iw_sc_aeq_create, - .aeq_destroy = i40iw_sc_aeq_destroy, - .get_next_aeqe = i40iw_sc_get_next_aeqe, - .repost_aeq_entries = i40iw_sc_repost_aeq_entries, - .aeq_create_done = i40iw_sc_aeq_create_done, - .aeq_destroy_done = i40iw_sc_aeq_destroy_done -}; - -/* iwarp pd ops */ -static const struct i40iw_pd_ops iw_pd_ops = { - .pd_init = i40iw_sc_pd_init, -}; - -static const struct i40iw_priv_qp_ops iw_priv_qp_ops = { - .qp_init = i40iw_sc_qp_init, - .qp_create = i40iw_sc_qp_create, - .qp_modify = i40iw_sc_qp_modify, - .qp_destroy = i40iw_sc_qp_destroy, - .qp_flush_wqes = i40iw_sc_qp_flush_wqes, - .qp_upload_context = i40iw_sc_qp_upload_context, - .qp_setctx = i40iw_sc_qp_setctx, - .qp_send_lsmm = i40iw_sc_send_lsmm, - .qp_send_lsmm_nostag = i40iw_sc_send_lsmm_nostag, - .qp_send_rtt = i40iw_sc_send_rtt, - .qp_post_wqe0 = i40iw_sc_post_wqe0, - .iw_mr_fast_register = i40iw_sc_mr_fast_register -}; - -static const struct i40iw_priv_cq_ops iw_priv_cq_ops = { - .cq_init = i40iw_sc_cq_init, - .cq_create = i40iw_sc_cq_create, - .cq_destroy = i40iw_sc_cq_destroy, - .cq_modify = i40iw_sc_cq_modify, -}; - -static const struct i40iw_mr_ops iw_mr_ops = { - .alloc_stag = i40iw_sc_alloc_stag, - .mr_reg_non_shared = i40iw_sc_mr_reg_non_shared, - .mr_reg_shared = i40iw_sc_mr_reg_shared, - .dealloc_stag = i40iw_sc_dealloc_stag, - .query_stag = i40iw_sc_query_stag, - .mw_alloc = i40iw_sc_mw_alloc -}; - -static const struct i40iw_cqp_misc_ops iw_cqp_misc_ops = { - .manage_hmc_pm_func_table = i40iw_sc_manage_hmc_pm_func_table, - .set_hmc_resource_profile = i40iw_sc_set_hmc_resource_profile, - .commit_fpm_values = i40iw_sc_commit_fpm_values, - .query_fpm_values = i40iw_sc_query_fpm_values, - .static_hmc_pages_allocated = i40iw_sc_static_hmc_pages_allocated, - .add_arp_cache_entry = i40iw_sc_add_arp_cache_entry, - .del_arp_cache_entry = i40iw_sc_del_arp_cache_entry, - .query_arp_cache_entry = i40iw_sc_query_arp_cache_entry, - .manage_apbvt_entry = i40iw_sc_manage_apbvt_entry, - .manage_qhash_table_entry = i40iw_sc_manage_qhash_table_entry, - .alloc_local_mac_ipaddr_table_entry = i40iw_sc_alloc_local_mac_ipaddr_entry, - .add_local_mac_ipaddr_entry = i40iw_sc_add_local_mac_ipaddr_entry, - .del_local_mac_ipaddr_entry = i40iw_sc_del_local_mac_ipaddr_entry, - .cqp_nop = i40iw_sc_cqp_nop, - .commit_fpm_values_done = i40iw_sc_commit_fpm_values_done, - .query_fpm_values_done = i40iw_sc_query_fpm_values_done, - .manage_hmc_pm_func_table_done = i40iw_sc_manage_hmc_pm_func_table_done, - .update_suspend_qp = i40iw_sc_suspend_qp, - .update_resume_qp = i40iw_sc_resume_qp -}; - -static const struct i40iw_hmc_ops iw_hmc_ops = { - .init_iw_hmc = i40iw_sc_init_iw_hmc, - .parse_fpm_query_buf = i40iw_sc_parse_fpm_query_buf, - .configure_iw_fpm = i40iw_sc_configure_iw_fpm, - .parse_fpm_commit_buf = i40iw_sc_parse_fpm_commit_buf, - .create_hmc_object = i40iw_sc_create_hmc_obj, - .del_hmc_object = i40iw_sc_del_hmc_obj -}; - -/** - * i40iw_device_init - Initialize IWARP device - * @dev: IWARP device pointer - * @info: IWARP init info - */ -enum i40iw_status_code i40iw_device_init(struct i40iw_sc_dev *dev, - struct i40iw_device_init_info *info) -{ - u32 val; - u32 vchnl_ver = 0; - u16 hmc_fcn = 0; - enum i40iw_status_code ret_code = 0; - u8 db_size; - - spin_lock_init(&dev->cqp_lock); - - i40iw_device_init_uk(&dev->dev_uk); - - dev->debug_mask = info->debug_mask; - - dev->hmc_fn_id = info->hmc_fn_id; - dev->is_pf = info->is_pf; - - dev->fpm_query_buf_pa = info->fpm_query_buf_pa; - dev->fpm_query_buf = info->fpm_query_buf; - - dev->fpm_commit_buf_pa = info->fpm_commit_buf_pa; - dev->fpm_commit_buf = info->fpm_commit_buf; - - dev->hw = info->hw; - dev->hw->hw_addr = info->bar0; - - if (dev->is_pf) { - val = i40iw_rd32(dev->hw, I40E_GLPCI_DREVID); - dev->hw_rev = (u8)RS_32(val, I40E_GLPCI_DREVID_DEFAULT_REVID); - - val = i40iw_rd32(dev->hw, I40E_GLPCI_LBARCTRL); - db_size = (u8)RS_32(val, I40E_GLPCI_LBARCTRL_PE_DB_SIZE); - if ((db_size != I40IW_PE_DB_SIZE_4M) && - (db_size != I40IW_PE_DB_SIZE_8M)) { - i40iw_debug(dev, I40IW_DEBUG_DEV, - "%s: PE doorbell is not enabled in CSR val 0x%x\n", - __func__, val); - ret_code = I40IW_ERR_PE_DOORBELL_NOT_ENABLED; - return ret_code; - } - dev->db_addr = dev->hw->hw_addr + I40IW_DB_ADDR_OFFSET; - dev->vchnl_if.vchnl_recv = i40iw_vchnl_recv_pf; - } else { - dev->db_addr = dev->hw->hw_addr + I40IW_VF_DB_ADDR_OFFSET; - } - - dev->cqp_ops = &iw_cqp_ops; - dev->ccq_ops = &iw_ccq_ops; - dev->ceq_ops = &iw_ceq_ops; - dev->aeq_ops = &iw_aeq_ops; - dev->cqp_misc_ops = &iw_cqp_misc_ops; - dev->iw_pd_ops = &iw_pd_ops; - dev->iw_priv_qp_ops = &iw_priv_qp_ops; - dev->iw_priv_cq_ops = &iw_priv_cq_ops; - dev->mr_ops = &iw_mr_ops; - dev->hmc_ops = &iw_hmc_ops; - dev->vchnl_if.vchnl_send = info->vchnl_send; - if (dev->vchnl_if.vchnl_send) - dev->vchnl_up = true; - else - dev->vchnl_up = false; - if (!dev->is_pf) { - dev->vchnl_if.vchnl_recv = i40iw_vchnl_recv_vf; - ret_code = i40iw_vchnl_vf_get_ver(dev, &vchnl_ver); - if (!ret_code) { - i40iw_debug(dev, I40IW_DEBUG_DEV, - "%s: Get Channel version rc = 0x%0x, version is %u\n", - __func__, ret_code, vchnl_ver); - ret_code = i40iw_vchnl_vf_get_hmc_fcn(dev, &hmc_fcn); - if (!ret_code) { - i40iw_debug(dev, I40IW_DEBUG_DEV, - "%s Get HMC function rc = 0x%0x, hmc fcn is %u\n", - __func__, ret_code, hmc_fcn); - dev->hmc_fn_id = (u8)hmc_fcn; - } - } - } - dev->iw_vf_cqp_ops = &iw_vf_cqp_ops; - - return ret_code; -} diff --git a/drivers/infiniband/hw/i40iw/i40iw_d.h b/drivers/infiniband/hw/i40iw/i40iw_d.h deleted file mode 100644 index 86d5a33c57cc..000000000000 --- a/drivers/infiniband/hw/i40iw/i40iw_d.h +++ /dev/null @@ -1,1746 +0,0 @@ -/******************************************************************************* -* -* Copyright (c) 2015-2016 Intel Corporation. All rights reserved. -* -* This software is available to you under a choice of one of two -* licenses. You may choose to be licensed under the terms of the GNU -* General Public License (GPL) Version 2, available from the file -* COPYING in the main directory of this source tree, or the -* OpenFabrics.org BSD license below: -* -* Redistribution and use in source and binary forms, with or -* without modification, are permitted provided that the following -* conditions are met: -* -* - Redistributions of source code must retain the above -* copyright notice, this list of conditions and the following -* disclaimer. -* -* - Redistributions in binary form must reproduce the above -* copyright notice, this list of conditions and the following -* disclaimer in the documentation and/or other materials -* provided with the distribution. -* -* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS -* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN -* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -* SOFTWARE. -* -*******************************************************************************/ - -#ifndef I40IW_D_H -#define I40IW_D_H - -#define I40IW_FIRST_USER_QP_ID 2 - -#define I40IW_DB_ADDR_OFFSET (4 * 1024 * 1024 - 64 * 1024) -#define I40IW_VF_DB_ADDR_OFFSET (64 * 1024) - -#define I40IW_PE_DB_SIZE_4M 1 -#define I40IW_PE_DB_SIZE_8M 2 - -#define I40IW_DDP_VER 1 -#define I40IW_RDMAP_VER 1 - -#define I40IW_RDMA_MODE_RDMAC 0 -#define I40IW_RDMA_MODE_IETF 1 - -#define I40IW_QP_STATE_INVALID 0 -#define I40IW_QP_STATE_IDLE 1 -#define I40IW_QP_STATE_RTS 2 -#define I40IW_QP_STATE_CLOSING 3 -#define I40IW_QP_STATE_RESERVED 4 -#define I40IW_QP_STATE_TERMINATE 5 -#define I40IW_QP_STATE_ERROR 6 - -#define I40IW_STAG_STATE_INVALID 0 -#define I40IW_STAG_STATE_VALID 1 - -#define I40IW_STAG_TYPE_SHARED 0 -#define I40IW_STAG_TYPE_NONSHARED 1 - -#define I40IW_MAX_USER_PRIORITY 8 -#define I40IW_MAX_STATS_COUNT 16 -#define I40IW_FIRST_NON_PF_STAT 4 - - -#define I40IW_MTU_TO_MSS_IPV4 40 -#define I40IW_MTU_TO_MSS_IPV6 60 -#define I40IW_DEFAULT_MTU 1500 - -#define LS_64_1(val, bits) ((u64)(uintptr_t)val << bits) -#define RS_64_1(val, bits) ((u64)(uintptr_t)val >> bits) -#define LS_32_1(val, bits) (u32)(val << bits) -#define RS_32_1(val, bits) (u32)(val >> bits) -#define I40E_HI_DWORD(x) ((u32)((((x) >> 16) >> 16) & 0xFFFFFFFF)) - -#define QS_HANDLE_UNKNOWN 0xffff - -#define LS_64(val, field) (((u64)val << field ## _SHIFT) & (field ## _MASK)) - -#define RS_64(val, field) ((u64)(val & field ## _MASK) >> field ## _SHIFT) -#define LS_32(val, field) ((val << field ## _SHIFT) & (field ## _MASK)) -#define RS_32(val, field) ((val & field ## _MASK) >> field ## _SHIFT) - -#define TERM_DDP_LEN_TAGGED 14 -#define TERM_DDP_LEN_UNTAGGED 18 -#define TERM_RDMA_LEN 28 -#define RDMA_OPCODE_MASK 0x0f -#define RDMA_READ_REQ_OPCODE 1 -#define Q2_BAD_FRAME_OFFSET 72 -#define Q2_FPSN_OFFSET 64 -#define CQE_MAJOR_DRV 0x8000 - -#define I40IW_TERM_SENT 0x01 -#define I40IW_TERM_RCVD 0x02 -#define I40IW_TERM_DONE 0x04 -#define I40IW_MAC_HLEN 14 - -#define I40IW_INVALID_WQE_INDEX 0xffffffff - -#define I40IW_CQP_WAIT_POLL_REGS 1 -#define I40IW_CQP_WAIT_POLL_CQ 2 -#define I40IW_CQP_WAIT_EVENT 3 - -#define I40IW_CQP_INIT_WQE(wqe) memset(wqe, 0, 64) - -#define I40IW_GET_CURRENT_CQ_ELEMENT(_cq) \ - ( \ - &((_cq)->cq_base[I40IW_RING_GETCURRENT_HEAD((_cq)->cq_ring)]) \ - ) -#define I40IW_GET_CURRENT_EXTENDED_CQ_ELEMENT(_cq) \ - ( \ - &(((struct i40iw_extended_cqe *) \ - ((_cq)->cq_base))[I40IW_RING_GETCURRENT_HEAD((_cq)->cq_ring)]) \ - ) - -#define I40IW_GET_CURRENT_AEQ_ELEMENT(_aeq) \ - ( \ - &_aeq->aeqe_base[I40IW_RING_GETCURRENT_TAIL(_aeq->aeq_ring)] \ - ) - -#define I40IW_GET_CURRENT_CEQ_ELEMENT(_ceq) \ - ( \ - &_ceq->ceqe_base[I40IW_RING_GETCURRENT_TAIL(_ceq->ceq_ring)] \ - ) - -#define I40IW_AE_SOURCE_RSVD 0x0 -#define I40IW_AE_SOURCE_RQ 0x1 -#define I40IW_AE_SOURCE_RQ_0011 0x3 - -#define I40IW_AE_SOURCE_CQ 0x2 -#define I40IW_AE_SOURCE_CQ_0110 0x6 -#define I40IW_AE_SOURCE_CQ_1010 0xA -#define I40IW_AE_SOURCE_CQ_1110 0xE - -#define I40IW_AE_SOURCE_SQ 0x5 -#define I40IW_AE_SOURCE_SQ_0111 0x7 - -#define I40IW_AE_SOURCE_IN_RR_WR 0x9 -#define I40IW_AE_SOURCE_IN_RR_WR_1011 0xB -#define I40IW_AE_SOURCE_OUT_RR 0xD -#define I40IW_AE_SOURCE_OUT_RR_1111 0xF - -#define I40IW_TCP_STATE_NON_EXISTENT 0 -#define I40IW_TCP_STATE_CLOSED 1 -#define I40IW_TCP_STATE_LISTEN 2 -#define I40IW_STATE_SYN_SEND 3 -#define I40IW_TCP_STATE_SYN_RECEIVED 4 -#define I40IW_TCP_STATE_ESTABLISHED 5 -#define I40IW_TCP_STATE_CLOSE_WAIT 6 -#define I40IW_TCP_STATE_FIN_WAIT_1 7 -#define I40IW_TCP_STATE_CLOSING 8 -#define I40IW_TCP_STATE_LAST_ACK 9 -#define I40IW_TCP_STATE_FIN_WAIT_2 10 -#define I40IW_TCP_STATE_TIME_WAIT 11 -#define I40IW_TCP_STATE_RESERVED_1 12 -#define I40IW_TCP_STATE_RESERVED_2 13 -#define I40IW_TCP_STATE_RESERVED_3 14 -#define I40IW_TCP_STATE_RESERVED_4 15 - -/* ILQ CQP hash table fields */ -#define I40IW_CQPSQ_QHASH_VLANID_SHIFT 32 -#define I40IW_CQPSQ_QHASH_VLANID_MASK \ - ((u64)0xfff << I40IW_CQPSQ_QHASH_VLANID_SHIFT) - -#define I40IW_CQPSQ_QHASH_QPN_SHIFT 32 -#define I40IW_CQPSQ_QHASH_QPN_MASK \ - ((u64)0x3ffff << I40IW_CQPSQ_QHASH_QPN_SHIFT) - -#define I40IW_CQPSQ_QHASH_QS_HANDLE_SHIFT 0 -#define I40IW_CQPSQ_QHASH_QS_HANDLE_MASK ((u64)0x3ff << I40IW_CQPSQ_QHASH_QS_HANDLE_SHIFT) - -#define I40IW_CQPSQ_QHASH_SRC_PORT_SHIFT 16 -#define I40IW_CQPSQ_QHASH_SRC_PORT_MASK \ - ((u64)0xffff << I40IW_CQPSQ_QHASH_SRC_PORT_SHIFT) - -#define I40IW_CQPSQ_QHASH_DEST_PORT_SHIFT 0 -#define I40IW_CQPSQ_QHASH_DEST_PORT_MASK \ - ((u64)0xffff << I40IW_CQPSQ_QHASH_DEST_PORT_SHIFT) - -#define I40IW_CQPSQ_QHASH_ADDR0_SHIFT 32 -#define I40IW_CQPSQ_QHASH_ADDR0_MASK \ - ((u64)0xffffffff << I40IW_CQPSQ_QHASH_ADDR0_SHIFT) - -#define I40IW_CQPSQ_QHASH_ADDR1_SHIFT 0 -#define I40IW_CQPSQ_QHASH_ADDR1_MASK \ - ((u64)0xffffffff << I40IW_CQPSQ_QHASH_ADDR1_SHIFT) - -#define I40IW_CQPSQ_QHASH_ADDR2_SHIFT 32 -#define I40IW_CQPSQ_QHASH_ADDR2_MASK \ - ((u64)0xffffffff << I40IW_CQPSQ_QHASH_ADDR2_SHIFT) - -#define I40IW_CQPSQ_QHASH_ADDR3_SHIFT 0 -#define I40IW_CQPSQ_QHASH_ADDR3_MASK \ - ((u64)0xffffffff << I40IW_CQPSQ_QHASH_ADDR3_SHIFT) - -#define I40IW_CQPSQ_QHASH_WQEVALID_SHIFT 63 -#define I40IW_CQPSQ_QHASH_WQEVALID_MASK \ - ((u64)0x1 << I40IW_CQPSQ_QHASH_WQEVALID_SHIFT) -#define I40IW_CQPSQ_QHASH_OPCODE_SHIFT 32 -#define I40IW_CQPSQ_QHASH_OPCODE_MASK \ - ((u64)0x3f << I40IW_CQPSQ_QHASH_OPCODE_SHIFT) - -#define I40IW_CQPSQ_QHASH_MANAGE_SHIFT 61 -#define I40IW_CQPSQ_QHASH_MANAGE_MASK \ - ((u64)0x3 << I40IW_CQPSQ_QHASH_MANAGE_SHIFT) - -#define I40IW_CQPSQ_QHASH_IPV4VALID_SHIFT 60 -#define I40IW_CQPSQ_QHASH_IPV4VALID_MASK \ - ((u64)0x1 << I40IW_CQPSQ_QHASH_IPV4VALID_SHIFT) - -#define I40IW_CQPSQ_QHASH_VLANVALID_SHIFT 59 -#define I40IW_CQPSQ_QHASH_VLANVALID_MASK \ - ((u64)0x1 << I40IW_CQPSQ_QHASH_VLANVALID_SHIFT) - -#define I40IW_CQPSQ_QHASH_ENTRYTYPE_SHIFT 42 -#define I40IW_CQPSQ_QHASH_ENTRYTYPE_MASK \ - ((u64)0x7 << I40IW_CQPSQ_QHASH_ENTRYTYPE_SHIFT) -/* CQP Host Context */ -#define I40IW_CQPHC_EN_DC_TCP_SHIFT 0 -#define I40IW_CQPHC_EN_DC_TCP_MASK (1UL << I40IW_CQPHC_EN_DC_TCP_SHIFT) - -#define I40IW_CQPHC_SQSIZE_SHIFT 8 -#define I40IW_CQPHC_SQSIZE_MASK (0xfUL << I40IW_CQPHC_SQSIZE_SHIFT) - -#define I40IW_CQPHC_DISABLE_PFPDUS_SHIFT 1 -#define I40IW_CQPHC_DISABLE_PFPDUS_MASK (0x1UL << I40IW_CQPHC_DISABLE_PFPDUS_SHIFT) - -#define I40IW_CQPHC_ENABLED_VFS_SHIFT 32 -#define I40IW_CQPHC_ENABLED_VFS_MASK (0x3fULL << I40IW_CQPHC_ENABLED_VFS_SHIFT) - -#define I40IW_CQPHC_HMC_PROFILE_SHIFT 0 -#define I40IW_CQPHC_HMC_PROFILE_MASK (0x7ULL << I40IW_CQPHC_HMC_PROFILE_SHIFT) - -#define I40IW_CQPHC_SVER_SHIFT 24 -#define I40IW_CQPHC_SVER_MASK (0xffUL << I40IW_CQPHC_SVER_SHIFT) - -#define I40IW_CQPHC_SQBASE_SHIFT 9 -#define I40IW_CQPHC_SQBASE_MASK \ - (0xfffffffffffffeULL << I40IW_CQPHC_SQBASE_SHIFT) - -#define I40IW_CQPHC_QPCTX_SHIFT 0 -#define I40IW_CQPHC_QPCTX_MASK \ - (0xffffffffffffffffULL << I40IW_CQPHC_QPCTX_SHIFT) -#define I40IW_CQPHC_SVER 1 - -#define I40IW_CQP_SW_SQSIZE_4 4 -#define I40IW_CQP_SW_SQSIZE_2048 2048 - -/* iWARP QP Doorbell shadow area */ -#define I40IW_QP_DBSA_HW_SQ_TAIL_SHIFT 0 -#define I40IW_QP_DBSA_HW_SQ_TAIL_MASK \ - (0x3fffUL << I40IW_QP_DBSA_HW_SQ_TAIL_SHIFT) - -/* Completion Queue Doorbell shadow area */ -#define I40IW_CQ_DBSA_CQEIDX_SHIFT 0 -#define I40IW_CQ_DBSA_CQEIDX_MASK (0xfffffUL << I40IW_CQ_DBSA_CQEIDX_SHIFT) - -#define I40IW_CQ_DBSA_SW_CQ_SELECT_SHIFT 0 -#define I40IW_CQ_DBSA_SW_CQ_SELECT_MASK \ - (0x3fffUL << I40IW_CQ_DBSA_SW_CQ_SELECT_SHIFT) - -#define I40IW_CQ_DBSA_ARM_NEXT_SHIFT 14 -#define I40IW_CQ_DBSA_ARM_NEXT_MASK (1UL << I40IW_CQ_DBSA_ARM_NEXT_SHIFT) - -#define I40IW_CQ_DBSA_ARM_NEXT_SE_SHIFT 15 -#define I40IW_CQ_DBSA_ARM_NEXT_SE_MASK (1UL << I40IW_CQ_DBSA_ARM_NEXT_SE_SHIFT) - -#define I40IW_CQ_DBSA_ARM_SEQ_NUM_SHIFT 16 -#define I40IW_CQ_DBSA_ARM_SEQ_NUM_MASK \ - (0x3UL << I40IW_CQ_DBSA_ARM_SEQ_NUM_SHIFT) - -/* CQP and iWARP Completion Queue */ -#define I40IW_CQ_QPCTX_SHIFT I40IW_CQPHC_QPCTX_SHIFT -#define I40IW_CQ_QPCTX_MASK I40IW_CQPHC_QPCTX_MASK - -#define I40IW_CCQ_OPRETVAL_SHIFT 0 -#define I40IW_CCQ_OPRETVAL_MASK (0xffffffffUL << I40IW_CCQ_OPRETVAL_SHIFT) - -#define I40IW_CQ_MINERR_SHIFT 0 -#define I40IW_CQ_MINERR_MASK (0xffffUL << I40IW_CQ_MINERR_SHIFT) - -#define I40IW_CQ_MAJERR_SHIFT 16 -#define I40IW_CQ_MAJERR_MASK (0xffffUL << I40IW_CQ_MAJERR_SHIFT) - -#define I40IW_CQ_WQEIDX_SHIFT 32 -#define I40IW_CQ_WQEIDX_MASK (0x3fffULL << I40IW_CQ_WQEIDX_SHIFT) - -#define I40IW_CQ_ERROR_SHIFT 55 -#define I40IW_CQ_ERROR_MASK (1ULL << I40IW_CQ_ERROR_SHIFT) - -#define I40IW_CQ_SQ_SHIFT 62 -#define I40IW_CQ_SQ_MASK (1ULL << I40IW_CQ_SQ_SHIFT) - -#define I40IW_CQ_VALID_SHIFT 63 -#define I40IW_CQ_VALID_MASK (1ULL << I40IW_CQ_VALID_SHIFT) - -#define I40IWCQ_PAYLDLEN_SHIFT 0 -#define I40IWCQ_PAYLDLEN_MASK (0xffffffffUL << I40IWCQ_PAYLDLEN_SHIFT) - -#define I40IWCQ_TCPSEQNUM_SHIFT 32 -#define I40IWCQ_TCPSEQNUM_MASK (0xffffffffULL << I40IWCQ_TCPSEQNUM_SHIFT) - -#define I40IWCQ_INVSTAG_SHIFT 0 -#define I40IWCQ_INVSTAG_MASK (0xffffffffUL << I40IWCQ_INVSTAG_SHIFT) - -#define I40IWCQ_QPID_SHIFT 32 -#define I40IWCQ_QPID_MASK (0x3ffffULL << I40IWCQ_QPID_SHIFT) - -#define I40IWCQ_PSHDROP_SHIFT 51 -#define I40IWCQ_PSHDROP_MASK (1ULL << I40IWCQ_PSHDROP_SHIFT) - -#define I40IWCQ_SRQ_SHIFT 52 -#define I40IWCQ_SRQ_MASK (1ULL << I40IWCQ_SRQ_SHIFT) - -#define I40IWCQ_STAG_SHIFT 53 -#define I40IWCQ_STAG_MASK (1ULL << I40IWCQ_STAG_SHIFT) - -#define I40IWCQ_SOEVENT_SHIFT 54 -#define I40IWCQ_SOEVENT_MASK (1ULL << I40IWCQ_SOEVENT_SHIFT) - -#define I40IWCQ_OP_SHIFT 56 -#define I40IWCQ_OP_MASK (0x3fULL << I40IWCQ_OP_SHIFT) - -/* CEQE format */ -#define I40IW_CEQE_CQCTX_SHIFT 0 -#define I40IW_CEQE_CQCTX_MASK \ - (0x7fffffffffffffffULL << I40IW_CEQE_CQCTX_SHIFT) - -#define I40IW_CEQE_VALID_SHIFT 63 -#define I40IW_CEQE_VALID_MASK (1ULL << I40IW_CEQE_VALID_SHIFT) - -/* AEQE format */ -#define I40IW_AEQE_COMPCTX_SHIFT I40IW_CQPHC_QPCTX_SHIFT -#define I40IW_AEQE_COMPCTX_MASK I40IW_CQPHC_QPCTX_MASK - -#define I40IW_AEQE_QPCQID_SHIFT 0 -#define I40IW_AEQE_QPCQID_MASK (0x3ffffUL << I40IW_AEQE_QPCQID_SHIFT) - -#define I40IW_AEQE_WQDESCIDX_SHIFT 18 -#define I40IW_AEQE_WQDESCIDX_MASK (0x3fffULL << I40IW_AEQE_WQDESCIDX_SHIFT) - -#define I40IW_AEQE_OVERFLOW_SHIFT 33 -#define I40IW_AEQE_OVERFLOW_MASK (1ULL << I40IW_AEQE_OVERFLOW_SHIFT) - -#define I40IW_AEQE_AECODE_SHIFT 34 -#define I40IW_AEQE_AECODE_MASK (0xffffULL << I40IW_AEQE_AECODE_SHIFT) - -#define I40IW_AEQE_AESRC_SHIFT 50 -#define I40IW_AEQE_AESRC_MASK (0xfULL << I40IW_AEQE_AESRC_SHIFT) - -#define I40IW_AEQE_IWSTATE_SHIFT 54 -#define I40IW_AEQE_IWSTATE_MASK (0x7ULL << I40IW_AEQE_IWSTATE_SHIFT) - -#define I40IW_AEQE_TCPSTATE_SHIFT 57 -#define I40IW_AEQE_TCPSTATE_MASK (0xfULL << I40IW_AEQE_TCPSTATE_SHIFT) - -#define I40IW_AEQE_Q2DATA_SHIFT 61 -#define I40IW_AEQE_Q2DATA_MASK (0x3ULL << I40IW_AEQE_Q2DATA_SHIFT) - -#define I40IW_AEQE_VALID_SHIFT 63 -#define I40IW_AEQE_VALID_MASK (1ULL << I40IW_AEQE_VALID_SHIFT) - -/* CQP SQ WQES */ -#define I40IW_QP_TYPE_IWARP 1 -#define I40IW_QP_TYPE_UDA 2 -#define I40IW_QP_TYPE_CQP 4 - -#define I40IW_CQ_TYPE_IWARP 1 -#define I40IW_CQ_TYPE_ILQ 2 -#define I40IW_CQ_TYPE_IEQ 3 -#define I40IW_CQ_TYPE_CQP 4 - -#define I40IWQP_TERM_SEND_TERM_AND_FIN 0 -#define I40IWQP_TERM_SEND_TERM_ONLY 1 -#define I40IWQP_TERM_SEND_FIN_ONLY 2 -#define I40IWQP_TERM_DONOT_SEND_TERM_OR_FIN 3 - -#define I40IW_CQP_OP_CREATE_QP 0 -#define I40IW_CQP_OP_MODIFY_QP 0x1 -#define I40IW_CQP_OP_DESTROY_QP 0x02 -#define I40IW_CQP_OP_CREATE_CQ 0x03 -#define I40IW_CQP_OP_MODIFY_CQ 0x04 -#define I40IW_CQP_OP_DESTROY_CQ 0x05 -#define I40IW_CQP_OP_CREATE_SRQ 0x06 -#define I40IW_CQP_OP_MODIFY_SRQ 0x07 -#define I40IW_CQP_OP_DESTROY_SRQ 0x08 -#define I40IW_CQP_OP_ALLOC_STAG 0x09 -#define I40IW_CQP_OP_REG_MR 0x0a -#define I40IW_CQP_OP_QUERY_STAG 0x0b -#define I40IW_CQP_OP_REG_SMR 0x0c -#define I40IW_CQP_OP_DEALLOC_STAG 0x0d -#define I40IW_CQP_OP_MANAGE_LOC_MAC_IP_TABLE 0x0e -#define I40IW_CQP_OP_MANAGE_ARP 0x0f -#define I40IW_CQP_OP_MANAGE_VF_PBLE_BP 0x10 -#define I40IW_CQP_OP_QUERY_RDMA_FEATURES 0x12 -#define I40IW_CQP_OP_UPLOAD_CONTEXT 0x13 -#define I40IW_CQP_OP_ALLOCATE_LOC_MAC_IP_TABLE_ENTRY 0x14 -#define I40IW_CQP_OP_MANAGE_HMC_PM_FUNC_TABLE 0x15 -#define I40IW_CQP_OP_CREATE_CEQ 0x16 -#define I40IW_CQP_OP_DESTROY_CEQ 0x18 -#define I40IW_CQP_OP_CREATE_AEQ 0x19 -#define I40IW_CQP_OP_DESTROY_AEQ 0x1b -#define I40IW_CQP_OP_CREATE_ADDR_VECT 0x1c -#define I40IW_CQP_OP_MODIFY_ADDR_VECT 0x1d -#define I40IW_CQP_OP_DESTROY_ADDR_VECT 0x1e -#define I40IW_CQP_OP_UPDATE_PE_SDS 0x1f -#define I40IW_CQP_OP_QUERY_FPM_VALUES 0x20 -#define I40IW_CQP_OP_COMMIT_FPM_VALUES 0x21 -#define I40IW_CQP_OP_FLUSH_WQES 0x22 -/* I40IW_CQP_OP_GEN_AE is the same value as I40IW_CQP_OP_FLUSH_WQES */ -#define I40IW_CQP_OP_GEN_AE 0x22 -#define I40IW_CQP_OP_MANAGE_APBVT 0x23 -#define I40IW_CQP_OP_NOP 0x24 -#define I40IW_CQP_OP_MANAGE_QUAD_HASH_TABLE_ENTRY 0x25 -#define I40IW_CQP_OP_CREATE_UDA_MCAST_GROUP 0x26 -#define I40IW_CQP_OP_MODIFY_UDA_MCAST_GROUP 0x27 -#define I40IW_CQP_OP_DESTROY_UDA_MCAST_GROUP 0x28 -#define I40IW_CQP_OP_SUSPEND_QP 0x29 -#define I40IW_CQP_OP_RESUME_QP 0x2a -#define I40IW_CQP_OP_SHMC_PAGES_ALLOCATED 0x2b -#define I40IW_CQP_OP_SET_HMC_RESOURCE_PROFILE 0x2d - -#define I40IW_FEATURE_BUF_SIZE (8 * I40IW_MAX_FEATURES) - -#define I40IW_FW_VER_MINOR_SHIFT 0 -#define I40IW_FW_VER_MINOR_MASK \ - (0xffffULL << I40IW_FW_VER_MINOR_SHIFT) - -#define I40IW_FW_VER_MAJOR_SHIFT 16 -#define I40IW_FW_VER_MAJOR_MASK \ - (0xffffULL << I40IW_FW_VER_MAJOR_SHIFT) - -#define I40IW_FEATURE_INFO_SHIFT 0 -#define I40IW_FEATURE_INFO_MASK \ - (0xffffULL << I40IW_FEATURE_INFO_SHIFT) - -#define I40IW_FEATURE_CNT_SHIFT 32 -#define I40IW_FEATURE_CNT_MASK \ - (0xffffULL << I40IW_FEATURE_CNT_SHIFT) - -#define I40IW_UDA_QPSQ_NEXT_HEADER_SHIFT 16 -#define I40IW_UDA_QPSQ_NEXT_HEADER_MASK ((u64)0xff << I40IW_UDA_QPSQ_NEXT_HEADER_SHIFT) - -#define I40IW_UDA_QPSQ_OPCODE_SHIFT 32 -#define I40IW_UDA_QPSQ_OPCODE_MASK ((u64)0x3f << I40IW_UDA_QPSQ_OPCODE_SHIFT) - -#define I40IW_UDA_QPSQ_MACLEN_SHIFT 56 -#define I40IW_UDA_QPSQ_MACLEN_MASK \ - ((u64)0x7f << I40IW_UDA_QPSQ_MACLEN_SHIFT) - -#define I40IW_UDA_QPSQ_IPLEN_SHIFT 48 -#define I40IW_UDA_QPSQ_IPLEN_MASK \ - ((u64)0x7f << I40IW_UDA_QPSQ_IPLEN_SHIFT) - -#define I40IW_UDA_QPSQ_L4T_SHIFT 30 -#define I40IW_UDA_QPSQ_L4T_MASK \ - ((u64)0x3 << I40IW_UDA_QPSQ_L4T_SHIFT) - -#define I40IW_UDA_QPSQ_IIPT_SHIFT 28 -#define I40IW_UDA_QPSQ_IIPT_MASK \ - ((u64)0x3 << I40IW_UDA_QPSQ_IIPT_SHIFT) - -#define I40IW_UDA_QPSQ_L4LEN_SHIFT 24 -#define I40IW_UDA_QPSQ_L4LEN_MASK ((u64)0xf << I40IW_UDA_QPSQ_L4LEN_SHIFT) - -#define I40IW_UDA_QPSQ_AVIDX_SHIFT 0 -#define I40IW_UDA_QPSQ_AVIDX_MASK ((u64)0xffff << I40IW_UDA_QPSQ_AVIDX_SHIFT) - -#define I40IW_UDA_QPSQ_VALID_SHIFT 63 -#define I40IW_UDA_QPSQ_VALID_MASK \ - ((u64)0x1 << I40IW_UDA_QPSQ_VALID_SHIFT) - -#define I40IW_UDA_QPSQ_SIGCOMPL_SHIFT 62 -#define I40IW_UDA_QPSQ_SIGCOMPL_MASK ((u64)0x1 << I40IW_UDA_QPSQ_SIGCOMPL_SHIFT) - -#define I40IW_UDA_PAYLOADLEN_SHIFT 0 -#define I40IW_UDA_PAYLOADLEN_MASK ((u64)0x3fff << I40IW_UDA_PAYLOADLEN_SHIFT) - -#define I40IW_UDA_HDRLEN_SHIFT 16 -#define I40IW_UDA_HDRLEN_MASK ((u64)0x1ff << I40IW_UDA_HDRLEN_SHIFT) - -#define I40IW_VLAN_TAG_VALID_SHIFT 50 -#define I40IW_VLAN_TAG_VALID_MASK ((u64)0x1 << I40IW_VLAN_TAG_VALID_SHIFT) - -#define I40IW_UDA_L3PROTO_SHIFT 0 -#define I40IW_UDA_L3PROTO_MASK ((u64)0x3 << I40IW_UDA_L3PROTO_SHIFT) - -#define I40IW_UDA_L4PROTO_SHIFT 16 -#define I40IW_UDA_L4PROTO_MASK ((u64)0x3 << I40IW_UDA_L4PROTO_SHIFT) - -#define I40IW_UDA_QPSQ_DOLOOPBACK_SHIFT 44 -#define I40IW_UDA_QPSQ_DOLOOPBACK_MASK \ - ((u64)0x1 << I40IW_UDA_QPSQ_DOLOOPBACK_SHIFT) - -/* CQP SQ WQE common fields */ -#define I40IW_CQPSQ_OPCODE_SHIFT 32 -#define I40IW_CQPSQ_OPCODE_MASK (0x3fULL << I40IW_CQPSQ_OPCODE_SHIFT) - -#define I40IW_CQPSQ_WQEVALID_SHIFT 63 -#define I40IW_CQPSQ_WQEVALID_MASK (1ULL << I40IW_CQPSQ_WQEVALID_SHIFT) - -#define I40IW_CQPSQ_TPHVAL_SHIFT 0 -#define I40IW_CQPSQ_TPHVAL_MASK (0xffUL << I40IW_CQPSQ_TPHVAL_SHIFT) - -#define I40IW_CQPSQ_TPHEN_SHIFT 60 -#define I40IW_CQPSQ_TPHEN_MASK (1ULL << I40IW_CQPSQ_TPHEN_SHIFT) - -#define I40IW_CQPSQ_PBUFADDR_SHIFT I40IW_CQPHC_QPCTX_SHIFT -#define I40IW_CQPSQ_PBUFADDR_MASK I40IW_CQPHC_QPCTX_MASK - -/* Create/Modify/Destroy QP */ - -#define I40IW_CQPSQ_QP_NEWMSS_SHIFT 32 -#define I40IW_CQPSQ_QP_NEWMSS_MASK (0x3fffULL << I40IW_CQPSQ_QP_NEWMSS_SHIFT) - -#define I40IW_CQPSQ_QP_TERMLEN_SHIFT 48 -#define I40IW_CQPSQ_QP_TERMLEN_MASK (0xfULL << I40IW_CQPSQ_QP_TERMLEN_SHIFT) - -#define I40IW_CQPSQ_QP_QPCTX_SHIFT I40IW_CQPHC_QPCTX_SHIFT -#define I40IW_CQPSQ_QP_QPCTX_MASK I40IW_CQPHC_QPCTX_MASK - -#define I40IW_CQPSQ_QP_QPID_SHIFT 0 -#define I40IW_CQPSQ_QP_QPID_MASK (0x3FFFFUL) -/* I40IWCQ_QPID_MASK */ - -#define I40IW_CQPSQ_QP_OP_SHIFT 32 -#define I40IW_CQPSQ_QP_OP_MASK I40IWCQ_OP_MASK - -#define I40IW_CQPSQ_QP_ORDVALID_SHIFT 42 -#define I40IW_CQPSQ_QP_ORDVALID_MASK (1ULL << I40IW_CQPSQ_QP_ORDVALID_SHIFT) - -#define I40IW_CQPSQ_QP_TOECTXVALID_SHIFT 43 -#define I40IW_CQPSQ_QP_TOECTXVALID_MASK \ - (1ULL << I40IW_CQPSQ_QP_TOECTXVALID_SHIFT) - -#define I40IW_CQPSQ_QP_CACHEDVARVALID_SHIFT 44 -#define I40IW_CQPSQ_QP_CACHEDVARVALID_MASK \ - (1ULL << I40IW_CQPSQ_QP_CACHEDVARVALID_SHIFT) - -#define I40IW_CQPSQ_QP_VQ_SHIFT 45 -#define I40IW_CQPSQ_QP_VQ_MASK (1ULL << I40IW_CQPSQ_QP_VQ_SHIFT) - -#define I40IW_CQPSQ_QP_FORCELOOPBACK_SHIFT 46 -#define I40IW_CQPSQ_QP_FORCELOOPBACK_MASK \ - (1ULL << I40IW_CQPSQ_QP_FORCELOOPBACK_SHIFT) - -#define I40IW_CQPSQ_QP_CQNUMVALID_SHIFT 47 -#define I40IW_CQPSQ_QP_CQNUMVALID_MASK \ - (1ULL << I40IW_CQPSQ_QP_CQNUMVALID_SHIFT) - -#define I40IW_CQPSQ_QP_QPTYPE_SHIFT 48 -#define I40IW_CQPSQ_QP_QPTYPE_MASK (0x3ULL << I40IW_CQPSQ_QP_QPTYPE_SHIFT) - -#define I40IW_CQPSQ_QP_MSSCHANGE_SHIFT 52 -#define I40IW_CQPSQ_QP_MSSCHANGE_MASK (1ULL << I40IW_CQPSQ_QP_MSSCHANGE_SHIFT) - -#define I40IW_CQPSQ_QP_IGNOREMWBOUND_SHIFT 54 -#define I40IW_CQPSQ_QP_IGNOREMWBOUND_MASK \ - (1ULL << I40IW_CQPSQ_QP_IGNOREMWBOUND_SHIFT) - -#define I40IW_CQPSQ_QP_REMOVEHASHENTRY_SHIFT 55 -#define I40IW_CQPSQ_QP_REMOVEHASHENTRY_MASK \ - (1ULL << I40IW_CQPSQ_QP_REMOVEHASHENTRY_SHIFT) - -#define I40IW_CQPSQ_QP_TERMACT_SHIFT 56 -#define I40IW_CQPSQ_QP_TERMACT_MASK (0x3ULL << I40IW_CQPSQ_QP_TERMACT_SHIFT) - -#define I40IW_CQPSQ_QP_RESETCON_SHIFT 58 -#define I40IW_CQPSQ_QP_RESETCON_MASK (1ULL << I40IW_CQPSQ_QP_RESETCON_SHIFT) - -#define I40IW_CQPSQ_QP_ARPTABIDXVALID_SHIFT 59 -#define I40IW_CQPSQ_QP_ARPTABIDXVALID_MASK \ - (1ULL << I40IW_CQPSQ_QP_ARPTABIDXVALID_SHIFT) - -#define I40IW_CQPSQ_QP_NEXTIWSTATE_SHIFT 60 -#define I40IW_CQPSQ_QP_NEXTIWSTATE_MASK \ - (0x7ULL << I40IW_CQPSQ_QP_NEXTIWSTATE_SHIFT) - -#define I40IW_CQPSQ_QP_DBSHADOWADDR_SHIFT I40IW_CQPHC_QPCTX_SHIFT -#define I40IW_CQPSQ_QP_DBSHADOWADDR_MASK I40IW_CQPHC_QPCTX_MASK - -/* Create/Modify/Destroy CQ */ -#define I40IW_CQPSQ_CQ_CQSIZE_SHIFT 0 -#define I40IW_CQPSQ_CQ_CQSIZE_MASK (0x3ffffUL << I40IW_CQPSQ_CQ_CQSIZE_SHIFT) - -#define I40IW_CQPSQ_CQ_CQCTX_SHIFT 0 -#define I40IW_CQPSQ_CQ_CQCTX_MASK \ - (0x7fffffffffffffffULL << I40IW_CQPSQ_CQ_CQCTX_SHIFT) - -#define I40IW_CQPSQ_CQ_CQCTX_SHIFT 0 -#define I40IW_CQPSQ_CQ_CQCTX_MASK \ - (0x7fffffffffffffffULL << I40IW_CQPSQ_CQ_CQCTX_SHIFT) - -#define I40IW_CQPSQ_CQ_SHADOW_READ_THRESHOLD_SHIFT 0 -#define I40IW_CQPSQ_CQ_SHADOW_READ_THRESHOLD_MASK \ - (0x3ffff << I40IW_CQPSQ_CQ_SHADOW_READ_THRESHOLD_SHIFT) - -#define I40IW_CQPSQ_CQ_CEQID_SHIFT 24 -#define I40IW_CQPSQ_CQ_CEQID_MASK (0x7fUL << I40IW_CQPSQ_CQ_CEQID_SHIFT) - -#define I40IW_CQPSQ_CQ_OP_SHIFT 32 -#define I40IW_CQPSQ_CQ_OP_MASK (0x3fULL << I40IW_CQPSQ_CQ_OP_SHIFT) - -#define I40IW_CQPSQ_CQ_CQRESIZE_SHIFT 43 -#define I40IW_CQPSQ_CQ_CQRESIZE_MASK (1ULL << I40IW_CQPSQ_CQ_CQRESIZE_SHIFT) - -#define I40IW_CQPSQ_CQ_LPBLSIZE_SHIFT 44 -#define I40IW_CQPSQ_CQ_LPBLSIZE_MASK (3ULL << I40IW_CQPSQ_CQ_LPBLSIZE_SHIFT) - -#define I40IW_CQPSQ_CQ_CHKOVERFLOW_SHIFT 46 -#define I40IW_CQPSQ_CQ_CHKOVERFLOW_MASK \ - (1ULL << I40IW_CQPSQ_CQ_CHKOVERFLOW_SHIFT) - -#define I40IW_CQPSQ_CQ_VIRTMAP_SHIFT 47 -#define I40IW_CQPSQ_CQ_VIRTMAP_MASK (1ULL << I40IW_CQPSQ_CQ_VIRTMAP_SHIFT) - -#define I40IW_CQPSQ_CQ_ENCEQEMASK_SHIFT 48 -#define I40IW_CQPSQ_CQ_ENCEQEMASK_MASK \ - (1ULL << I40IW_CQPSQ_CQ_ENCEQEMASK_SHIFT) - -#define I40IW_CQPSQ_CQ_CEQIDVALID_SHIFT 49 -#define I40IW_CQPSQ_CQ_CEQIDVALID_MASK \ - (1ULL << I40IW_CQPSQ_CQ_CEQIDVALID_SHIFT) - -#define I40IW_CQPSQ_CQ_AVOIDMEMCNFLCT_SHIFT 61 -#define I40IW_CQPSQ_CQ_AVOIDMEMCNFLCT_MASK \ - (1ULL << I40IW_CQPSQ_CQ_AVOIDMEMCNFLCT_SHIFT) - -/* Create/Modify/Destroy Shared Receive Queue */ - -#define I40IW_CQPSQ_SRQ_RQSIZE_SHIFT 0 -#define I40IW_CQPSQ_SRQ_RQSIZE_MASK (0xfUL << I40IW_CQPSQ_SRQ_RQSIZE_SHIFT) - -#define I40IW_CQPSQ_SRQ_RQWQESIZE_SHIFT 4 -#define I40IW_CQPSQ_SRQ_RQWQESIZE_MASK \ - (0x7UL << I40IW_CQPSQ_SRQ_RQWQESIZE_SHIFT) - -#define I40IW_CQPSQ_SRQ_SRQLIMIT_SHIFT 32 -#define I40IW_CQPSQ_SRQ_SRQLIMIT_MASK \ - (0xfffULL << I40IW_CQPSQ_SRQ_SRQLIMIT_SHIFT) - -#define I40IW_CQPSQ_SRQ_SRQCTX_SHIFT I40IW_CQPHC_QPCTX_SHIFT -#define I40IW_CQPSQ_SRQ_SRQCTX_MASK I40IW_CQPHC_QPCTX_MASK - -#define I40IW_CQPSQ_SRQ_PDID_SHIFT 16 -#define I40IW_CQPSQ_SRQ_PDID_MASK \ - (0x7fffULL << I40IW_CQPSQ_SRQ_PDID_SHIFT) - -#define I40IW_CQPSQ_SRQ_SRQID_SHIFT 0 -#define I40IW_CQPSQ_SRQ_SRQID_MASK (0x7fffUL << I40IW_CQPSQ_SRQ_SRQID_SHIFT) - -#define I40IW_CQPSQ_SRQ_LPBLSIZE_SHIFT I40IW_CQPSQ_CQ_LPBLSIZE_SHIFT -#define I40IW_CQPSQ_SRQ_LPBLSIZE_MASK I40IW_CQPSQ_CQ_LPBLSIZE_MASK - -#define I40IW_CQPSQ_SRQ_VIRTMAP_SHIFT I40IW_CQPSQ_CQ_VIRTMAP_SHIFT -#define I40IW_CQPSQ_SRQ_VIRTMAP_MASK I40IW_CQPSQ_CQ_VIRTMAP_MASK - -#define I40IW_CQPSQ_SRQ_TPHEN_SHIFT I40IW_CQPSQ_TPHEN_SHIFT -#define I40IW_CQPSQ_SRQ_TPHEN_MASK I40IW_CQPSQ_TPHEN_MASK - -#define I40IW_CQPSQ_SRQ_ARMLIMITEVENT_SHIFT 61 -#define I40IW_CQPSQ_SRQ_ARMLIMITEVENT_MASK \ - (1ULL << I40IW_CQPSQ_SRQ_ARMLIMITEVENT_SHIFT) - -#define I40IW_CQPSQ_SRQ_DBSHADOWAREA_SHIFT 6 -#define I40IW_CQPSQ_SRQ_DBSHADOWAREA_MASK \ - (0x3ffffffffffffffULL << I40IW_CQPSQ_SRQ_DBSHADOWAREA_SHIFT) - -#define I40IW_CQPSQ_SRQ_FIRSTPMPBLIDX_SHIFT 0 -#define I40IW_CQPSQ_SRQ_FIRSTPMPBLIDX_MASK \ - (0xfffffffUL << I40IW_CQPSQ_SRQ_FIRSTPMPBLIDX_SHIFT) - -/* Allocate/Register/Register Shared/Deallocate Stag */ -#define I40IW_CQPSQ_STAG_VA_FBO_SHIFT I40IW_CQPHC_QPCTX_SHIFT -#define I40IW_CQPSQ_STAG_VA_FBO_MASK I40IW_CQPHC_QPCTX_MASK - -#define I40IW_CQPSQ_STAG_STAGLEN_SHIFT 0 -#define I40IW_CQPSQ_STAG_STAGLEN_MASK \ - (0x3fffffffffffULL << I40IW_CQPSQ_STAG_STAGLEN_SHIFT) - -#define I40IW_CQPSQ_STAG_PDID_SHIFT 48 -#define I40IW_CQPSQ_STAG_PDID_MASK (0x7fffULL << I40IW_CQPSQ_STAG_PDID_SHIFT) - -#define I40IW_CQPSQ_STAG_KEY_SHIFT 0 -#define I40IW_CQPSQ_STAG_KEY_MASK (0xffUL << I40IW_CQPSQ_STAG_KEY_SHIFT) - -#define I40IW_CQPSQ_STAG_IDX_SHIFT 8 -#define I40IW_CQPSQ_STAG_IDX_MASK (0xffffffUL << I40IW_CQPSQ_STAG_IDX_SHIFT) - -#define I40IW_CQPSQ_STAG_PARENTSTAGIDX_SHIFT 32 -#define I40IW_CQPSQ_STAG_PARENTSTAGIDX_MASK \ - (0xffffffULL << I40IW_CQPSQ_STAG_PARENTSTAGIDX_SHIFT) - -#define I40IW_CQPSQ_STAG_MR_SHIFT 43 -#define I40IW_CQPSQ_STAG_MR_MASK (1ULL << I40IW_CQPSQ_STAG_MR_SHIFT) - -#define I40IW_CQPSQ_STAG_LPBLSIZE_SHIFT I40IW_CQPSQ_CQ_LPBLSIZE_SHIFT -#define I40IW_CQPSQ_STAG_LPBLSIZE_MASK I40IW_CQPSQ_CQ_LPBLSIZE_MASK - -#define I40IW_CQPSQ_STAG_HPAGESIZE_SHIFT 46 -#define I40IW_CQPSQ_STAG_HPAGESIZE_MASK \ - (1ULL << I40IW_CQPSQ_STAG_HPAGESIZE_SHIFT) - -#define I40IW_CQPSQ_STAG_ARIGHTS_SHIFT 48 -#define I40IW_CQPSQ_STAG_ARIGHTS_MASK \ - (0x1fULL << I40IW_CQPSQ_STAG_ARIGHTS_SHIFT) - -#define I40IW_CQPSQ_STAG_REMACCENABLED_SHIFT 53 -#define I40IW_CQPSQ_STAG_REMACCENABLED_MASK \ - (1ULL << I40IW_CQPSQ_STAG_REMACCENABLED_SHIFT) - -#define I40IW_CQPSQ_STAG_VABASEDTO_SHIFT 59 -#define I40IW_CQPSQ_STAG_VABASEDTO_MASK \ - (1ULL << I40IW_CQPSQ_STAG_VABASEDTO_SHIFT) - -#define I40IW_CQPSQ_STAG_USEHMCFNIDX_SHIFT 60 -#define I40IW_CQPSQ_STAG_USEHMCFNIDX_MASK \ - (1ULL << I40IW_CQPSQ_STAG_USEHMCFNIDX_SHIFT) - -#define I40IW_CQPSQ_STAG_USEPFRID_SHIFT 61 -#define I40IW_CQPSQ_STAG_USEPFRID_MASK \ - (1ULL << I40IW_CQPSQ_STAG_USEPFRID_SHIFT) - -#define I40IW_CQPSQ_STAG_PBA_SHIFT I40IW_CQPHC_QPCTX_SHIFT -#define I40IW_CQPSQ_STAG_PBA_MASK I40IW_CQPHC_QPCTX_MASK - -#define I40IW_CQPSQ_STAG_HMCFNIDX_SHIFT 0 -#define I40IW_CQPSQ_STAG_HMCFNIDX_MASK \ - (0x3fUL << I40IW_CQPSQ_STAG_HMCFNIDX_SHIFT) - -#define I40IW_CQPSQ_STAG_FIRSTPMPBLIDX_SHIFT 0 -#define I40IW_CQPSQ_STAG_FIRSTPMPBLIDX_MASK \ - (0xfffffffUL << I40IW_CQPSQ_STAG_FIRSTPMPBLIDX_SHIFT) - -/* Query stag */ -#define I40IW_CQPSQ_QUERYSTAG_IDX_SHIFT I40IW_CQPSQ_STAG_IDX_SHIFT -#define I40IW_CQPSQ_QUERYSTAG_IDX_MASK I40IW_CQPSQ_STAG_IDX_MASK - -/* Allocate Local IP Address Entry */ - -/* Manage Local IP Address Table - MLIPA */ -#define I40IW_CQPSQ_MLIPA_IPV6LO_SHIFT I40IW_CQPHC_QPCTX_SHIFT -#define I40IW_CQPSQ_MLIPA_IPV6LO_MASK I40IW_CQPHC_QPCTX_MASK - -#define I40IW_CQPSQ_MLIPA_IPV6HI_SHIFT I40IW_CQPHC_QPCTX_SHIFT -#define I40IW_CQPSQ_MLIPA_IPV6HI_MASK I40IW_CQPHC_QPCTX_MASK - -#define I40IW_CQPSQ_MLIPA_IPV4_SHIFT 0 -#define I40IW_CQPSQ_MLIPA_IPV4_MASK \ - (0xffffffffUL << I40IW_CQPSQ_MLIPA_IPV4_SHIFT) - -#define I40IW_CQPSQ_MLIPA_IPTABLEIDX_SHIFT 0 -#define I40IW_CQPSQ_MLIPA_IPTABLEIDX_MASK \ - (0x3fUL << I40IW_CQPSQ_MLIPA_IPTABLEIDX_SHIFT) - -#define I40IW_CQPSQ_MLIPA_IPV4VALID_SHIFT 42 -#define I40IW_CQPSQ_MLIPA_IPV4VALID_MASK \ - (1ULL << I40IW_CQPSQ_MLIPA_IPV4VALID_SHIFT) - -#define I40IW_CQPSQ_MLIPA_IPV6VALID_SHIFT 43 -#define I40IW_CQPSQ_MLIPA_IPV6VALID_MASK \ - (1ULL << I40IW_CQPSQ_MLIPA_IPV6VALID_SHIFT) - -#define I40IW_CQPSQ_MLIPA_FREEENTRY_SHIFT 62 -#define I40IW_CQPSQ_MLIPA_FREEENTRY_MASK \ - (1ULL << I40IW_CQPSQ_MLIPA_FREEENTRY_SHIFT) - -#define I40IW_CQPSQ_MLIPA_IGNORE_REF_CNT_SHIFT 61 -#define I40IW_CQPSQ_MLIPA_IGNORE_REF_CNT_MASK \ - (1ULL << I40IW_CQPSQ_MLIPA_IGNORE_REF_CNT_SHIFT) - -#define I40IW_CQPSQ_MLIPA_MAC0_SHIFT 0 -#define I40IW_CQPSQ_MLIPA_MAC0_MASK (0xffUL << I40IW_CQPSQ_MLIPA_MAC0_SHIFT) - -#define I40IW_CQPSQ_MLIPA_MAC1_SHIFT 8 -#define I40IW_CQPSQ_MLIPA_MAC1_MASK (0xffUL << I40IW_CQPSQ_MLIPA_MAC1_SHIFT) - -#define I40IW_CQPSQ_MLIPA_MAC2_SHIFT 16 -#define I40IW_CQPSQ_MLIPA_MAC2_MASK (0xffUL << I40IW_CQPSQ_MLIPA_MAC2_SHIFT) - -#define I40IW_CQPSQ_MLIPA_MAC3_SHIFT 24 -#define I40IW_CQPSQ_MLIPA_MAC3_MASK (0xffUL << I40IW_CQPSQ_MLIPA_MAC3_SHIFT) - -#define I40IW_CQPSQ_MLIPA_MAC4_SHIFT 32 -#define I40IW_CQPSQ_MLIPA_MAC4_MASK (0xffULL << I40IW_CQPSQ_MLIPA_MAC4_SHIFT) - -#define I40IW_CQPSQ_MLIPA_MAC5_SHIFT 40 -#define I40IW_CQPSQ_MLIPA_MAC5_MASK (0xffULL << I40IW_CQPSQ_MLIPA_MAC5_SHIFT) - -/* Manage ARP Table - MAT */ -#define I40IW_CQPSQ_MAT_REACHMAX_SHIFT 0 -#define I40IW_CQPSQ_MAT_REACHMAX_MASK \ - (0xffffffffUL << I40IW_CQPSQ_MAT_REACHMAX_SHIFT) - -#define I40IW_CQPSQ_MAT_MACADDR_SHIFT 0 -#define I40IW_CQPSQ_MAT_MACADDR_MASK \ - (0xffffffffffffULL << I40IW_CQPSQ_MAT_MACADDR_SHIFT) - -#define I40IW_CQPSQ_MAT_ARPENTRYIDX_SHIFT 0 -#define I40IW_CQPSQ_MAT_ARPENTRYIDX_MASK \ - (0xfffUL << I40IW_CQPSQ_MAT_ARPENTRYIDX_SHIFT) - -#define I40IW_CQPSQ_MAT_ENTRYVALID_SHIFT 42 -#define I40IW_CQPSQ_MAT_ENTRYVALID_MASK \ - (1ULL << I40IW_CQPSQ_MAT_ENTRYVALID_SHIFT) - -#define I40IW_CQPSQ_MAT_PERMANENT_SHIFT 43 -#define I40IW_CQPSQ_MAT_PERMANENT_MASK \ - (1ULL << I40IW_CQPSQ_MAT_PERMANENT_SHIFT) - -#define I40IW_CQPSQ_MAT_QUERY_SHIFT 44 -#define I40IW_CQPSQ_MAT_QUERY_MASK (1ULL << I40IW_CQPSQ_MAT_QUERY_SHIFT) - -/* Manage VF PBLE Backing Pages - MVPBP*/ -#define I40IW_CQPSQ_MVPBP_PD_ENTRY_CNT_SHIFT 0 -#define I40IW_CQPSQ_MVPBP_PD_ENTRY_CNT_MASK \ - (0x3ffULL << I40IW_CQPSQ_MVPBP_PD_ENTRY_CNT_SHIFT) - -#define I40IW_CQPSQ_MVPBP_FIRST_PD_INX_SHIFT 16 -#define I40IW_CQPSQ_MVPBP_FIRST_PD_INX_MASK \ - (0x1ffULL << I40IW_CQPSQ_MVPBP_FIRST_PD_INX_SHIFT) - -#define I40IW_CQPSQ_MVPBP_SD_INX_SHIFT 32 -#define I40IW_CQPSQ_MVPBP_SD_INX_MASK \ - (0xfffULL << I40IW_CQPSQ_MVPBP_SD_INX_SHIFT) - -#define I40IW_CQPSQ_MVPBP_INV_PD_ENT_SHIFT 62 -#define I40IW_CQPSQ_MVPBP_INV_PD_ENT_MASK \ - (0x1ULL << I40IW_CQPSQ_MVPBP_INV_PD_ENT_SHIFT) - -#define I40IW_CQPSQ_MVPBP_PD_PLPBA_SHIFT 3 -#define I40IW_CQPSQ_MVPBP_PD_PLPBA_MASK \ - (0x1fffffffffffffffULL << I40IW_CQPSQ_MVPBP_PD_PLPBA_SHIFT) - -#define I40IW_INVALID_PUSH_PAGE_INDEX 0xffff - -#define I40IW_CQPSQ_MPP_QS_HANDLE_SHIFT 0 -#define I40IW_CQPSQ_MPP_QS_HANDLE_MASK (0xffffUL << \ - I40IW_CQPSQ_MPP_QS_HANDLE_SHIFT) - -#define I40IW_CQPSQ_MPP_PPIDX_SHIFT 0 -#define I40IW_CQPSQ_MPP_PPIDX_MASK (0x3ffUL << I40IW_CQPSQ_MPP_PPIDX_SHIFT) - -#define I40IW_CQPSQ_MPP_FREE_PAGE_SHIFT 62 -#define I40IW_CQPSQ_MPP_FREE_PAGE_MASK (1ULL << I40IW_CQPSQ_MPP_FREE_PAGE_SHIFT) - -/* Upload Context - UCTX */ -#define I40IW_CQPSQ_UCTX_QPCTXADDR_SHIFT I40IW_CQPHC_QPCTX_SHIFT -#define I40IW_CQPSQ_UCTX_QPCTXADDR_MASK I40IW_CQPHC_QPCTX_MASK - -#define I40IW_CQPSQ_UCTX_QPID_SHIFT 0 -#define I40IW_CQPSQ_UCTX_QPID_MASK (0x3ffffUL << I40IW_CQPSQ_UCTX_QPID_SHIFT) - -#define I40IW_CQPSQ_UCTX_QPTYPE_SHIFT 48 -#define I40IW_CQPSQ_UCTX_QPTYPE_MASK (0xfULL << I40IW_CQPSQ_UCTX_QPTYPE_SHIFT) - -#define I40IW_CQPSQ_UCTX_RAWFORMAT_SHIFT 61 -#define I40IW_CQPSQ_UCTX_RAWFORMAT_MASK \ - (1ULL << I40IW_CQPSQ_UCTX_RAWFORMAT_SHIFT) - -#define I40IW_CQPSQ_UCTX_FREEZEQP_SHIFT 62 -#define I40IW_CQPSQ_UCTX_FREEZEQP_MASK \ - (1ULL << I40IW_CQPSQ_UCTX_FREEZEQP_SHIFT) - -/* Manage HMC PM Function Table - MHMC */ -#define I40IW_CQPSQ_MHMC_VFIDX_SHIFT 0 -#define I40IW_CQPSQ_MHMC_VFIDX_MASK (0x7fUL << I40IW_CQPSQ_MHMC_VFIDX_SHIFT) - -#define I40IW_CQPSQ_MHMC_FREEPMFN_SHIFT 62 -#define I40IW_CQPSQ_MHMC_FREEPMFN_MASK \ - (1ULL << I40IW_CQPSQ_MHMC_FREEPMFN_SHIFT) - -/* Set HMC Resource Profile - SHMCRP */ -#define I40IW_CQPSQ_SHMCRP_HMC_PROFILE_SHIFT 0 -#define I40IW_CQPSQ_SHMCRP_HMC_PROFILE_MASK \ - (0x7ULL << I40IW_CQPSQ_SHMCRP_HMC_PROFILE_SHIFT) -#define I40IW_CQPSQ_SHMCRP_VFNUM_SHIFT 32 -#define I40IW_CQPSQ_SHMCRP_VFNUM_MASK (0x3fULL << I40IW_CQPSQ_SHMCRP_VFNUM_SHIFT) - -/* Create/Destroy CEQ */ -#define I40IW_CQPSQ_CEQ_CEQSIZE_SHIFT 0 -#define I40IW_CQPSQ_CEQ_CEQSIZE_MASK \ - (0x1ffffUL << I40IW_CQPSQ_CEQ_CEQSIZE_SHIFT) - -#define I40IW_CQPSQ_CEQ_CEQID_SHIFT 0 -#define I40IW_CQPSQ_CEQ_CEQID_MASK (0x7fUL << I40IW_CQPSQ_CEQ_CEQID_SHIFT) - -#define I40IW_CQPSQ_CEQ_LPBLSIZE_SHIFT I40IW_CQPSQ_CQ_LPBLSIZE_SHIFT -#define I40IW_CQPSQ_CEQ_LPBLSIZE_MASK I40IW_CQPSQ_CQ_LPBLSIZE_MASK - -#define I40IW_CQPSQ_CEQ_VMAP_SHIFT 47 -#define I40IW_CQPSQ_CEQ_VMAP_MASK (1ULL << I40IW_CQPSQ_CEQ_VMAP_SHIFT) - -#define I40IW_CQPSQ_CEQ_FIRSTPMPBLIDX_SHIFT 0 -#define I40IW_CQPSQ_CEQ_FIRSTPMPBLIDX_MASK \ - (0xfffffffUL << I40IW_CQPSQ_CEQ_FIRSTPMPBLIDX_SHIFT) - -/* Create/Destroy AEQ */ -#define I40IW_CQPSQ_AEQ_AEQECNT_SHIFT 0 -#define I40IW_CQPSQ_AEQ_AEQECNT_MASK \ - (0x7ffffUL << I40IW_CQPSQ_AEQ_AEQECNT_SHIFT) - -#define I40IW_CQPSQ_AEQ_LPBLSIZE_SHIFT I40IW_CQPSQ_CQ_LPBLSIZE_SHIFT -#define I40IW_CQPSQ_AEQ_LPBLSIZE_MASK I40IW_CQPSQ_CQ_LPBLSIZE_MASK - -#define I40IW_CQPSQ_AEQ_VMAP_SHIFT 47 -#define I40IW_CQPSQ_AEQ_VMAP_MASK (1ULL << I40IW_CQPSQ_AEQ_VMAP_SHIFT) - -#define I40IW_CQPSQ_AEQ_FIRSTPMPBLIDX_SHIFT 0 -#define I40IW_CQPSQ_AEQ_FIRSTPMPBLIDX_MASK \ - (0xfffffffUL << I40IW_CQPSQ_AEQ_FIRSTPMPBLIDX_SHIFT) - -/* Commit FPM Values - CFPM */ -#define I40IW_CQPSQ_CFPM_HMCFNID_SHIFT 0 -#define I40IW_CQPSQ_CFPM_HMCFNID_MASK (0x3fUL << I40IW_CQPSQ_CFPM_HMCFNID_SHIFT) - -/* Flush WQEs - FWQE */ -#define I40IW_CQPSQ_FWQE_AECODE_SHIFT 0 -#define I40IW_CQPSQ_FWQE_AECODE_MASK (0xffffUL << I40IW_CQPSQ_FWQE_AECODE_SHIFT) - -#define I40IW_CQPSQ_FWQE_AESOURCE_SHIFT 16 -#define I40IW_CQPSQ_FWQE_AESOURCE_MASK \ - (0xfUL << I40IW_CQPSQ_FWQE_AESOURCE_SHIFT) - -#define I40IW_CQPSQ_FWQE_RQMNERR_SHIFT 0 -#define I40IW_CQPSQ_FWQE_RQMNERR_MASK \ - (0xffffUL << I40IW_CQPSQ_FWQE_RQMNERR_SHIFT) - -#define I40IW_CQPSQ_FWQE_RQMJERR_SHIFT 16 -#define I40IW_CQPSQ_FWQE_RQMJERR_MASK \ - (0xffffUL << I40IW_CQPSQ_FWQE_RQMJERR_SHIFT) - -#define I40IW_CQPSQ_FWQE_SQMNERR_SHIFT 32 -#define I40IW_CQPSQ_FWQE_SQMNERR_MASK \ - (0xffffULL << I40IW_CQPSQ_FWQE_SQMNERR_SHIFT) - -#define I40IW_CQPSQ_FWQE_SQMJERR_SHIFT 48 -#define I40IW_CQPSQ_FWQE_SQMJERR_MASK \ - (0xffffULL << I40IW_CQPSQ_FWQE_SQMJERR_SHIFT) - -#define I40IW_CQPSQ_FWQE_QPID_SHIFT 0 -#define I40IW_CQPSQ_FWQE_QPID_MASK (0x3ffffULL << I40IW_CQPSQ_FWQE_QPID_SHIFT) - -#define I40IW_CQPSQ_FWQE_GENERATE_AE_SHIFT 59 -#define I40IW_CQPSQ_FWQE_GENERATE_AE_MASK (1ULL << \ - I40IW_CQPSQ_FWQE_GENERATE_AE_SHIFT) - -#define I40IW_CQPSQ_FWQE_USERFLCODE_SHIFT 60 -#define I40IW_CQPSQ_FWQE_USERFLCODE_MASK \ - (1ULL << I40IW_CQPSQ_FWQE_USERFLCODE_SHIFT) - -#define I40IW_CQPSQ_FWQE_FLUSHSQ_SHIFT 61 -#define I40IW_CQPSQ_FWQE_FLUSHSQ_MASK (1ULL << I40IW_CQPSQ_FWQE_FLUSHSQ_SHIFT) - -#define I40IW_CQPSQ_FWQE_FLUSHRQ_SHIFT 62 -#define I40IW_CQPSQ_FWQE_FLUSHRQ_MASK (1ULL << I40IW_CQPSQ_FWQE_FLUSHRQ_SHIFT) - -/* Manage Accelerated Port Table - MAPT */ -#define I40IW_CQPSQ_MAPT_PORT_SHIFT 0 -#define I40IW_CQPSQ_MAPT_PORT_MASK (0xffffUL << I40IW_CQPSQ_MAPT_PORT_SHIFT) - -#define I40IW_CQPSQ_MAPT_ADDPORT_SHIFT 62 -#define I40IW_CQPSQ_MAPT_ADDPORT_MASK (1ULL << I40IW_CQPSQ_MAPT_ADDPORT_SHIFT) - -/* Update Protocol Engine SDs */ -#define I40IW_CQPSQ_UPESD_SDCMD_SHIFT 0 -#define I40IW_CQPSQ_UPESD_SDCMD_MASK (0xffffffffUL << I40IW_CQPSQ_UPESD_SDCMD_SHIFT) - -#define I40IW_CQPSQ_UPESD_SDDATALOW_SHIFT 0 -#define I40IW_CQPSQ_UPESD_SDDATALOW_MASK \ - (0xffffffffUL << I40IW_CQPSQ_UPESD_SDDATALOW_SHIFT) - -#define I40IW_CQPSQ_UPESD_SDDATAHI_SHIFT 32 -#define I40IW_CQPSQ_UPESD_SDDATAHI_MASK \ - (0xffffffffULL << I40IW_CQPSQ_UPESD_SDDATAHI_SHIFT) -#define I40IW_CQPSQ_UPESD_HMCFNID_SHIFT 0 -#define I40IW_CQPSQ_UPESD_HMCFNID_MASK \ - (0x3fUL << I40IW_CQPSQ_UPESD_HMCFNID_SHIFT) - -#define I40IW_CQPSQ_UPESD_ENTRY_VALID_SHIFT 63 -#define I40IW_CQPSQ_UPESD_ENTRY_VALID_MASK \ - ((u64)1 << I40IW_CQPSQ_UPESD_ENTRY_VALID_SHIFT) - -#define I40IW_CQPSQ_UPESD_ENTRY_COUNT_SHIFT 0 -#define I40IW_CQPSQ_UPESD_ENTRY_COUNT_MASK \ - (0xfUL << I40IW_CQPSQ_UPESD_ENTRY_COUNT_SHIFT) - -#define I40IW_CQPSQ_UPESD_SKIP_ENTRY_SHIFT 7 -#define I40IW_CQPSQ_UPESD_SKIP_ENTRY_MASK \ - (0x1UL << I40IW_CQPSQ_UPESD_SKIP_ENTRY_SHIFT) - -/* Suspend QP */ -#define I40IW_CQPSQ_SUSPENDQP_QPID_SHIFT 0 -#define I40IW_CQPSQ_SUSPENDQP_QPID_MASK (0x3FFFFUL) -/* I40IWCQ_QPID_MASK */ - -/* Resume QP */ -#define I40IW_CQPSQ_RESUMEQP_QSHANDLE_SHIFT 0 -#define I40IW_CQPSQ_RESUMEQP_QSHANDLE_MASK \ - (0xffffffffUL << I40IW_CQPSQ_RESUMEQP_QSHANDLE_SHIFT) - -#define I40IW_CQPSQ_RESUMEQP_QPID_SHIFT 0 -#define I40IW_CQPSQ_RESUMEQP_QPID_MASK (0x3FFFFUL) -/* I40IWCQ_QPID_MASK */ - -/* IW QP Context */ -#define I40IWQPC_DDP_VER_SHIFT 0 -#define I40IWQPC_DDP_VER_MASK (3UL << I40IWQPC_DDP_VER_SHIFT) - -#define I40IWQPC_SNAP_SHIFT 2 -#define I40IWQPC_SNAP_MASK (1UL << I40IWQPC_SNAP_SHIFT) - -#define I40IWQPC_IPV4_SHIFT 3 -#define I40IWQPC_IPV4_MASK (1UL << I40IWQPC_IPV4_SHIFT) - -#define I40IWQPC_NONAGLE_SHIFT 4 -#define I40IWQPC_NONAGLE_MASK (1UL << I40IWQPC_NONAGLE_SHIFT) - -#define I40IWQPC_INSERTVLANTAG_SHIFT 5 -#define I40IWQPC_INSERTVLANTAG_MASK (1 << I40IWQPC_INSERTVLANTAG_SHIFT) - -#define I40IWQPC_USESRQ_SHIFT 6 -#define I40IWQPC_USESRQ_MASK (1UL << I40IWQPC_USESRQ_SHIFT) - -#define I40IWQPC_TIMESTAMP_SHIFT 7 -#define I40IWQPC_TIMESTAMP_MASK (1UL << I40IWQPC_TIMESTAMP_SHIFT) - -#define I40IWQPC_RQWQESIZE_SHIFT 8 -#define I40IWQPC_RQWQESIZE_MASK (3UL << I40IWQPC_RQWQESIZE_SHIFT) - -#define I40IWQPC_INSERTL2TAG2_SHIFT 11 -#define I40IWQPC_INSERTL2TAG2_MASK (1UL << I40IWQPC_INSERTL2TAG2_SHIFT) - -#define I40IWQPC_LIMIT_SHIFT 12 -#define I40IWQPC_LIMIT_MASK (3UL << I40IWQPC_LIMIT_SHIFT) - -#define I40IWQPC_DROPOOOSEG_SHIFT 15 -#define I40IWQPC_DROPOOOSEG_MASK (1UL << I40IWQPC_DROPOOOSEG_SHIFT) - -#define I40IWQPC_DUPACK_THRESH_SHIFT 16 -#define I40IWQPC_DUPACK_THRESH_MASK (7UL << I40IWQPC_DUPACK_THRESH_SHIFT) - -#define I40IWQPC_ERR_RQ_IDX_VALID_SHIFT 19 -#define I40IWQPC_ERR_RQ_IDX_VALID_MASK (1UL << I40IWQPC_ERR_RQ_IDX_VALID_SHIFT) - -#define I40IWQPC_DIS_VLAN_CHECKS_SHIFT 19 -#define I40IWQPC_DIS_VLAN_CHECKS_MASK (7UL << I40IWQPC_DIS_VLAN_CHECKS_SHIFT) - -#define I40IWQPC_RCVTPHEN_SHIFT 28 -#define I40IWQPC_RCVTPHEN_MASK (1UL << I40IWQPC_RCVTPHEN_SHIFT) - -#define I40IWQPC_XMITTPHEN_SHIFT 29 -#define I40IWQPC_XMITTPHEN_MASK (1ULL << I40IWQPC_XMITTPHEN_SHIFT) - -#define I40IWQPC_RQTPHEN_SHIFT 30 -#define I40IWQPC_RQTPHEN_MASK (1UL << I40IWQPC_RQTPHEN_SHIFT) - -#define I40IWQPC_SQTPHEN_SHIFT 31 -#define I40IWQPC_SQTPHEN_MASK (1ULL << I40IWQPC_SQTPHEN_SHIFT) - -#define I40IWQPC_PPIDX_SHIFT 32 -#define I40IWQPC_PPIDX_MASK (0x3ffULL << I40IWQPC_PPIDX_SHIFT) - -#define I40IWQPC_PMENA_SHIFT 47 -#define I40IWQPC_PMENA_MASK (1ULL << I40IWQPC_PMENA_SHIFT) - -#define I40IWQPC_RDMAP_VER_SHIFT 62 -#define I40IWQPC_RDMAP_VER_MASK (3ULL << I40IWQPC_RDMAP_VER_SHIFT) - -#define I40IWQPC_SQADDR_SHIFT I40IW_CQPHC_QPCTX_SHIFT -#define I40IWQPC_SQADDR_MASK I40IW_CQPHC_QPCTX_MASK - -#define I40IWQPC_RQADDR_SHIFT I40IW_CQPHC_QPCTX_SHIFT -#define I40IWQPC_RQADDR_MASK I40IW_CQPHC_QPCTX_MASK - -#define I40IWQPC_TTL_SHIFT 0 -#define I40IWQPC_TTL_MASK (0xffUL << I40IWQPC_TTL_SHIFT) - -#define I40IWQPC_RQSIZE_SHIFT 8 -#define I40IWQPC_RQSIZE_MASK (0xfUL << I40IWQPC_RQSIZE_SHIFT) - -#define I40IWQPC_SQSIZE_SHIFT 12 -#define I40IWQPC_SQSIZE_MASK (0xfUL << I40IWQPC_SQSIZE_SHIFT) - -#define I40IWQPC_SRCMACADDRIDX_SHIFT 16 -#define I40IWQPC_SRCMACADDRIDX_MASK (0x3fUL << I40IWQPC_SRCMACADDRIDX_SHIFT) - -#define I40IWQPC_AVOIDSTRETCHACK_SHIFT 23 -#define I40IWQPC_AVOIDSTRETCHACK_MASK (1UL << I40IWQPC_AVOIDSTRETCHACK_SHIFT) - -#define I40IWQPC_TOS_SHIFT 24 -#define I40IWQPC_TOS_MASK (0xffUL << I40IWQPC_TOS_SHIFT) - -#define I40IWQPC_SRCPORTNUM_SHIFT 32 -#define I40IWQPC_SRCPORTNUM_MASK (0xffffULL << I40IWQPC_SRCPORTNUM_SHIFT) - -#define I40IWQPC_DESTPORTNUM_SHIFT 48 -#define I40IWQPC_DESTPORTNUM_MASK (0xffffULL << I40IWQPC_DESTPORTNUM_SHIFT) - -#define I40IWQPC_DESTIPADDR0_SHIFT 32 -#define I40IWQPC_DESTIPADDR0_MASK \ - (0xffffffffULL << I40IWQPC_DESTIPADDR0_SHIFT) - -#define I40IWQPC_DESTIPADDR1_SHIFT 0 -#define I40IWQPC_DESTIPADDR1_MASK \ - (0xffffffffULL << I40IWQPC_DESTIPADDR1_SHIFT) - -#define I40IWQPC_DESTIPADDR2_SHIFT 32 -#define I40IWQPC_DESTIPADDR2_MASK \ - (0xffffffffULL << I40IWQPC_DESTIPADDR2_SHIFT) - -#define I40IWQPC_DESTIPADDR3_SHIFT 0 -#define I40IWQPC_DESTIPADDR3_MASK \ - (0xffffffffULL << I40IWQPC_DESTIPADDR3_SHIFT) - -#define I40IWQPC_SNDMSS_SHIFT 16 -#define I40IWQPC_SNDMSS_MASK (0x3fffUL << I40IWQPC_SNDMSS_SHIFT) - -#define I40IW_UDA_QPC_MAXFRAMESIZE_SHIFT 16 -#define I40IW_UDA_QPC_MAXFRAMESIZE_MASK (0x3fffUL << I40IW_UDA_QPC_MAXFRAMESIZE_SHIFT) - -#define I40IWQPC_VLANTAG_SHIFT 32 -#define I40IWQPC_VLANTAG_MASK (0xffffULL << I40IWQPC_VLANTAG_SHIFT) - -#define I40IWQPC_ARPIDX_SHIFT 48 -#define I40IWQPC_ARPIDX_MASK (0xffffULL << I40IWQPC_ARPIDX_SHIFT) - -#define I40IWQPC_FLOWLABEL_SHIFT 0 -#define I40IWQPC_FLOWLABEL_MASK (0xfffffUL << I40IWQPC_FLOWLABEL_SHIFT) - -#define I40IWQPC_WSCALE_SHIFT 20 -#define I40IWQPC_WSCALE_MASK (1UL << I40IWQPC_WSCALE_SHIFT) - -#define I40IWQPC_KEEPALIVE_SHIFT 21 -#define I40IWQPC_KEEPALIVE_MASK (1UL << I40IWQPC_KEEPALIVE_SHIFT) - -#define I40IWQPC_IGNORE_TCP_OPT_SHIFT 22 -#define I40IWQPC_IGNORE_TCP_OPT_MASK (1UL << I40IWQPC_IGNORE_TCP_OPT_SHIFT) - -#define I40IWQPC_IGNORE_TCP_UNS_OPT_SHIFT 23 -#define I40IWQPC_IGNORE_TCP_UNS_OPT_MASK \ - (1UL << I40IWQPC_IGNORE_TCP_UNS_OPT_SHIFT) - -#define I40IWQPC_TCPSTATE_SHIFT 28 -#define I40IWQPC_TCPSTATE_MASK (0xfUL << I40IWQPC_TCPSTATE_SHIFT) - -#define I40IWQPC_RCVSCALE_SHIFT 32 -#define I40IWQPC_RCVSCALE_MASK (0xfULL << I40IWQPC_RCVSCALE_SHIFT) - -#define I40IWQPC_SNDSCALE_SHIFT 40 -#define I40IWQPC_SNDSCALE_MASK (0xfULL << I40IWQPC_SNDSCALE_SHIFT) - -#define I40IWQPC_PDIDX_SHIFT 48 -#define I40IWQPC_PDIDX_MASK (0x7fffULL << I40IWQPC_PDIDX_SHIFT) - -#define I40IWQPC_KALIVE_TIMER_MAX_PROBES_SHIFT 16 -#define I40IWQPC_KALIVE_TIMER_MAX_PROBES_MASK \ - (0xffUL << I40IWQPC_KALIVE_TIMER_MAX_PROBES_SHIFT) - -#define I40IWQPC_KEEPALIVE_INTERVAL_SHIFT 24 -#define I40IWQPC_KEEPALIVE_INTERVAL_MASK \ - (0xffUL << I40IWQPC_KEEPALIVE_INTERVAL_SHIFT) - -#define I40IWQPC_TIMESTAMP_RECENT_SHIFT 0 -#define I40IWQPC_TIMESTAMP_RECENT_MASK \ - (0xffffffffUL << I40IWQPC_TIMESTAMP_RECENT_SHIFT) - -#define I40IWQPC_TIMESTAMP_AGE_SHIFT 32 -#define I40IWQPC_TIMESTAMP_AGE_MASK \ - (0xffffffffULL << I40IWQPC_TIMESTAMP_AGE_SHIFT) - -#define I40IWQPC_SNDNXT_SHIFT 0 -#define I40IWQPC_SNDNXT_MASK (0xffffffffUL << I40IWQPC_SNDNXT_SHIFT) - -#define I40IWQPC_SNDWND_SHIFT 32 -#define I40IWQPC_SNDWND_MASK (0xffffffffULL << I40IWQPC_SNDWND_SHIFT) - -#define I40IWQPC_RCVNXT_SHIFT 0 -#define I40IWQPC_RCVNXT_MASK (0xffffffffUL << I40IWQPC_RCVNXT_SHIFT) - -#define I40IWQPC_RCVWND_SHIFT 32 -#define I40IWQPC_RCVWND_MASK (0xffffffffULL << I40IWQPC_RCVWND_SHIFT) - -#define I40IWQPC_SNDMAX_SHIFT 0 -#define I40IWQPC_SNDMAX_MASK (0xffffffffUL << I40IWQPC_SNDMAX_SHIFT) - -#define I40IWQPC_SNDUNA_SHIFT 32 -#define I40IWQPC_SNDUNA_MASK (0xffffffffULL << I40IWQPC_SNDUNA_SHIFT) - -#define I40IWQPC_SRTT_SHIFT 0 -#define I40IWQPC_SRTT_MASK (0xffffffffUL << I40IWQPC_SRTT_SHIFT) - -#define I40IWQPC_RTTVAR_SHIFT 32 -#define I40IWQPC_RTTVAR_MASK (0xffffffffULL << I40IWQPC_RTTVAR_SHIFT) - -#define I40IWQPC_SSTHRESH_SHIFT 0 -#define I40IWQPC_SSTHRESH_MASK (0xffffffffUL << I40IWQPC_SSTHRESH_SHIFT) - -#define I40IWQPC_CWND_SHIFT 32 -#define I40IWQPC_CWND_MASK (0xffffffffULL << I40IWQPC_CWND_SHIFT) - -#define I40IWQPC_SNDWL1_SHIFT 0 -#define I40IWQPC_SNDWL1_MASK (0xffffffffUL << I40IWQPC_SNDWL1_SHIFT) - -#define I40IWQPC_SNDWL2_SHIFT 32 -#define I40IWQPC_SNDWL2_MASK (0xffffffffULL << I40IWQPC_SNDWL2_SHIFT) - -#define I40IWQPC_ERR_RQ_IDX_SHIFT 32 -#define I40IWQPC_ERR_RQ_IDX_MASK (0x3fffULL << I40IWQPC_ERR_RQ_IDX_SHIFT) - -#define I40IWQPC_MAXSNDWND_SHIFT 0 -#define I40IWQPC_MAXSNDWND_MASK (0xffffffffUL << I40IWQPC_MAXSNDWND_SHIFT) - -#define I40IWQPC_REXMIT_THRESH_SHIFT 48 -#define I40IWQPC_REXMIT_THRESH_MASK (0x3fULL << I40IWQPC_REXMIT_THRESH_SHIFT) - -#define I40IWQPC_TXCQNUM_SHIFT 0 -#define I40IWQPC_TXCQNUM_MASK (0x1ffffUL << I40IWQPC_TXCQNUM_SHIFT) - -#define I40IWQPC_RXCQNUM_SHIFT 32 -#define I40IWQPC_RXCQNUM_MASK (0x1ffffULL << I40IWQPC_RXCQNUM_SHIFT) - -#define I40IWQPC_STAT_INDEX_SHIFT 0 -#define I40IWQPC_STAT_INDEX_MASK (0x1fULL << I40IWQPC_STAT_INDEX_SHIFT) - -#define I40IWQPC_Q2ADDR_SHIFT 0 -#define I40IWQPC_Q2ADDR_MASK (0xffffffffffffff00ULL << I40IWQPC_Q2ADDR_SHIFT) - -#define I40IWQPC_LASTBYTESENT_SHIFT 0 -#define I40IWQPC_LASTBYTESENT_MASK (0xffUL << I40IWQPC_LASTBYTESENT_SHIFT) - -#define I40IWQPC_SRQID_SHIFT 32 -#define I40IWQPC_SRQID_MASK (0xffULL << I40IWQPC_SRQID_SHIFT) - -#define I40IWQPC_ORDSIZE_SHIFT 0 -#define I40IWQPC_ORDSIZE_MASK (0x7fUL << I40IWQPC_ORDSIZE_SHIFT) - -#define I40IWQPC_IRDSIZE_SHIFT 16 -#define I40IWQPC_IRDSIZE_MASK (0x3UL << I40IWQPC_IRDSIZE_SHIFT) - -#define I40IWQPC_WRRDRSPOK_SHIFT 20 -#define I40IWQPC_WRRDRSPOK_MASK (1UL << I40IWQPC_WRRDRSPOK_SHIFT) - -#define I40IWQPC_RDOK_SHIFT 21 -#define I40IWQPC_RDOK_MASK (1UL << I40IWQPC_RDOK_SHIFT) - -#define I40IWQPC_SNDMARKERS_SHIFT 22 -#define I40IWQPC_SNDMARKERS_MASK (1UL << I40IWQPC_SNDMARKERS_SHIFT) - -#define I40IWQPC_BINDEN_SHIFT 23 -#define I40IWQPC_BINDEN_MASK (1UL << I40IWQPC_BINDEN_SHIFT) - -#define I40IWQPC_FASTREGEN_SHIFT 24 -#define I40IWQPC_FASTREGEN_MASK (1UL << I40IWQPC_FASTREGEN_SHIFT) - -#define I40IWQPC_PRIVEN_SHIFT 25 -#define I40IWQPC_PRIVEN_MASK (1UL << I40IWQPC_PRIVEN_SHIFT) - -#define I40IWQPC_USESTATSINSTANCE_SHIFT 26 -#define I40IWQPC_USESTATSINSTANCE_MASK (1UL << I40IWQPC_USESTATSINSTANCE_SHIFT) - -#define I40IWQPC_IWARPMODE_SHIFT 28 -#define I40IWQPC_IWARPMODE_MASK (1UL << I40IWQPC_IWARPMODE_SHIFT) - -#define I40IWQPC_RCVMARKERS_SHIFT 29 -#define I40IWQPC_RCVMARKERS_MASK (1UL << I40IWQPC_RCVMARKERS_SHIFT) - -#define I40IWQPC_ALIGNHDRS_SHIFT 30 -#define I40IWQPC_ALIGNHDRS_MASK (1UL << I40IWQPC_ALIGNHDRS_SHIFT) - -#define I40IWQPC_RCVNOMPACRC_SHIFT 31 -#define I40IWQPC_RCVNOMPACRC_MASK (1UL << I40IWQPC_RCVNOMPACRC_SHIFT) - -#define I40IWQPC_RCVMARKOFFSET_SHIFT 33 -#define I40IWQPC_RCVMARKOFFSET_MASK (0x1ffULL << I40IWQPC_RCVMARKOFFSET_SHIFT) - -#define I40IWQPC_SNDMARKOFFSET_SHIFT 48 -#define I40IWQPC_SNDMARKOFFSET_MASK (0x1ffULL << I40IWQPC_SNDMARKOFFSET_SHIFT) - -#define I40IWQPC_QPCOMPCTX_SHIFT I40IW_CQPHC_QPCTX_SHIFT -#define I40IWQPC_QPCOMPCTX_MASK I40IW_CQPHC_QPCTX_MASK - -#define I40IWQPC_SQTPHVAL_SHIFT 0 -#define I40IWQPC_SQTPHVAL_MASK (0xffUL << I40IWQPC_SQTPHVAL_SHIFT) - -#define I40IWQPC_RQTPHVAL_SHIFT 8 -#define I40IWQPC_RQTPHVAL_MASK (0xffUL << I40IWQPC_RQTPHVAL_SHIFT) - -#define I40IWQPC_QSHANDLE_SHIFT 16 -#define I40IWQPC_QSHANDLE_MASK (0x3ffUL << I40IWQPC_QSHANDLE_SHIFT) - -#define I40IWQPC_EXCEPTION_LAN_QUEUE_SHIFT 32 -#define I40IWQPC_EXCEPTION_LAN_QUEUE_MASK (0xfffULL << \ - I40IWQPC_EXCEPTION_LAN_QUEUE_SHIFT) - -#define I40IWQPC_LOCAL_IPADDR3_SHIFT 0 -#define I40IWQPC_LOCAL_IPADDR3_MASK \ - (0xffffffffUL << I40IWQPC_LOCAL_IPADDR3_SHIFT) - -#define I40IWQPC_LOCAL_IPADDR2_SHIFT 32 -#define I40IWQPC_LOCAL_IPADDR2_MASK \ - (0xffffffffULL << I40IWQPC_LOCAL_IPADDR2_SHIFT) - -#define I40IWQPC_LOCAL_IPADDR1_SHIFT 0 -#define I40IWQPC_LOCAL_IPADDR1_MASK \ - (0xffffffffUL << I40IWQPC_LOCAL_IPADDR1_SHIFT) - -#define I40IWQPC_LOCAL_IPADDR0_SHIFT 32 -#define I40IWQPC_LOCAL_IPADDR0_MASK \ - (0xffffffffULL << I40IWQPC_LOCAL_IPADDR0_SHIFT) - -/* wqe size considering 32 bytes per wqe*/ -#define I40IW_QP_SW_MIN_WQSIZE 4 /*in WRs*/ -#define I40IW_SQ_RSVD 2 -#define I40IW_RQ_RSVD 1 -#define I40IW_MAX_QUANTAS_PER_WR 2 -#define I40IW_QP_SW_MAX_SQ_QUANTAS 2048 -#define I40IW_QP_SW_MAX_RQ_QUANTAS 16384 -#define I40IW_MAX_QP_WRS ((I40IW_QP_SW_MAX_SQ_QUANTAS / I40IW_MAX_QUANTAS_PER_WR) - 1) - -#define I40IWQP_OP_RDMA_WRITE 0 -#define I40IWQP_OP_RDMA_READ 1 -#define I40IWQP_OP_RDMA_SEND 3 -#define I40IWQP_OP_RDMA_SEND_INV 4 -#define I40IWQP_OP_RDMA_SEND_SOL_EVENT 5 -#define I40IWQP_OP_RDMA_SEND_SOL_EVENT_INV 6 -#define I40IWQP_OP_BIND_MW 8 -#define I40IWQP_OP_FAST_REGISTER 9 -#define I40IWQP_OP_LOCAL_INVALIDATE 10 -#define I40IWQP_OP_RDMA_READ_LOC_INV 11 -#define I40IWQP_OP_NOP 12 - -#define I40IW_RSVD_SHIFT 41 -#define I40IW_RSVD_MASK (0x7fffULL << I40IW_RSVD_SHIFT) - -/* iwarp QP SQ WQE common fields */ -#define I40IWQPSQ_OPCODE_SHIFT 32 -#define I40IWQPSQ_OPCODE_MASK (0x3fULL << I40IWQPSQ_OPCODE_SHIFT) - -#define I40IWQPSQ_ADDFRAGCNT_SHIFT 38 -#define I40IWQPSQ_ADDFRAGCNT_MASK (0x7ULL << I40IWQPSQ_ADDFRAGCNT_SHIFT) - -#define I40IWQPSQ_STREAMMODE_SHIFT 58 -#define I40IWQPSQ_STREAMMODE_MASK (1ULL << I40IWQPSQ_STREAMMODE_SHIFT) - -#define I40IWQPSQ_WAITFORRCVPDU_SHIFT 59 -#define I40IWQPSQ_WAITFORRCVPDU_MASK (1ULL << I40IWQPSQ_WAITFORRCVPDU_SHIFT) - -#define I40IWQPSQ_READFENCE_SHIFT 60 -#define I40IWQPSQ_READFENCE_MASK (1ULL << I40IWQPSQ_READFENCE_SHIFT) - -#define I40IWQPSQ_LOCALFENCE_SHIFT 61 -#define I40IWQPSQ_LOCALFENCE_MASK (1ULL << I40IWQPSQ_LOCALFENCE_SHIFT) - -#define I40IWQPSQ_SIGCOMPL_SHIFT 62 -#define I40IWQPSQ_SIGCOMPL_MASK (1ULL << I40IWQPSQ_SIGCOMPL_SHIFT) - -#define I40IWQPSQ_VALID_SHIFT 63 -#define I40IWQPSQ_VALID_MASK (1ULL << I40IWQPSQ_VALID_SHIFT) - -#define I40IWQPSQ_FRAG_TO_SHIFT I40IW_CQPHC_QPCTX_SHIFT -#define I40IWQPSQ_FRAG_TO_MASK I40IW_CQPHC_QPCTX_MASK - -#define I40IWQPSQ_FRAG_LEN_SHIFT 0 -#define I40IWQPSQ_FRAG_LEN_MASK (0xffffffffUL << I40IWQPSQ_FRAG_LEN_SHIFT) - -#define I40IWQPSQ_FRAG_STAG_SHIFT 32 -#define I40IWQPSQ_FRAG_STAG_MASK (0xffffffffULL << I40IWQPSQ_FRAG_STAG_SHIFT) - -#define I40IWQPSQ_REMSTAGINV_SHIFT 0 -#define I40IWQPSQ_REMSTAGINV_MASK (0xffffffffUL << I40IWQPSQ_REMSTAGINV_SHIFT) - -#define I40IWQPSQ_INLINEDATAFLAG_SHIFT 57 -#define I40IWQPSQ_INLINEDATAFLAG_MASK (1ULL << I40IWQPSQ_INLINEDATAFLAG_SHIFT) - -#define I40IWQPSQ_INLINEDATALEN_SHIFT 48 -#define I40IWQPSQ_INLINEDATALEN_MASK \ - (0x7fULL << I40IWQPSQ_INLINEDATALEN_SHIFT) - -/* iwarp send with push mode */ -#define I40IWQPSQ_WQDESCIDX_SHIFT 0 -#define I40IWQPSQ_WQDESCIDX_MASK (0x3fffUL << I40IWQPSQ_WQDESCIDX_SHIFT) - -/* rdma write */ -#define I40IWQPSQ_REMSTAG_SHIFT 0 -#define I40IWQPSQ_REMSTAG_MASK (0xffffffffUL << I40IWQPSQ_REMSTAG_SHIFT) - -#define I40IWQPSQ_REMTO_SHIFT I40IW_CQPHC_QPCTX_SHIFT -#define I40IWQPSQ_REMTO_MASK I40IW_CQPHC_QPCTX_MASK - -/* memory window */ -#define I40IWQPSQ_STAGRIGHTS_SHIFT 48 -#define I40IWQPSQ_STAGRIGHTS_MASK (0x1fULL << I40IWQPSQ_STAGRIGHTS_SHIFT) - -#define I40IWQPSQ_VABASEDTO_SHIFT 53 -#define I40IWQPSQ_VABASEDTO_MASK (1ULL << I40IWQPSQ_VABASEDTO_SHIFT) - -#define I40IWQPSQ_MWLEN_SHIFT I40IW_CQPHC_QPCTX_SHIFT -#define I40IWQPSQ_MWLEN_MASK I40IW_CQPHC_QPCTX_MASK - -#define I40IWQPSQ_PARENTMRSTAG_SHIFT 0 -#define I40IWQPSQ_PARENTMRSTAG_MASK \ - (0xffffffffUL << I40IWQPSQ_PARENTMRSTAG_SHIFT) - -#define I40IWQPSQ_MWSTAG_SHIFT 32 -#define I40IWQPSQ_MWSTAG_MASK (0xffffffffULL << I40IWQPSQ_MWSTAG_SHIFT) - -#define I40IWQPSQ_BASEVA_TO_FBO_SHIFT I40IW_CQPHC_QPCTX_SHIFT -#define I40IWQPSQ_BASEVA_TO_FBO_MASK I40IW_CQPHC_QPCTX_MASK - -/* Local Invalidate */ -#define I40IWQPSQ_LOCSTAG_SHIFT 32 -#define I40IWQPSQ_LOCSTAG_MASK (0xffffffffULL << I40IWQPSQ_LOCSTAG_SHIFT) - -/* Fast Register */ -#define I40IWQPSQ_STAGKEY_SHIFT 0 -#define I40IWQPSQ_STAGKEY_MASK (0xffUL << I40IWQPSQ_STAGKEY_SHIFT) - -#define I40IWQPSQ_STAGINDEX_SHIFT 8 -#define I40IWQPSQ_STAGINDEX_MASK (0xffffffUL << I40IWQPSQ_STAGINDEX_SHIFT) - -#define I40IWQPSQ_COPYHOSTPBLS_SHIFT 43 -#define I40IWQPSQ_COPYHOSTPBLS_MASK (1ULL << I40IWQPSQ_COPYHOSTPBLS_SHIFT) - -#define I40IWQPSQ_LPBLSIZE_SHIFT 44 -#define I40IWQPSQ_LPBLSIZE_MASK (3ULL << I40IWQPSQ_LPBLSIZE_SHIFT) - -#define I40IWQPSQ_HPAGESIZE_SHIFT 46 -#define I40IWQPSQ_HPAGESIZE_MASK (3ULL << I40IWQPSQ_HPAGESIZE_SHIFT) - -#define I40IWQPSQ_STAGLEN_SHIFT 0 -#define I40IWQPSQ_STAGLEN_MASK (0x1ffffffffffULL << I40IWQPSQ_STAGLEN_SHIFT) - -#define I40IWQPSQ_FIRSTPMPBLIDXLO_SHIFT 48 -#define I40IWQPSQ_FIRSTPMPBLIDXLO_MASK \ - (0xffffULL << I40IWQPSQ_FIRSTPMPBLIDXLO_SHIFT) - -#define I40IWQPSQ_FIRSTPMPBLIDXHI_SHIFT 0 -#define I40IWQPSQ_FIRSTPMPBLIDXHI_MASK \ - (0xfffUL << I40IWQPSQ_FIRSTPMPBLIDXHI_SHIFT) - -#define I40IWQPSQ_PBLADDR_SHIFT 12 -#define I40IWQPSQ_PBLADDR_MASK (0xfffffffffffffULL << I40IWQPSQ_PBLADDR_SHIFT) - -/* iwarp QP RQ WQE common fields */ -#define I40IWQPRQ_ADDFRAGCNT_SHIFT I40IWQPSQ_ADDFRAGCNT_SHIFT -#define I40IWQPRQ_ADDFRAGCNT_MASK I40IWQPSQ_ADDFRAGCNT_MASK - -#define I40IWQPRQ_VALID_SHIFT I40IWQPSQ_VALID_SHIFT -#define I40IWQPRQ_VALID_MASK I40IWQPSQ_VALID_MASK - -#define I40IWQPRQ_COMPLCTX_SHIFT I40IW_CQPHC_QPCTX_SHIFT -#define I40IWQPRQ_COMPLCTX_MASK I40IW_CQPHC_QPCTX_MASK - -#define I40IWQPRQ_FRAG_LEN_SHIFT I40IWQPSQ_FRAG_LEN_SHIFT -#define I40IWQPRQ_FRAG_LEN_MASK I40IWQPSQ_FRAG_LEN_MASK - -#define I40IWQPRQ_STAG_SHIFT I40IWQPSQ_FRAG_STAG_SHIFT -#define I40IWQPRQ_STAG_MASK I40IWQPSQ_FRAG_STAG_MASK - -#define I40IWQPRQ_TO_SHIFT I40IWQPSQ_FRAG_TO_SHIFT -#define I40IWQPRQ_TO_MASK I40IWQPSQ_FRAG_TO_MASK - -/* Query FPM CQP buf */ -#define I40IW_QUERY_FPM_MAX_QPS_SHIFT 0 -#define I40IW_QUERY_FPM_MAX_QPS_MASK \ - (0x7ffffUL << I40IW_QUERY_FPM_MAX_QPS_SHIFT) - -#define I40IW_QUERY_FPM_MAX_CQS_SHIFT 0 -#define I40IW_QUERY_FPM_MAX_CQS_MASK \ - (0x3ffffUL << I40IW_QUERY_FPM_MAX_CQS_SHIFT) - -#define I40IW_QUERY_FPM_FIRST_PE_SD_INDEX_SHIFT 0 -#define I40IW_QUERY_FPM_FIRST_PE_SD_INDEX_MASK \ - (0x3fffUL << I40IW_QUERY_FPM_FIRST_PE_SD_INDEX_SHIFT) - -#define I40IW_QUERY_FPM_MAX_PE_SDS_SHIFT 32 -#define I40IW_QUERY_FPM_MAX_PE_SDS_MASK \ - (0x3fffULL << I40IW_QUERY_FPM_MAX_PE_SDS_SHIFT) - -#define I40IW_QUERY_FPM_MAX_QPS_SHIFT 0 -#define I40IW_QUERY_FPM_MAX_QPS_MASK \ - (0x7ffffUL << I40IW_QUERY_FPM_MAX_QPS_SHIFT) - -#define I40IW_QUERY_FPM_MAX_CQS_SHIFT 0 -#define I40IW_QUERY_FPM_MAX_CQS_MASK \ - (0x3ffffUL << I40IW_QUERY_FPM_MAX_CQS_SHIFT) - -#define I40IW_QUERY_FPM_MAX_CEQS_SHIFT 0 -#define I40IW_QUERY_FPM_MAX_CEQS_MASK \ - (0xffUL << I40IW_QUERY_FPM_MAX_CEQS_SHIFT) - -#define I40IW_QUERY_FPM_XFBLOCKSIZE_SHIFT 32 -#define I40IW_QUERY_FPM_XFBLOCKSIZE_MASK \ - (0xffffffffULL << I40IW_QUERY_FPM_XFBLOCKSIZE_SHIFT) - -#define I40IW_QUERY_FPM_Q1BLOCKSIZE_SHIFT 32 -#define I40IW_QUERY_FPM_Q1BLOCKSIZE_MASK \ - (0xffffffffULL << I40IW_QUERY_FPM_Q1BLOCKSIZE_SHIFT) - -#define I40IW_QUERY_FPM_HTMULTIPLIER_SHIFT 16 -#define I40IW_QUERY_FPM_HTMULTIPLIER_MASK \ - (0xfUL << I40IW_QUERY_FPM_HTMULTIPLIER_SHIFT) - -#define I40IW_QUERY_FPM_TIMERBUCKET_SHIFT 32 -#define I40IW_QUERY_FPM_TIMERBUCKET_MASK \ - (0xffFFULL << I40IW_QUERY_FPM_TIMERBUCKET_SHIFT) - -/* Static HMC pages allocated buf */ -#define I40IW_SHMC_PAGE_ALLOCATED_HMC_FN_ID_SHIFT 0 -#define I40IW_SHMC_PAGE_ALLOCATED_HMC_FN_ID_MASK \ - (0x3fUL << I40IW_SHMC_PAGE_ALLOCATED_HMC_FN_ID_SHIFT) - -#define I40IW_HW_PAGE_SIZE 4096 -#define I40IW_DONE_COUNT 1000 -#define I40IW_SLEEP_COUNT 10 - -enum { - I40IW_QUEUES_ALIGNMENT_MASK = (128 - 1), - I40IW_AEQ_ALIGNMENT_MASK = (256 - 1), - I40IW_Q2_ALIGNMENT_MASK = (256 - 1), - I40IW_CEQ_ALIGNMENT_MASK = (256 - 1), - I40IW_CQ0_ALIGNMENT_MASK = (256 - 1), - I40IW_HOST_CTX_ALIGNMENT_MASK = (4 - 1), - I40IW_SHADOWAREA_MASK = (128 - 1), - I40IW_FPM_QUERY_BUF_ALIGNMENT_MASK = (4 - 1), - I40IW_FPM_COMMIT_BUF_ALIGNMENT_MASK = (4 - 1) -}; - -enum i40iw_alignment { - I40IW_CQP_ALIGNMENT = 0x200, - I40IW_AEQ_ALIGNMENT = 0x100, - I40IW_CEQ_ALIGNMENT = 0x100, - I40IW_CQ0_ALIGNMENT = 0x100, - I40IW_SD_BUF_ALIGNMENT = 0x80, - I40IW_FEATURE_BUF_ALIGNMENT = 0x8 -}; - -#define I40IW_WQE_SIZE_64 64 - -#define I40IW_QP_WQE_MIN_SIZE 32 -#define I40IW_QP_WQE_MAX_SIZE 128 - -#define I40IW_UPDATE_SD_BUF_SIZE 128 - -#define I40IW_CQE_QTYPE_RQ 0 -#define I40IW_CQE_QTYPE_SQ 1 - -#define I40IW_RING_INIT(_ring, _size) \ - { \ - (_ring).head = 0; \ - (_ring).tail = 0; \ - (_ring).size = (_size); \ - } -#define I40IW_RING_GETSIZE(_ring) ((_ring).size) -#define I40IW_RING_GETCURRENT_HEAD(_ring) ((_ring).head) -#define I40IW_RING_GETCURRENT_TAIL(_ring) ((_ring).tail) - -#define I40IW_RING_MOVE_HEAD(_ring, _retcode) \ - { \ - register u32 size; \ - size = (_ring).size; \ - if (!I40IW_RING_FULL_ERR(_ring)) { \ - (_ring).head = ((_ring).head + 1) % size; \ - (_retcode) = 0; \ - } else { \ - (_retcode) = I40IW_ERR_RING_FULL; \ - } \ - } - -#define I40IW_RING_MOVE_HEAD_BY_COUNT(_ring, _count, _retcode) \ - { \ - register u32 size; \ - size = (_ring).size; \ - if ((I40IW_RING_WORK_AVAILABLE(_ring) + (_count)) < size) { \ - (_ring).head = ((_ring).head + (_count)) % size; \ - (_retcode) = 0; \ - } else { \ - (_retcode) = I40IW_ERR_RING_FULL; \ - } \ - } - -#define I40IW_RING_MOVE_TAIL(_ring) \ - (_ring).tail = ((_ring).tail + 1) % (_ring).size - -#define I40IW_RING_MOVE_HEAD_NOCHECK(_ring) \ - (_ring).head = ((_ring).head + 1) % (_ring).size - -#define I40IW_RING_MOVE_TAIL_BY_COUNT(_ring, _count) \ - (_ring).tail = ((_ring).tail + (_count)) % (_ring).size - -#define I40IW_RING_SET_TAIL(_ring, _pos) \ - (_ring).tail = (_pos) % (_ring).size - -#define I40IW_RING_FULL_ERR(_ring) \ - ( \ - (I40IW_RING_WORK_AVAILABLE(_ring) == ((_ring).size - 1)) \ - ) - -#define I40IW_ERR_RING_FULL2(_ring) \ - ( \ - (I40IW_RING_WORK_AVAILABLE(_ring) == ((_ring).size - 2)) \ - ) - -#define I40IW_ERR_RING_FULL3(_ring) \ - ( \ - (I40IW_RING_WORK_AVAILABLE(_ring) == ((_ring).size - 3)) \ - ) - -#define I40IW_RING_MORE_WORK(_ring) \ - ( \ - (I40IW_RING_WORK_AVAILABLE(_ring) != 0) \ - ) - -#define I40IW_RING_WORK_AVAILABLE(_ring) \ - ( \ - (((_ring).head + (_ring).size - (_ring).tail) % (_ring).size) \ - ) - -#define I40IW_RING_GET_WQES_AVAILABLE(_ring) \ - ( \ - ((_ring).size - I40IW_RING_WORK_AVAILABLE(_ring) - 1) \ - ) - -#define I40IW_ATOMIC_RING_MOVE_HEAD(_ring, index, _retcode) \ - { \ - index = I40IW_RING_GETCURRENT_HEAD(_ring); \ - I40IW_RING_MOVE_HEAD(_ring, _retcode); \ - } - -/* Async Events codes */ -#define I40IW_AE_AMP_UNALLOCATED_STAG 0x0102 -#define I40IW_AE_AMP_INVALID_STAG 0x0103 -#define I40IW_AE_AMP_BAD_QP 0x0104 -#define I40IW_AE_AMP_BAD_PD 0x0105 -#define I40IW_AE_AMP_BAD_STAG_KEY 0x0106 -#define I40IW_AE_AMP_BAD_STAG_INDEX 0x0107 -#define I40IW_AE_AMP_BOUNDS_VIOLATION 0x0108 -#define I40IW_AE_AMP_RIGHTS_VIOLATION 0x0109 -#define I40IW_AE_AMP_TO_WRAP 0x010a -#define I40IW_AE_AMP_FASTREG_SHARED 0x010b -#define I40IW_AE_AMP_FASTREG_VALID_STAG 0x010c -#define I40IW_AE_AMP_FASTREG_MW_STAG 0x010d -#define I40IW_AE_AMP_FASTREG_INVALID_RIGHTS 0x010e -#define I40IW_AE_AMP_FASTREG_PBL_TABLE_OVERFLOW 0x010f -#define I40IW_AE_AMP_FASTREG_INVALID_LENGTH 0x0110 -#define I40IW_AE_AMP_INVALIDATE_SHARED 0x0111 -#define I40IW_AE_AMP_INVALIDATE_NO_REMOTE_ACCESS_RIGHTS 0x0112 -#define I40IW_AE_AMP_INVALIDATE_MR_WITH_BOUND_WINDOWS 0x0113 -#define I40IW_AE_AMP_MWBIND_VALID_STAG 0x0114 -#define I40IW_AE_AMP_MWBIND_OF_MR_STAG 0x0115 -#define I40IW_AE_AMP_MWBIND_TO_ZERO_BASED_STAG 0x0116 -#define I40IW_AE_AMP_MWBIND_TO_MW_STAG 0x0117 -#define I40IW_AE_AMP_MWBIND_INVALID_RIGHTS 0x0118 -#define I40IW_AE_AMP_MWBIND_INVALID_BOUNDS 0x0119 -#define I40IW_AE_AMP_MWBIND_TO_INVALID_PARENT 0x011a -#define I40IW_AE_AMP_MWBIND_BIND_DISABLED 0x011b -#define I40IW_AE_UDA_XMIT_DGRAM_TOO_LONG 0x0132 -#define I40IW_AE_UDA_XMIT_DGRAM_TOO_SHORT 0x0134 -#define I40IW_AE_BAD_CLOSE 0x0201 -#define I40IW_AE_RDMAP_ROE_BAD_LLP_CLOSE 0x0202 -#define I40IW_AE_CQ_OPERATION_ERROR 0x0203 -#define I40IW_AE_PRIV_OPERATION_DENIED 0x011c -#define I40IW_AE_RDMA_READ_WHILE_ORD_ZERO 0x0205 -#define I40IW_AE_STAG_ZERO_INVALID 0x0206 -#define I40IW_AE_IB_RREQ_AND_Q1_FULL 0x0207 -#define I40IW_AE_WQE_UNEXPECTED_OPCODE 0x020a -#define I40IW_AE_WQE_INVALID_PARAMETER 0x020b -#define I40IW_AE_WQE_LSMM_TOO_LONG 0x0220 -#define I40IW_AE_DDP_INVALID_MSN_GAP_IN_MSN 0x0301 -#define I40IW_AE_DDP_UBE_DDP_MESSAGE_TOO_LONG_FOR_AVAILABLE_BUFFER 0x0303 -#define I40IW_AE_DDP_UBE_INVALID_DDP_VERSION 0x0304 -#define I40IW_AE_DDP_UBE_INVALID_MO 0x0305 -#define I40IW_AE_DDP_UBE_INVALID_MSN_NO_BUFFER_AVAILABLE 0x0306 -#define I40IW_AE_DDP_UBE_INVALID_QN 0x0307 -#define I40IW_AE_DDP_NO_L_BIT 0x0308 -#define I40IW_AE_RDMAP_ROE_INVALID_RDMAP_VERSION 0x0311 -#define I40IW_AE_RDMAP_ROE_UNEXPECTED_OPCODE 0x0312 -#define I40IW_AE_ROE_INVALID_RDMA_READ_REQUEST 0x0313 -#define I40IW_AE_ROE_INVALID_RDMA_WRITE_OR_READ_RESP 0x0314 -#define I40IW_AE_INVALID_ARP_ENTRY 0x0401 -#define I40IW_AE_INVALID_TCP_OPTION_RCVD 0x0402 -#define I40IW_AE_STALE_ARP_ENTRY 0x0403 -#define I40IW_AE_INVALID_MAC_ENTRY 0x0405 -#define I40IW_AE_LLP_CLOSE_COMPLETE 0x0501 -#define I40IW_AE_LLP_CONNECTION_RESET 0x0502 -#define I40IW_AE_LLP_FIN_RECEIVED 0x0503 -#define I40IW_AE_LLP_RECEIVED_MPA_CRC_ERROR 0x0505 -#define I40IW_AE_LLP_SEGMENT_TOO_LARGE 0x0506 -#define I40IW_AE_LLP_SEGMENT_TOO_SMALL 0x0507 -#define I40IW_AE_LLP_SYN_RECEIVED 0x0508 -#define I40IW_AE_LLP_TERMINATE_RECEIVED 0x0509 -#define I40IW_AE_LLP_TOO_MANY_RETRIES 0x050a -#define I40IW_AE_LLP_TOO_MANY_KEEPALIVE_RETRIES 0x050b -#define I40IW_AE_LLP_DOUBT_REACHABILITY 0x050c -#define I40IW_AE_LLP_RX_VLAN_MISMATCH 0x050d -#define I40IW_AE_RESOURCE_EXHAUSTION 0x0520 -#define I40IW_AE_RESET_SENT 0x0601 -#define I40IW_AE_TERMINATE_SENT 0x0602 -#define I40IW_AE_RESET_NOT_SENT 0x0603 -#define I40IW_AE_LCE_QP_CATASTROPHIC 0x0700 -#define I40IW_AE_LCE_FUNCTION_CATASTROPHIC 0x0701 -#define I40IW_AE_LCE_CQ_CATASTROPHIC 0x0702 -#define I40IW_AE_QP_SUSPEND_COMPLETE 0x0900 - -#define OP_DELETE_LOCAL_MAC_IPADDR_ENTRY 1 -#define OP_CEQ_DESTROY 2 -#define OP_AEQ_DESTROY 3 -#define OP_DELETE_ARP_CACHE_ENTRY 4 -#define OP_MANAGE_APBVT_ENTRY 5 -#define OP_CEQ_CREATE 6 -#define OP_AEQ_CREATE 7 -#define OP_ALLOC_LOCAL_MAC_IPADDR_ENTRY 8 -#define OP_ADD_LOCAL_MAC_IPADDR_ENTRY 9 -#define OP_MANAGE_QHASH_TABLE_ENTRY 10 -#define OP_QP_MODIFY 11 -#define OP_QP_UPLOAD_CONTEXT 12 -#define OP_CQ_CREATE 13 -#define OP_CQ_DESTROY 14 -#define OP_QP_CREATE 15 -#define OP_QP_DESTROY 16 -#define OP_ALLOC_STAG 17 -#define OP_MR_REG_NON_SHARED 18 -#define OP_DEALLOC_STAG 19 -#define OP_MW_ALLOC 20 -#define OP_QP_FLUSH_WQES 21 -#define OP_ADD_ARP_CACHE_ENTRY 22 -#define OP_UPDATE_PE_SDS 23 -#define OP_MANAGE_HMC_PM_FUNC_TABLE 24 -#define OP_SUSPEND 25 -#define OP_RESUME 26 -#define OP_MANAGE_VF_PBLE_BP 27 -#define OP_QUERY_FPM_VALUES 28 -#define OP_COMMIT_FPM_VALUES 29 -#define OP_REQUESTED_COMMANDS 30 -#define OP_COMPLETED_COMMANDS 31 -#define OP_GEN_AE 32 -#define OP_QUERY_RDMA_FEATURES 33 -#define OP_SIZE_CQP_STAT_ARRAY 34 - -#endif diff --git a/drivers/infiniband/hw/i40iw/i40iw_hmc.c b/drivers/infiniband/hw/i40iw/i40iw_hmc.c deleted file mode 100644 index b44bfc1d239b..000000000000 --- a/drivers/infiniband/hw/i40iw/i40iw_hmc.c +++ /dev/null @@ -1,821 +0,0 @@ -/******************************************************************************* -* -* Copyright (c) 2015-2016 Intel Corporation. All rights reserved. -* -* This software is available to you under a choice of one of two -* licenses. You may choose to be licensed under the terms of the GNU -* General Public License (GPL) Version 2, available from the file -* COPYING in the main directory of this source tree, or the -* OpenFabrics.org BSD license below: -* -* Redistribution and use in source and binary forms, with or -* without modification, are permitted provided that the following -* conditions are met: -* -* - Redistributions of source code must retain the above -* copyright notice, this list of conditions and the following -* disclaimer. -* -* - Redistributions in binary form must reproduce the above -* copyright notice, this list of conditions and the following -* disclaimer in the documentation and/or other materials -* provided with the distribution. -* -* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS -* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN -* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -* SOFTWARE. -* -*******************************************************************************/ - -#include "i40iw_osdep.h" -#include "i40iw_register.h" -#include "i40iw_status.h" -#include "i40iw_hmc.h" -#include "i40iw_d.h" -#include "i40iw_type.h" -#include "i40iw_p.h" -#include "i40iw_vf.h" -#include "i40iw_virtchnl.h" - -/** - * i40iw_find_sd_index_limit - finds segment descriptor index limit - * @hmc_info: pointer to the HMC configuration information structure - * @type: type of HMC resources we're searching - * @idx: starting index for the object - * @cnt: number of objects we're trying to create - * @sd_idx: pointer to return index of the segment descriptor in question - * @sd_limit: pointer to return the maximum number of segment descriptors - * - * This function calculates the segment descriptor index and index limit - * for the resource defined by i40iw_hmc_rsrc_type. - */ - -static inline void i40iw_find_sd_index_limit(struct i40iw_hmc_info *hmc_info, - u32 type, - u32 idx, - u32 cnt, - u32 *sd_idx, - u32 *sd_limit) -{ - u64 fpm_addr, fpm_limit; - - fpm_addr = hmc_info->hmc_obj[(type)].base + - hmc_info->hmc_obj[type].size * idx; - fpm_limit = fpm_addr + hmc_info->hmc_obj[type].size * cnt; - *sd_idx = (u32)(fpm_addr / I40IW_HMC_DIRECT_BP_SIZE); - *sd_limit = (u32)((fpm_limit - 1) / I40IW_HMC_DIRECT_BP_SIZE); - *sd_limit += 1; -} - -/** - * i40iw_find_pd_index_limit - finds page descriptor index limit - * @hmc_info: pointer to the HMC configuration information struct - * @type: HMC resource type we're examining - * @idx: starting index for the object - * @cnt: number of objects we're trying to create - * @pd_idx: pointer to return page descriptor index - * @pd_limit: pointer to return page descriptor index limit - * - * Calculates the page descriptor index and index limit for the resource - * defined by i40iw_hmc_rsrc_type. - */ - -static inline void i40iw_find_pd_index_limit(struct i40iw_hmc_info *hmc_info, - u32 type, - u32 idx, - u32 cnt, - u32 *pd_idx, - u32 *pd_limit) -{ - u64 fpm_adr, fpm_limit; - - fpm_adr = hmc_info->hmc_obj[type].base + - hmc_info->hmc_obj[type].size * idx; - fpm_limit = fpm_adr + (hmc_info)->hmc_obj[(type)].size * (cnt); - *(pd_idx) = (u32)(fpm_adr / I40IW_HMC_PAGED_BP_SIZE); - *(pd_limit) = (u32)((fpm_limit - 1) / I40IW_HMC_PAGED_BP_SIZE); - *(pd_limit) += 1; -} - -/** - * i40iw_set_sd_entry - setup entry for sd programming - * @pa: physical addr - * @idx: sd index - * @type: paged or direct sd - * @entry: sd entry ptr - */ -static inline void i40iw_set_sd_entry(u64 pa, - u32 idx, - enum i40iw_sd_entry_type type, - struct update_sd_entry *entry) -{ - entry->data = pa | (I40IW_HMC_MAX_BP_COUNT << I40E_PFHMC_SDDATALOW_PMSDBPCOUNT_SHIFT) | - (((type == I40IW_SD_TYPE_PAGED) ? 0 : 1) << - I40E_PFHMC_SDDATALOW_PMSDTYPE_SHIFT) | - (1 << I40E_PFHMC_SDDATALOW_PMSDVALID_SHIFT); - entry->cmd = (idx | (1 << I40E_PFHMC_SDCMD_PMSDWR_SHIFT) | (1 << 15)); -} - -/** - * i40iw_clr_sd_entry - setup entry for sd clear - * @idx: sd index - * @type: paged or direct sd - * @entry: sd entry ptr - */ -static inline void i40iw_clr_sd_entry(u32 idx, enum i40iw_sd_entry_type type, - struct update_sd_entry *entry) -{ - entry->data = (I40IW_HMC_MAX_BP_COUNT << - I40E_PFHMC_SDDATALOW_PMSDBPCOUNT_SHIFT) | - (((type == I40IW_SD_TYPE_PAGED) ? 0 : 1) << - I40E_PFHMC_SDDATALOW_PMSDTYPE_SHIFT); - entry->cmd = (idx | (1 << I40E_PFHMC_SDCMD_PMSDWR_SHIFT) | (1 << 15)); -} - -/** - * i40iw_hmc_sd_one - setup 1 sd entry for cqp - * @dev: pointer to the device structure - * @hmc_fn_id: hmc's function id - * @pa: physical addr - * @sd_idx: sd index - * @type: paged or direct sd - * @setsd: flag to set or clear sd - */ -enum i40iw_status_code i40iw_hmc_sd_one(struct i40iw_sc_dev *dev, - u8 hmc_fn_id, - u64 pa, u32 sd_idx, - enum i40iw_sd_entry_type type, - bool setsd) -{ - struct i40iw_update_sds_info sdinfo; - - sdinfo.cnt = 1; - sdinfo.hmc_fn_id = hmc_fn_id; - if (setsd) - i40iw_set_sd_entry(pa, sd_idx, type, sdinfo.entry); - else - i40iw_clr_sd_entry(sd_idx, type, sdinfo.entry); - - return dev->cqp->process_cqp_sds(dev, &sdinfo); -} - -/** - * i40iw_hmc_sd_grp - setup group od sd entries for cqp - * @dev: pointer to the device structure - * @hmc_info: pointer to the HMC configuration information struct - * @sd_index: sd index - * @sd_cnt: number of sd entries - * @setsd: flag to set or clear sd - */ -static enum i40iw_status_code i40iw_hmc_sd_grp(struct i40iw_sc_dev *dev, - struct i40iw_hmc_info *hmc_info, - u32 sd_index, - u32 sd_cnt, - bool setsd) -{ - struct i40iw_hmc_sd_entry *sd_entry; - struct i40iw_update_sds_info sdinfo; - u64 pa; - u32 i; - enum i40iw_status_code ret_code = 0; - - memset(&sdinfo, 0, sizeof(sdinfo)); - sdinfo.hmc_fn_id = hmc_info->hmc_fn_id; - for (i = sd_index; i < sd_index + sd_cnt; i++) { - sd_entry = &hmc_info->sd_table.sd_entry[i]; - if (!sd_entry || - (!sd_entry->valid && setsd) || - (sd_entry->valid && !setsd)) - continue; - if (setsd) { - pa = (sd_entry->entry_type == I40IW_SD_TYPE_PAGED) ? - sd_entry->u.pd_table.pd_page_addr.pa : - sd_entry->u.bp.addr.pa; - i40iw_set_sd_entry(pa, i, sd_entry->entry_type, - &sdinfo.entry[sdinfo.cnt]); - } else { - i40iw_clr_sd_entry(i, sd_entry->entry_type, - &sdinfo.entry[sdinfo.cnt]); - } - sdinfo.cnt++; - if (sdinfo.cnt == I40IW_MAX_SD_ENTRIES) { - ret_code = dev->cqp->process_cqp_sds(dev, &sdinfo); - if (ret_code) { - i40iw_debug(dev, I40IW_DEBUG_HMC, - "i40iw_hmc_sd_grp: sd_programming failed err=%d\n", - ret_code); - return ret_code; - } - sdinfo.cnt = 0; - } - } - if (sdinfo.cnt) - ret_code = dev->cqp->process_cqp_sds(dev, &sdinfo); - - return ret_code; -} - -/** - * i40iw_vfdev_from_fpm - return vf dev ptr for hmc function id - * @dev: pointer to the device structure - * @hmc_fn_id: hmc's function id - */ -struct i40iw_vfdev *i40iw_vfdev_from_fpm(struct i40iw_sc_dev *dev, u8 hmc_fn_id) -{ - struct i40iw_vfdev *vf_dev = NULL; - u16 idx; - - for (idx = 0; idx < I40IW_MAX_PE_ENABLED_VF_COUNT; idx++) { - if (dev->vf_dev[idx] && - ((u8)dev->vf_dev[idx]->pmf_index == hmc_fn_id)) { - vf_dev = dev->vf_dev[idx]; - break; - } - } - return vf_dev; -} - -/** - * i40iw_vf_hmcinfo_from_fpm - get ptr to hmc for func_id - * @dev: pointer to the device structure - * @hmc_fn_id: hmc's function id - */ -struct i40iw_hmc_info *i40iw_vf_hmcinfo_from_fpm(struct i40iw_sc_dev *dev, - u8 hmc_fn_id) -{ - struct i40iw_hmc_info *hmc_info = NULL; - u16 idx; - - for (idx = 0; idx < I40IW_MAX_PE_ENABLED_VF_COUNT; idx++) { - if (dev->vf_dev[idx] && - ((u8)dev->vf_dev[idx]->pmf_index == hmc_fn_id)) { - hmc_info = &dev->vf_dev[idx]->hmc_info; - break; - } - } - return hmc_info; -} - -/** - * i40iw_hmc_finish_add_sd_reg - program sd entries for objects - * @dev: pointer to the device structure - * @info: create obj info - */ -static enum i40iw_status_code i40iw_hmc_finish_add_sd_reg(struct i40iw_sc_dev *dev, - struct i40iw_hmc_create_obj_info *info) -{ - if (info->start_idx >= info->hmc_info->hmc_obj[info->rsrc_type].cnt) - return I40IW_ERR_INVALID_HMC_OBJ_INDEX; - - if ((info->start_idx + info->count) > - info->hmc_info->hmc_obj[info->rsrc_type].cnt) - return I40IW_ERR_INVALID_HMC_OBJ_COUNT; - - if (!info->add_sd_cnt) - return 0; - - return i40iw_hmc_sd_grp(dev, info->hmc_info, - info->hmc_info->sd_indexes[0], - info->add_sd_cnt, true); -} - -/** - * i40iw_sc_create_hmc_obj - allocate backing store for hmc objects - * @dev: pointer to the device structure - * @info: pointer to i40iw_hmc_iw_create_obj_info struct - * - * This will allocate memory for PDs and backing pages and populate - * the sd and pd entries. - */ -enum i40iw_status_code i40iw_sc_create_hmc_obj(struct i40iw_sc_dev *dev, - struct i40iw_hmc_create_obj_info *info) -{ - struct i40iw_hmc_sd_entry *sd_entry; - u32 sd_idx, sd_lmt; - u32 pd_idx = 0, pd_lmt = 0; - u32 pd_idx1 = 0, pd_lmt1 = 0; - u32 i, j; - bool pd_error = false; - enum i40iw_status_code ret_code = 0; - - if (info->start_idx >= info->hmc_info->hmc_obj[info->rsrc_type].cnt) - return I40IW_ERR_INVALID_HMC_OBJ_INDEX; - - if ((info->start_idx + info->count) > - info->hmc_info->hmc_obj[info->rsrc_type].cnt) { - i40iw_debug(dev, I40IW_DEBUG_HMC, - "%s: error type %u, start = %u, req cnt %u, cnt = %u\n", - __func__, info->rsrc_type, info->start_idx, info->count, - info->hmc_info->hmc_obj[info->rsrc_type].cnt); - return I40IW_ERR_INVALID_HMC_OBJ_COUNT; - } - - if (!dev->is_pf) - return i40iw_vchnl_vf_add_hmc_objs(dev, info->rsrc_type, 0, info->count); - - i40iw_find_sd_index_limit(info->hmc_info, info->rsrc_type, - info->start_idx, info->count, - &sd_idx, &sd_lmt); - if (sd_idx >= info->hmc_info->sd_table.sd_cnt || - sd_lmt > info->hmc_info->sd_table.sd_cnt) { - return I40IW_ERR_INVALID_SD_INDEX; - } - i40iw_find_pd_index_limit(info->hmc_info, info->rsrc_type, - info->start_idx, info->count, &pd_idx, &pd_lmt); - - for (j = sd_idx; j < sd_lmt; j++) { - ret_code = i40iw_add_sd_table_entry(dev->hw, info->hmc_info, - j, - info->entry_type, - I40IW_HMC_DIRECT_BP_SIZE); - if (ret_code) - goto exit_sd_error; - sd_entry = &info->hmc_info->sd_table.sd_entry[j]; - - if ((sd_entry->entry_type == I40IW_SD_TYPE_PAGED) && - ((dev->hmc_info == info->hmc_info) && - (info->rsrc_type != I40IW_HMC_IW_PBLE))) { - pd_idx1 = max(pd_idx, (j * I40IW_HMC_MAX_BP_COUNT)); - pd_lmt1 = min(pd_lmt, - (j + 1) * I40IW_HMC_MAX_BP_COUNT); - for (i = pd_idx1; i < pd_lmt1; i++) { - /* update the pd table entry */ - ret_code = i40iw_add_pd_table_entry(dev->hw, info->hmc_info, - i, NULL); - if (ret_code) { - pd_error = true; - break; - } - } - if (pd_error) { - while (i && (i > pd_idx1)) { - i40iw_remove_pd_bp(dev->hw, info->hmc_info, (i - 1), - info->is_pf); - i--; - } - } - } - if (sd_entry->valid) - continue; - - info->hmc_info->sd_indexes[info->add_sd_cnt] = (u16)j; - info->add_sd_cnt++; - sd_entry->valid = true; - } - return i40iw_hmc_finish_add_sd_reg(dev, info); - -exit_sd_error: - while (j && (j > sd_idx)) { - sd_entry = &info->hmc_info->sd_table.sd_entry[j - 1]; - switch (sd_entry->entry_type) { - case I40IW_SD_TYPE_PAGED: - pd_idx1 = max(pd_idx, - (j - 1) * I40IW_HMC_MAX_BP_COUNT); - pd_lmt1 = min(pd_lmt, (j * I40IW_HMC_MAX_BP_COUNT)); - for (i = pd_idx1; i < pd_lmt1; i++) - i40iw_prep_remove_pd_page(info->hmc_info, i); - break; - case I40IW_SD_TYPE_DIRECT: - i40iw_prep_remove_pd_page(info->hmc_info, (j - 1)); - break; - default: - ret_code = I40IW_ERR_INVALID_SD_TYPE; - break; - } - j--; - } - - return ret_code; -} - -/** - * i40iw_finish_del_sd_reg - delete sd entries for objects - * @dev: pointer to the device structure - * @info: dele obj info - * @reset: true if called before reset - */ -static enum i40iw_status_code i40iw_finish_del_sd_reg(struct i40iw_sc_dev *dev, - struct i40iw_hmc_del_obj_info *info, - bool reset) -{ - struct i40iw_hmc_sd_entry *sd_entry; - enum i40iw_status_code ret_code = 0; - u32 i, sd_idx; - struct i40iw_dma_mem *mem; - - if (dev->is_pf && !reset) - ret_code = i40iw_hmc_sd_grp(dev, info->hmc_info, - info->hmc_info->sd_indexes[0], - info->del_sd_cnt, false); - - if (ret_code) - i40iw_debug(dev, I40IW_DEBUG_HMC, "%s: error cqp sd sd_grp\n", __func__); - - for (i = 0; i < info->del_sd_cnt; i++) { - sd_idx = info->hmc_info->sd_indexes[i]; - sd_entry = &info->hmc_info->sd_table.sd_entry[sd_idx]; - if (!sd_entry) - continue; - mem = (sd_entry->entry_type == I40IW_SD_TYPE_PAGED) ? - &sd_entry->u.pd_table.pd_page_addr : - &sd_entry->u.bp.addr; - - if (!mem || !mem->va) - i40iw_debug(dev, I40IW_DEBUG_HMC, "%s: error cqp sd mem\n", __func__); - else - i40iw_free_dma_mem(dev->hw, mem); - } - return ret_code; -} - -/** - * i40iw_sc_del_hmc_obj - remove pe hmc objects - * @dev: pointer to the device structure - * @info: pointer to i40iw_hmc_del_obj_info struct - * @reset: true if called before reset - * - * This will de-populate the SDs and PDs. It frees - * the memory for PDS and backing storage. After this function is returned, - * caller should deallocate memory allocated previously for - * book-keeping information about PDs and backing storage. - */ -enum i40iw_status_code i40iw_sc_del_hmc_obj(struct i40iw_sc_dev *dev, - struct i40iw_hmc_del_obj_info *info, - bool reset) -{ - struct i40iw_hmc_pd_table *pd_table; - u32 sd_idx, sd_lmt; - u32 pd_idx, pd_lmt, rel_pd_idx; - u32 i, j; - enum i40iw_status_code ret_code = 0; - - if (info->start_idx >= info->hmc_info->hmc_obj[info->rsrc_type].cnt) { - i40iw_debug(dev, I40IW_DEBUG_HMC, - "%s: error start_idx[%04d] >= [type %04d].cnt[%04d]\n", - __func__, info->start_idx, info->rsrc_type, - info->hmc_info->hmc_obj[info->rsrc_type].cnt); - return I40IW_ERR_INVALID_HMC_OBJ_INDEX; - } - - if ((info->start_idx + info->count) > - info->hmc_info->hmc_obj[info->rsrc_type].cnt) { - i40iw_debug(dev, I40IW_DEBUG_HMC, - "%s: error start_idx[%04d] + count %04d >= [type %04d].cnt[%04d]\n", - __func__, info->start_idx, info->count, - info->rsrc_type, - info->hmc_info->hmc_obj[info->rsrc_type].cnt); - return I40IW_ERR_INVALID_HMC_OBJ_COUNT; - } - if (!dev->is_pf) { - ret_code = i40iw_vchnl_vf_del_hmc_obj(dev, info->rsrc_type, 0, - info->count); - if (info->rsrc_type != I40IW_HMC_IW_PBLE) - return ret_code; - } - - i40iw_find_pd_index_limit(info->hmc_info, info->rsrc_type, - info->start_idx, info->count, &pd_idx, &pd_lmt); - - for (j = pd_idx; j < pd_lmt; j++) { - sd_idx = j / I40IW_HMC_PD_CNT_IN_SD; - - if (info->hmc_info->sd_table.sd_entry[sd_idx].entry_type != - I40IW_SD_TYPE_PAGED) - continue; - - rel_pd_idx = j % I40IW_HMC_PD_CNT_IN_SD; - pd_table = &info->hmc_info->sd_table.sd_entry[sd_idx].u.pd_table; - if (pd_table->pd_entry[rel_pd_idx].valid) { - ret_code = i40iw_remove_pd_bp(dev->hw, info->hmc_info, j, - info->is_pf); - if (ret_code) { - i40iw_debug(dev, I40IW_DEBUG_HMC, "%s: error\n", __func__); - return ret_code; - } - } - } - - i40iw_find_sd_index_limit(info->hmc_info, info->rsrc_type, - info->start_idx, info->count, &sd_idx, &sd_lmt); - if (sd_idx >= info->hmc_info->sd_table.sd_cnt || - sd_lmt > info->hmc_info->sd_table.sd_cnt) { - i40iw_debug(dev, I40IW_DEBUG_HMC, "%s: error invalid sd_idx\n", __func__); - return I40IW_ERR_INVALID_SD_INDEX; - } - - for (i = sd_idx; i < sd_lmt; i++) { - if (!info->hmc_info->sd_table.sd_entry[i].valid) - continue; - switch (info->hmc_info->sd_table.sd_entry[i].entry_type) { - case I40IW_SD_TYPE_DIRECT: - ret_code = i40iw_prep_remove_sd_bp(info->hmc_info, i); - if (!ret_code) { - info->hmc_info->sd_indexes[info->del_sd_cnt] = (u16)i; - info->del_sd_cnt++; - } - break; - case I40IW_SD_TYPE_PAGED: - ret_code = i40iw_prep_remove_pd_page(info->hmc_info, i); - if (!ret_code) { - info->hmc_info->sd_indexes[info->del_sd_cnt] = (u16)i; - info->del_sd_cnt++; - } - break; - default: - break; - } - } - return i40iw_finish_del_sd_reg(dev, info, reset); -} - -/** - * i40iw_add_sd_table_entry - Adds a segment descriptor to the table - * @hw: pointer to our hw struct - * @hmc_info: pointer to the HMC configuration information struct - * @sd_index: segment descriptor index to manipulate - * @type: what type of segment descriptor we're manipulating - * @direct_mode_sz: size to alloc in direct mode - */ -enum i40iw_status_code i40iw_add_sd_table_entry(struct i40iw_hw *hw, - struct i40iw_hmc_info *hmc_info, - u32 sd_index, - enum i40iw_sd_entry_type type, - u64 direct_mode_sz) -{ - enum i40iw_status_code ret_code = 0; - struct i40iw_hmc_sd_entry *sd_entry; - bool dma_mem_alloc_done = false; - struct i40iw_dma_mem mem; - u64 alloc_len; - - sd_entry = &hmc_info->sd_table.sd_entry[sd_index]; - if (!sd_entry->valid) { - if (type == I40IW_SD_TYPE_PAGED) - alloc_len = I40IW_HMC_PAGED_BP_SIZE; - else - alloc_len = direct_mode_sz; - - /* allocate a 4K pd page or 2M backing page */ - ret_code = i40iw_allocate_dma_mem(hw, &mem, alloc_len, - I40IW_HMC_PD_BP_BUF_ALIGNMENT); - if (ret_code) - goto exit; - dma_mem_alloc_done = true; - if (type == I40IW_SD_TYPE_PAGED) { - ret_code = i40iw_allocate_virt_mem(hw, - &sd_entry->u.pd_table.pd_entry_virt_mem, - sizeof(struct i40iw_hmc_pd_entry) * 512); - if (ret_code) - goto exit; - sd_entry->u.pd_table.pd_entry = (struct i40iw_hmc_pd_entry *) - sd_entry->u.pd_table.pd_entry_virt_mem.va; - - memcpy(&sd_entry->u.pd_table.pd_page_addr, &mem, sizeof(struct i40iw_dma_mem)); - } else { - memcpy(&sd_entry->u.bp.addr, &mem, sizeof(struct i40iw_dma_mem)); - sd_entry->u.bp.sd_pd_index = sd_index; - } - - hmc_info->sd_table.sd_entry[sd_index].entry_type = type; - - I40IW_INC_SD_REFCNT(&hmc_info->sd_table); - } - if (sd_entry->entry_type == I40IW_SD_TYPE_DIRECT) - I40IW_INC_BP_REFCNT(&sd_entry->u.bp); -exit: - if (ret_code) - if (dma_mem_alloc_done) - i40iw_free_dma_mem(hw, &mem); - - return ret_code; -} - -/** - * i40iw_add_pd_table_entry - Adds page descriptor to the specified table - * @hw: pointer to our HW structure - * @hmc_info: pointer to the HMC configuration information structure - * @pd_index: which page descriptor index to manipulate - * @rsrc_pg: if not NULL, use preallocated page instead of allocating new one. - * - * This function: - * 1. Initializes the pd entry - * 2. Adds pd_entry in the pd_table - * 3. Mark the entry valid in i40iw_hmc_pd_entry structure - * 4. Initializes the pd_entry's ref count to 1 - * assumptions: - * 1. The memory for pd should be pinned down, physically contiguous and - * aligned on 4K boundary and zeroed memory. - * 2. It should be 4K in size. - */ -enum i40iw_status_code i40iw_add_pd_table_entry(struct i40iw_hw *hw, - struct i40iw_hmc_info *hmc_info, - u32 pd_index, - struct i40iw_dma_mem *rsrc_pg) -{ - enum i40iw_status_code ret_code = 0; - struct i40iw_hmc_pd_table *pd_table; - struct i40iw_hmc_pd_entry *pd_entry; - struct i40iw_dma_mem mem; - struct i40iw_dma_mem *page = &mem; - u32 sd_idx, rel_pd_idx; - u64 *pd_addr; - u64 page_desc; - - if (pd_index / I40IW_HMC_PD_CNT_IN_SD >= hmc_info->sd_table.sd_cnt) - return I40IW_ERR_INVALID_PAGE_DESC_INDEX; - - sd_idx = (pd_index / I40IW_HMC_PD_CNT_IN_SD); - if (hmc_info->sd_table.sd_entry[sd_idx].entry_type != I40IW_SD_TYPE_PAGED) - return 0; - - rel_pd_idx = (pd_index % I40IW_HMC_PD_CNT_IN_SD); - pd_table = &hmc_info->sd_table.sd_entry[sd_idx].u.pd_table; - pd_entry = &pd_table->pd_entry[rel_pd_idx]; - if (!pd_entry->valid) { - if (rsrc_pg) { - pd_entry->rsrc_pg = true; - page = rsrc_pg; - } else { - ret_code = i40iw_allocate_dma_mem(hw, page, - I40IW_HMC_PAGED_BP_SIZE, - I40IW_HMC_PD_BP_BUF_ALIGNMENT); - if (ret_code) - return ret_code; - pd_entry->rsrc_pg = false; - } - - memcpy(&pd_entry->bp.addr, page, sizeof(struct i40iw_dma_mem)); - pd_entry->bp.sd_pd_index = pd_index; - pd_entry->bp.entry_type = I40IW_SD_TYPE_PAGED; - page_desc = page->pa | 0x1; - - pd_addr = (u64 *)pd_table->pd_page_addr.va; - pd_addr += rel_pd_idx; - - memcpy(pd_addr, &page_desc, sizeof(*pd_addr)); - - pd_entry->sd_index = sd_idx; - pd_entry->valid = true; - I40IW_INC_PD_REFCNT(pd_table); - if (hmc_info->hmc_fn_id < I40IW_FIRST_VF_FPM_ID) - I40IW_INVALIDATE_PF_HMC_PD(hw, sd_idx, rel_pd_idx); - else if (hw->hmc.hmc_fn_id != hmc_info->hmc_fn_id) - I40IW_INVALIDATE_VF_HMC_PD(hw, sd_idx, rel_pd_idx, - hmc_info->hmc_fn_id); - } - I40IW_INC_BP_REFCNT(&pd_entry->bp); - - return 0; -} - -/** - * i40iw_remove_pd_bp - remove a backing page from a page descriptor - * @hw: pointer to our HW structure - * @hmc_info: pointer to the HMC configuration information structure - * @idx: the page index - * @is_pf: distinguishes a VF from a PF - * - * This function: - * 1. Marks the entry in pd table (for paged address mode) or in sd table - * (for direct address mode) invalid. - * 2. Write to register PMPDINV to invalidate the backing page in FV cache - * 3. Decrement the ref count for the pd _entry - * assumptions: - * 1. Caller can deallocate the memory used by backing storage after this - * function returns. - */ -enum i40iw_status_code i40iw_remove_pd_bp(struct i40iw_hw *hw, - struct i40iw_hmc_info *hmc_info, - u32 idx, - bool is_pf) -{ - struct i40iw_hmc_pd_entry *pd_entry; - struct i40iw_hmc_pd_table *pd_table; - struct i40iw_hmc_sd_entry *sd_entry; - u32 sd_idx, rel_pd_idx; - struct i40iw_dma_mem *mem; - u64 *pd_addr; - - sd_idx = idx / I40IW_HMC_PD_CNT_IN_SD; - rel_pd_idx = idx % I40IW_HMC_PD_CNT_IN_SD; - if (sd_idx >= hmc_info->sd_table.sd_cnt) - return I40IW_ERR_INVALID_PAGE_DESC_INDEX; - - sd_entry = &hmc_info->sd_table.sd_entry[sd_idx]; - if (sd_entry->entry_type != I40IW_SD_TYPE_PAGED) - return I40IW_ERR_INVALID_SD_TYPE; - - pd_table = &hmc_info->sd_table.sd_entry[sd_idx].u.pd_table; - pd_entry = &pd_table->pd_entry[rel_pd_idx]; - I40IW_DEC_BP_REFCNT(&pd_entry->bp); - if (pd_entry->bp.ref_cnt) - return 0; - - pd_entry->valid = false; - I40IW_DEC_PD_REFCNT(pd_table); - pd_addr = (u64 *)pd_table->pd_page_addr.va; - pd_addr += rel_pd_idx; - memset(pd_addr, 0, sizeof(u64)); - if (is_pf) - I40IW_INVALIDATE_PF_HMC_PD(hw, sd_idx, idx); - else - I40IW_INVALIDATE_VF_HMC_PD(hw, sd_idx, idx, - hmc_info->hmc_fn_id); - - if (!pd_entry->rsrc_pg) { - mem = &pd_entry->bp.addr; - if (!mem || !mem->va) - return I40IW_ERR_PARAM; - i40iw_free_dma_mem(hw, mem); - } - if (!pd_table->ref_cnt) - i40iw_free_virt_mem(hw, &pd_table->pd_entry_virt_mem); - - return 0; -} - -/** - * i40iw_prep_remove_sd_bp - Prepares to remove a backing page from a sd entry - * @hmc_info: pointer to the HMC configuration information structure - * @idx: the page index - */ -enum i40iw_status_code i40iw_prep_remove_sd_bp(struct i40iw_hmc_info *hmc_info, u32 idx) -{ - struct i40iw_hmc_sd_entry *sd_entry; - - sd_entry = &hmc_info->sd_table.sd_entry[idx]; - I40IW_DEC_BP_REFCNT(&sd_entry->u.bp); - if (sd_entry->u.bp.ref_cnt) - return I40IW_ERR_NOT_READY; - - I40IW_DEC_SD_REFCNT(&hmc_info->sd_table); - sd_entry->valid = false; - - return 0; -} - -/** - * i40iw_prep_remove_pd_page - Prepares to remove a PD page from sd entry. - * @hmc_info: pointer to the HMC configuration information structure - * @idx: segment descriptor index to find the relevant page descriptor - */ -enum i40iw_status_code i40iw_prep_remove_pd_page(struct i40iw_hmc_info *hmc_info, - u32 idx) -{ - struct i40iw_hmc_sd_entry *sd_entry; - - sd_entry = &hmc_info->sd_table.sd_entry[idx]; - - if (sd_entry->u.pd_table.ref_cnt) - return I40IW_ERR_NOT_READY; - - sd_entry->valid = false; - I40IW_DEC_SD_REFCNT(&hmc_info->sd_table); - - return 0; -} - -/** - * i40iw_pf_init_vfhmc - - * @vf_cnt_array: array of cnt values of iwarp hmc objects - * @vf_hmc_fn_id: hmc function id ofr vf driver - * @dev: pointer to i40iw_dev struct - * - * Called by pf driver to initialize hmc_info for vf driver instance. - */ -enum i40iw_status_code i40iw_pf_init_vfhmc(struct i40iw_sc_dev *dev, - u8 vf_hmc_fn_id, - u32 *vf_cnt_array) -{ - struct i40iw_hmc_info *hmc_info; - enum i40iw_status_code ret_code = 0; - u32 i; - - if ((vf_hmc_fn_id < I40IW_FIRST_VF_FPM_ID) || - (vf_hmc_fn_id >= I40IW_FIRST_VF_FPM_ID + - I40IW_MAX_PE_ENABLED_VF_COUNT)) { - i40iw_debug(dev, I40IW_DEBUG_HMC, "%s: invalid vf_hmc_fn_id 0x%x\n", - __func__, vf_hmc_fn_id); - return I40IW_ERR_INVALID_HMCFN_ID; - } - - ret_code = i40iw_sc_init_iw_hmc(dev, vf_hmc_fn_id); - if (ret_code) - return ret_code; - - hmc_info = i40iw_vf_hmcinfo_from_fpm(dev, vf_hmc_fn_id); - - for (i = I40IW_HMC_IW_QP; i < I40IW_HMC_IW_MAX; i++) - if (vf_cnt_array) - hmc_info->hmc_obj[i].cnt = - vf_cnt_array[i - I40IW_HMC_IW_QP]; - else - hmc_info->hmc_obj[i].cnt = hmc_info->hmc_obj[i].max_cnt; - - return 0; -} diff --git a/drivers/infiniband/hw/i40iw/i40iw_hmc.h b/drivers/infiniband/hw/i40iw/i40iw_hmc.h deleted file mode 100644 index 4c3fdd875621..000000000000 --- a/drivers/infiniband/hw/i40iw/i40iw_hmc.h +++ /dev/null @@ -1,241 +0,0 @@ -/******************************************************************************* -* -* Copyright (c) 2015-2016 Intel Corporation. All rights reserved. -* -* This software is available to you under a choice of one of two -* licenses. You may choose to be licensed under the terms of the GNU -* General Public License (GPL) Version 2, available from the file -* COPYING in the main directory of this source tree, or the -* OpenFabrics.org BSD license below: -* -* Redistribution and use in source and binary forms, with or -* without modification, are permitted provided that the following -* conditions are met: -* -* - Redistributions of source code must retain the above -* copyright notice, this list of conditions and the following -* disclaimer. -* -* - Redistributions in binary form must reproduce the above -* copyright notice, this list of conditions and the following -* disclaimer in the documentation and/or other materials -* provided with the distribution. -* -* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS -* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN -* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -* SOFTWARE. -* -*******************************************************************************/ - -#ifndef I40IW_HMC_H -#define I40IW_HMC_H - -#include "i40iw_d.h" - -struct i40iw_hw; -enum i40iw_status_code; - -#define I40IW_HMC_MAX_BP_COUNT 512 -#define I40IW_MAX_SD_ENTRIES 11 -#define I40IW_HW_DBG_HMC_INVALID_BP_MARK 0xCA - -#define I40IW_HMC_INFO_SIGNATURE 0x484D5347 -#define I40IW_HMC_PD_CNT_IN_SD 512 -#define I40IW_HMC_DIRECT_BP_SIZE 0x200000 -#define I40IW_HMC_MAX_SD_COUNT 4096 -#define I40IW_HMC_PAGED_BP_SIZE 4096 -#define I40IW_HMC_PD_BP_BUF_ALIGNMENT 4096 -#define I40IW_FIRST_VF_FPM_ID 16 -#define FPM_MULTIPLIER 1024 - -#define I40IW_INC_SD_REFCNT(sd_table) ((sd_table)->ref_cnt++) -#define I40IW_INC_PD_REFCNT(pd_table) ((pd_table)->ref_cnt++) -#define I40IW_INC_BP_REFCNT(bp) ((bp)->ref_cnt++) - -#define I40IW_DEC_SD_REFCNT(sd_table) ((sd_table)->ref_cnt--) -#define I40IW_DEC_PD_REFCNT(pd_table) ((pd_table)->ref_cnt--) -#define I40IW_DEC_BP_REFCNT(bp) ((bp)->ref_cnt--) - -/** - * I40IW_INVALIDATE_PF_HMC_PD - Invalidates the pd cache in the hardware - * @hw: pointer to our hw struct - * @sd_idx: segment descriptor index - * @pd_idx: page descriptor index - */ -#define I40IW_INVALIDATE_PF_HMC_PD(hw, sd_idx, pd_idx) \ - i40iw_wr32((hw), I40E_PFHMC_PDINV, \ - (((sd_idx) << I40E_PFHMC_PDINV_PMSDIDX_SHIFT) | \ - (0x1 << I40E_PFHMC_PDINV_PMSDPARTSEL_SHIFT) | \ - ((pd_idx) << I40E_PFHMC_PDINV_PMPDIDX_SHIFT))) - -/** - * I40IW_INVALIDATE_VF_HMC_PD - Invalidates the pd cache in the hardware - * @hw: pointer to our hw struct - * @sd_idx: segment descriptor index - * @pd_idx: page descriptor index - * @hmc_fn_id: VF's function id - */ -#define I40IW_INVALIDATE_VF_HMC_PD(hw, sd_idx, pd_idx, hmc_fn_id) \ - i40iw_wr32(hw, I40E_GLHMC_VFPDINV(hmc_fn_id - I40IW_FIRST_VF_FPM_ID), \ - ((sd_idx << I40E_PFHMC_PDINV_PMSDIDX_SHIFT) | \ - (pd_idx << I40E_PFHMC_PDINV_PMPDIDX_SHIFT))) - -struct i40iw_hmc_obj_info { - u64 base; - u32 max_cnt; - u32 cnt; - u64 size; -}; - -enum i40iw_sd_entry_type { - I40IW_SD_TYPE_INVALID = 0, - I40IW_SD_TYPE_PAGED = 1, - I40IW_SD_TYPE_DIRECT = 2 -}; - -struct i40iw_hmc_bp { - enum i40iw_sd_entry_type entry_type; - struct i40iw_dma_mem addr; - u32 sd_pd_index; - u32 ref_cnt; -}; - -struct i40iw_hmc_pd_entry { - struct i40iw_hmc_bp bp; - u32 sd_index; - bool rsrc_pg; - bool valid; -}; - -struct i40iw_hmc_pd_table { - struct i40iw_dma_mem pd_page_addr; - struct i40iw_hmc_pd_entry *pd_entry; - struct i40iw_virt_mem pd_entry_virt_mem; - u32 ref_cnt; - u32 sd_index; -}; - -struct i40iw_hmc_sd_entry { - enum i40iw_sd_entry_type entry_type; - bool valid; - - union { - struct i40iw_hmc_pd_table pd_table; - struct i40iw_hmc_bp bp; - } u; -}; - -struct i40iw_hmc_sd_table { - struct i40iw_virt_mem addr; - u32 sd_cnt; - u32 ref_cnt; - struct i40iw_hmc_sd_entry *sd_entry; -}; - -struct i40iw_hmc_info { - u32 signature; - u8 hmc_fn_id; - u16 first_sd_index; - - struct i40iw_hmc_obj_info *hmc_obj; - struct i40iw_virt_mem hmc_obj_virt_mem; - struct i40iw_hmc_sd_table sd_table; - u16 sd_indexes[I40IW_HMC_MAX_SD_COUNT]; -}; - -struct update_sd_entry { - u64 cmd; - u64 data; -}; - -struct i40iw_update_sds_info { - u32 cnt; - u8 hmc_fn_id; - struct update_sd_entry entry[I40IW_MAX_SD_ENTRIES]; -}; - -struct i40iw_ccq_cqe_info; -struct i40iw_hmc_fcn_info { - void (*callback_fcn)(struct i40iw_sc_dev *, void *, - struct i40iw_ccq_cqe_info *); - void *cqp_callback_param; - u32 vf_id; - u16 iw_vf_idx; - bool free_fcn; -}; - -enum i40iw_hmc_rsrc_type { - I40IW_HMC_IW_QP = 0, - I40IW_HMC_IW_CQ = 1, - I40IW_HMC_IW_SRQ = 2, - I40IW_HMC_IW_HTE = 3, - I40IW_HMC_IW_ARP = 4, - I40IW_HMC_IW_APBVT_ENTRY = 5, - I40IW_HMC_IW_MR = 6, - I40IW_HMC_IW_XF = 7, - I40IW_HMC_IW_XFFL = 8, - I40IW_HMC_IW_Q1 = 9, - I40IW_HMC_IW_Q1FL = 10, - I40IW_HMC_IW_TIMER = 11, - I40IW_HMC_IW_FSIMC = 12, - I40IW_HMC_IW_FSIAV = 13, - I40IW_HMC_IW_PBLE = 14, - I40IW_HMC_IW_MAX = 15, -}; - -struct i40iw_hmc_create_obj_info { - struct i40iw_hmc_info *hmc_info; - struct i40iw_virt_mem add_sd_virt_mem; - u32 rsrc_type; - u32 start_idx; - u32 count; - u32 add_sd_cnt; - enum i40iw_sd_entry_type entry_type; - bool is_pf; -}; - -struct i40iw_hmc_del_obj_info { - struct i40iw_hmc_info *hmc_info; - struct i40iw_virt_mem del_sd_virt_mem; - u32 rsrc_type; - u32 start_idx; - u32 count; - u32 del_sd_cnt; - bool is_pf; -}; - -enum i40iw_status_code i40iw_copy_dma_mem(struct i40iw_hw *hw, void *dest_buf, - struct i40iw_dma_mem *src_mem, u64 src_offset, u64 size); -enum i40iw_status_code i40iw_sc_create_hmc_obj(struct i40iw_sc_dev *dev, - struct i40iw_hmc_create_obj_info *info); -enum i40iw_status_code i40iw_sc_del_hmc_obj(struct i40iw_sc_dev *dev, - struct i40iw_hmc_del_obj_info *info, - bool reset); -enum i40iw_status_code i40iw_hmc_sd_one(struct i40iw_sc_dev *dev, u8 hmc_fn_id, - u64 pa, u32 sd_idx, enum i40iw_sd_entry_type type, - bool setsd); -enum i40iw_status_code i40iw_update_sds_noccq(struct i40iw_sc_dev *dev, - struct i40iw_update_sds_info *info); -struct i40iw_vfdev *i40iw_vfdev_from_fpm(struct i40iw_sc_dev *dev, u8 hmc_fn_id); -struct i40iw_hmc_info *i40iw_vf_hmcinfo_from_fpm(struct i40iw_sc_dev *dev, - u8 hmc_fn_id); -enum i40iw_status_code i40iw_add_sd_table_entry(struct i40iw_hw *hw, - struct i40iw_hmc_info *hmc_info, u32 sd_index, - enum i40iw_sd_entry_type type, u64 direct_mode_sz); -enum i40iw_status_code i40iw_add_pd_table_entry(struct i40iw_hw *hw, - struct i40iw_hmc_info *hmc_info, u32 pd_index, - struct i40iw_dma_mem *rsrc_pg); -enum i40iw_status_code i40iw_remove_pd_bp(struct i40iw_hw *hw, - struct i40iw_hmc_info *hmc_info, u32 idx, bool is_pf); -enum i40iw_status_code i40iw_prep_remove_sd_bp(struct i40iw_hmc_info *hmc_info, u32 idx); -enum i40iw_status_code i40iw_prep_remove_pd_page(struct i40iw_hmc_info *hmc_info, u32 idx); - -#define ENTER_SHARED_FUNCTION() -#define EXIT_SHARED_FUNCTION() - -#endif /* I40IW_HMC_H */ diff --git a/drivers/infiniband/hw/i40iw/i40iw_hw.c b/drivers/infiniband/hw/i40iw/i40iw_hw.c deleted file mode 100644 index d167ac10c751..000000000000 --- a/drivers/infiniband/hw/i40iw/i40iw_hw.c +++ /dev/null @@ -1,851 +0,0 @@ -/******************************************************************************* -* -* Copyright (c) 2015-2016 Intel Corporation. All rights reserved. -* -* This software is available to you under a choice of one of two -* licenses. You may choose to be licensed under the terms of the GNU -* General Public License (GPL) Version 2, available from the file -* COPYING in the main directory of this source tree, or the -* OpenFabrics.org BSD license below: -* -* Redistribution and use in source and binary forms, with or -* without modification, are permitted provided that the following -* conditions are met: -* -* - Redistributions of source code must retain the above -* copyright notice, this list of conditions and the following -* disclaimer. -* -* - Redistributions in binary form must reproduce the above -* copyright notice, this list of conditions and the following -* disclaimer in the documentation and/or other materials -* provided with the distribution. -* -* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS -* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN -* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -* SOFTWARE. -* -*******************************************************************************/ - -#include -#include -#include -#include -#include -#include -#include - -#include "i40iw.h" - -/** - * i40iw_initialize_hw_resources - initialize hw resource during open - * @iwdev: iwarp device - */ -u32 i40iw_initialize_hw_resources(struct i40iw_device *iwdev) -{ - unsigned long num_pds; - u32 resources_size; - u32 max_mr; - u32 max_qp; - u32 max_cq; - u32 arp_table_size; - u32 mrdrvbits; - void *resource_ptr; - - max_qp = iwdev->sc_dev.hmc_info->hmc_obj[I40IW_HMC_IW_QP].cnt; - max_cq = iwdev->sc_dev.hmc_info->hmc_obj[I40IW_HMC_IW_CQ].cnt; - max_mr = iwdev->sc_dev.hmc_info->hmc_obj[I40IW_HMC_IW_MR].cnt; - arp_table_size = iwdev->sc_dev.hmc_info->hmc_obj[I40IW_HMC_IW_ARP].cnt; - iwdev->max_cqe = 0xFFFFF; - num_pds = I40IW_MAX_PDS; - resources_size = sizeof(struct i40iw_arp_entry) * arp_table_size; - resources_size += sizeof(unsigned long) * BITS_TO_LONGS(max_qp); - resources_size += sizeof(unsigned long) * BITS_TO_LONGS(max_mr); - resources_size += sizeof(unsigned long) * BITS_TO_LONGS(max_cq); - resources_size += sizeof(unsigned long) * BITS_TO_LONGS(num_pds); - resources_size += sizeof(unsigned long) * BITS_TO_LONGS(arp_table_size); - resources_size += sizeof(struct i40iw_qp **) * max_qp; - iwdev->mem_resources = kzalloc(resources_size, GFP_KERNEL); - - if (!iwdev->mem_resources) - return -ENOMEM; - - iwdev->max_qp = max_qp; - iwdev->max_mr = max_mr; - iwdev->max_cq = max_cq; - iwdev->max_pd = num_pds; - iwdev->arp_table_size = arp_table_size; - iwdev->arp_table = (struct i40iw_arp_entry *)iwdev->mem_resources; - resource_ptr = iwdev->mem_resources + (sizeof(struct i40iw_arp_entry) * arp_table_size); - - iwdev->device_cap_flags = IB_DEVICE_LOCAL_DMA_LKEY | - IB_DEVICE_MEM_WINDOW | IB_DEVICE_MEM_MGT_EXTENSIONS; - - iwdev->allocated_qps = resource_ptr; - iwdev->allocated_cqs = &iwdev->allocated_qps[BITS_TO_LONGS(max_qp)]; - iwdev->allocated_mrs = &iwdev->allocated_cqs[BITS_TO_LONGS(max_cq)]; - iwdev->allocated_pds = &iwdev->allocated_mrs[BITS_TO_LONGS(max_mr)]; - iwdev->allocated_arps = &iwdev->allocated_pds[BITS_TO_LONGS(num_pds)]; - iwdev->qp_table = (struct i40iw_qp **)(&iwdev->allocated_arps[BITS_TO_LONGS(arp_table_size)]); - set_bit(0, iwdev->allocated_mrs); - set_bit(0, iwdev->allocated_qps); - set_bit(0, iwdev->allocated_cqs); - set_bit(0, iwdev->allocated_pds); - set_bit(0, iwdev->allocated_arps); - - /* Following for ILQ/IEQ */ - set_bit(1, iwdev->allocated_qps); - set_bit(1, iwdev->allocated_cqs); - set_bit(1, iwdev->allocated_pds); - set_bit(2, iwdev->allocated_cqs); - set_bit(2, iwdev->allocated_pds); - - spin_lock_init(&iwdev->resource_lock); - spin_lock_init(&iwdev->qptable_lock); - /* stag index mask has a minimum of 14 bits */ - mrdrvbits = 24 - max(get_count_order(iwdev->max_mr), 14); - iwdev->mr_stagmask = ~(((1 << mrdrvbits) - 1) << (32 - mrdrvbits)); - return 0; -} - -/** - * i40iw_cqp_ce_handler - handle cqp completions - * @iwdev: iwarp device - * @arm: flag to arm after completions - * @cq: cq for cqp completions - */ -static void i40iw_cqp_ce_handler(struct i40iw_device *iwdev, struct i40iw_sc_cq *cq, bool arm) -{ - struct i40iw_cqp_request *cqp_request; - struct i40iw_sc_dev *dev = &iwdev->sc_dev; - u32 cqe_count = 0; - struct i40iw_ccq_cqe_info info; - int ret; - - do { - memset(&info, 0, sizeof(info)); - ret = dev->ccq_ops->ccq_get_cqe_info(cq, &info); - if (ret) - break; - cqp_request = (struct i40iw_cqp_request *)(unsigned long)info.scratch; - if (info.error) - i40iw_pr_err("opcode = 0x%x maj_err_code = 0x%x min_err_code = 0x%x\n", - info.op_code, info.maj_err_code, info.min_err_code); - if (cqp_request) { - cqp_request->compl_info.maj_err_code = info.maj_err_code; - cqp_request->compl_info.min_err_code = info.min_err_code; - cqp_request->compl_info.op_ret_val = info.op_ret_val; - cqp_request->compl_info.error = info.error; - - if (cqp_request->waiting) { - cqp_request->request_done = true; - wake_up(&cqp_request->waitq); - i40iw_put_cqp_request(&iwdev->cqp, cqp_request); - } else { - if (cqp_request->callback_fcn) - cqp_request->callback_fcn(cqp_request, 1); - i40iw_put_cqp_request(&iwdev->cqp, cqp_request); - } - } - - cqe_count++; - } while (1); - - if (arm && cqe_count) { - i40iw_process_bh(dev); - dev->ccq_ops->ccq_arm(cq); - } -} - -/** - * i40iw_iwarp_ce_handler - handle iwarp completions - * @iwdev: iwarp device - * @iwcq: iwarp cq receiving event - */ -static void i40iw_iwarp_ce_handler(struct i40iw_device *iwdev, - struct i40iw_sc_cq *iwcq) -{ - struct i40iw_cq *i40iwcq = iwcq->back_cq; - - if (i40iwcq->ibcq.comp_handler) - i40iwcq->ibcq.comp_handler(&i40iwcq->ibcq, - i40iwcq->ibcq.cq_context); -} - -/** - * i40iw_puda_ce_handler - handle puda completion events - * @iwdev: iwarp device - * @cq: puda completion q for event - */ -static void i40iw_puda_ce_handler(struct i40iw_device *iwdev, - struct i40iw_sc_cq *cq) -{ - struct i40iw_sc_dev *dev = (struct i40iw_sc_dev *)&iwdev->sc_dev; - enum i40iw_status_code status; - u32 compl_error; - - do { - status = i40iw_puda_poll_completion(dev, cq, &compl_error); - if (status == I40IW_ERR_QUEUE_EMPTY) - break; - if (status) { - i40iw_pr_err("puda status = %d\n", status); - break; - } - if (compl_error) { - i40iw_pr_err("puda compl_err =0x%x\n", compl_error); - break; - } - } while (1); - - dev->ccq_ops->ccq_arm(cq); -} - -/** - * i40iw_process_ceq - handle ceq for completions - * @iwdev: iwarp device - * @ceq: ceq having cq for completion - */ -void i40iw_process_ceq(struct i40iw_device *iwdev, struct i40iw_ceq *ceq) -{ - struct i40iw_sc_dev *dev = &iwdev->sc_dev; - struct i40iw_sc_ceq *sc_ceq; - struct i40iw_sc_cq *cq; - bool arm = true; - - sc_ceq = &ceq->sc_ceq; - do { - cq = dev->ceq_ops->process_ceq(dev, sc_ceq); - if (!cq) - break; - - if (cq->cq_type == I40IW_CQ_TYPE_CQP) - i40iw_cqp_ce_handler(iwdev, cq, arm); - else if (cq->cq_type == I40IW_CQ_TYPE_IWARP) - i40iw_iwarp_ce_handler(iwdev, cq); - else if ((cq->cq_type == I40IW_CQ_TYPE_ILQ) || - (cq->cq_type == I40IW_CQ_TYPE_IEQ)) - i40iw_puda_ce_handler(iwdev, cq); - } while (1); -} - -/** - * i40iw_next_iw_state - modify qp state - * @iwqp: iwarp qp to modify - * @state: next state for qp - * @del_hash: del hash - * @term: term message - * @termlen: length of term message - */ -void i40iw_next_iw_state(struct i40iw_qp *iwqp, - u8 state, - u8 del_hash, - u8 term, - u8 termlen) -{ - struct i40iw_modify_qp_info info; - - memset(&info, 0, sizeof(info)); - info.next_iwarp_state = state; - info.remove_hash_idx = del_hash; - info.cq_num_valid = true; - info.arp_cache_idx_valid = true; - info.dont_send_term = true; - info.dont_send_fin = true; - info.termlen = termlen; - - if (term & I40IWQP_TERM_SEND_TERM_ONLY) - info.dont_send_term = false; - if (term & I40IWQP_TERM_SEND_FIN_ONLY) - info.dont_send_fin = false; - if (iwqp->sc_qp.term_flags && (state == I40IW_QP_STATE_ERROR)) - info.reset_tcp_conn = true; - iwqp->hw_iwarp_state = state; - i40iw_hw_modify_qp(iwqp->iwdev, iwqp, &info, 0); -} - -/** - * i40iw_process_aeq - handle aeq events - * @iwdev: iwarp device - */ -void i40iw_process_aeq(struct i40iw_device *iwdev) -{ - struct i40iw_sc_dev *dev = &iwdev->sc_dev; - struct i40iw_aeq *aeq = &iwdev->aeq; - struct i40iw_sc_aeq *sc_aeq = &aeq->sc_aeq; - struct i40iw_aeqe_info aeinfo; - struct i40iw_aeqe_info *info = &aeinfo; - int ret; - struct i40iw_qp *iwqp = NULL; - struct i40iw_sc_cq *cq = NULL; - struct i40iw_cq *iwcq = NULL; - struct i40iw_sc_qp *qp = NULL; - struct i40iw_qp_host_ctx_info *ctx_info = NULL; - unsigned long flags; - - u32 aeqcnt = 0; - - if (!sc_aeq->size) - return; - - do { - memset(info, 0, sizeof(*info)); - ret = dev->aeq_ops->get_next_aeqe(sc_aeq, info); - if (ret) - break; - - aeqcnt++; - i40iw_debug(dev, I40IW_DEBUG_AEQ, - "%s ae_id = 0x%x bool qp=%d qp_id = %d\n", - __func__, info->ae_id, info->qp, info->qp_cq_id); - if (info->qp) { - spin_lock_irqsave(&iwdev->qptable_lock, flags); - iwqp = iwdev->qp_table[info->qp_cq_id]; - if (!iwqp) { - spin_unlock_irqrestore(&iwdev->qptable_lock, flags); - i40iw_debug(dev, I40IW_DEBUG_AEQ, - "%s qp_id %d is already freed\n", - __func__, info->qp_cq_id); - continue; - } - i40iw_qp_add_ref(&iwqp->ibqp); - spin_unlock_irqrestore(&iwdev->qptable_lock, flags); - qp = &iwqp->sc_qp; - spin_lock_irqsave(&iwqp->lock, flags); - iwqp->hw_tcp_state = info->tcp_state; - iwqp->hw_iwarp_state = info->iwarp_state; - iwqp->last_aeq = info->ae_id; - spin_unlock_irqrestore(&iwqp->lock, flags); - ctx_info = &iwqp->ctx_info; - ctx_info->err_rq_idx_valid = true; - } else { - if (info->ae_id != I40IW_AE_CQ_OPERATION_ERROR) - continue; - } - - switch (info->ae_id) { - case I40IW_AE_LLP_FIN_RECEIVED: - if (qp->term_flags) - break; - if (atomic_inc_return(&iwqp->close_timer_started) == 1) { - iwqp->hw_tcp_state = I40IW_TCP_STATE_CLOSE_WAIT; - if ((iwqp->hw_tcp_state == I40IW_TCP_STATE_CLOSE_WAIT) && - (iwqp->ibqp_state == IB_QPS_RTS)) { - i40iw_next_iw_state(iwqp, - I40IW_QP_STATE_CLOSING, 0, 0, 0); - i40iw_cm_disconn(iwqp); - } - iwqp->cm_id->add_ref(iwqp->cm_id); - i40iw_schedule_cm_timer(iwqp->cm_node, - (struct i40iw_puda_buf *)iwqp, - I40IW_TIMER_TYPE_CLOSE, 1, 0); - } - break; - case I40IW_AE_LLP_CLOSE_COMPLETE: - if (qp->term_flags) - i40iw_terminate_done(qp, 0); - else - i40iw_cm_disconn(iwqp); - break; - case I40IW_AE_BAD_CLOSE: - case I40IW_AE_RESET_SENT: - i40iw_next_iw_state(iwqp, I40IW_QP_STATE_ERROR, 1, 0, 0); - i40iw_cm_disconn(iwqp); - break; - case I40IW_AE_LLP_CONNECTION_RESET: - if (atomic_read(&iwqp->close_timer_started)) - break; - i40iw_cm_disconn(iwqp); - break; - case I40IW_AE_QP_SUSPEND_COMPLETE: - i40iw_qp_suspend_resume(dev, &iwqp->sc_qp, false); - break; - case I40IW_AE_TERMINATE_SENT: - i40iw_terminate_send_fin(qp); - break; - case I40IW_AE_LLP_TERMINATE_RECEIVED: - i40iw_terminate_received(qp, info); - break; - case I40IW_AE_CQ_OPERATION_ERROR: - i40iw_pr_err("Processing an iWARP related AE for CQ misc = 0x%04X\n", - info->ae_id); - cq = (struct i40iw_sc_cq *)(unsigned long)info->compl_ctx; - iwcq = (struct i40iw_cq *)cq->back_cq; - - if (iwcq->ibcq.event_handler) { - struct ib_event ibevent; - - ibevent.device = iwcq->ibcq.device; - ibevent.event = IB_EVENT_CQ_ERR; - ibevent.element.cq = &iwcq->ibcq; - iwcq->ibcq.event_handler(&ibevent, iwcq->ibcq.cq_context); - } - break; - case I40IW_AE_LLP_DOUBT_REACHABILITY: - break; - case I40IW_AE_PRIV_OPERATION_DENIED: - case I40IW_AE_STAG_ZERO_INVALID: - case I40IW_AE_IB_RREQ_AND_Q1_FULL: - case I40IW_AE_DDP_UBE_INVALID_DDP_VERSION: - case I40IW_AE_DDP_UBE_INVALID_MO: - case I40IW_AE_DDP_UBE_INVALID_QN: - case I40IW_AE_DDP_NO_L_BIT: - case I40IW_AE_RDMAP_ROE_INVALID_RDMAP_VERSION: - case I40IW_AE_RDMAP_ROE_UNEXPECTED_OPCODE: - case I40IW_AE_ROE_INVALID_RDMA_READ_REQUEST: - case I40IW_AE_ROE_INVALID_RDMA_WRITE_OR_READ_RESP: - case I40IW_AE_INVALID_ARP_ENTRY: - case I40IW_AE_INVALID_TCP_OPTION_RCVD: - case I40IW_AE_STALE_ARP_ENTRY: - case I40IW_AE_LLP_RECEIVED_MPA_CRC_ERROR: - case I40IW_AE_LLP_SEGMENT_TOO_SMALL: - case I40IW_AE_LLP_SYN_RECEIVED: - case I40IW_AE_LLP_TOO_MANY_RETRIES: - case I40IW_AE_LCE_QP_CATASTROPHIC: - case I40IW_AE_LCE_FUNCTION_CATASTROPHIC: - case I40IW_AE_LCE_CQ_CATASTROPHIC: - case I40IW_AE_UDA_XMIT_DGRAM_TOO_LONG: - case I40IW_AE_UDA_XMIT_DGRAM_TOO_SHORT: - ctx_info->err_rq_idx_valid = false; - fallthrough; - default: - if (!info->sq && ctx_info->err_rq_idx_valid) { - ctx_info->err_rq_idx = info->wqe_idx; - ctx_info->tcp_info_valid = false; - ctx_info->iwarp_info_valid = false; - ret = dev->iw_priv_qp_ops->qp_setctx(&iwqp->sc_qp, - iwqp->host_ctx.va, - ctx_info); - } - i40iw_terminate_connection(qp, info); - break; - } - if (info->qp) - i40iw_qp_rem_ref(&iwqp->ibqp); - } while (1); - - if (aeqcnt) - dev->aeq_ops->repost_aeq_entries(dev, aeqcnt); -} - -/** - * i40iw_cqp_manage_abvpt_cmd - send cqp command manage abpvt - * @iwdev: iwarp device - * @accel_local_port: port for apbvt - * @add_port: add or delete port - */ -static enum i40iw_status_code -i40iw_cqp_manage_abvpt_cmd(struct i40iw_device *iwdev, - u16 accel_local_port, - bool add_port) -{ - struct i40iw_apbvt_info *info; - struct i40iw_cqp_request *cqp_request; - struct cqp_commands_info *cqp_info; - enum i40iw_status_code status; - - cqp_request = i40iw_get_cqp_request(&iwdev->cqp, add_port); - if (!cqp_request) - return I40IW_ERR_NO_MEMORY; - - cqp_info = &cqp_request->info; - info = &cqp_info->in.u.manage_apbvt_entry.info; - - memset(info, 0, sizeof(*info)); - info->add = add_port; - info->port = cpu_to_le16(accel_local_port); - - cqp_info->cqp_cmd = OP_MANAGE_APBVT_ENTRY; - cqp_info->post_sq = 1; - cqp_info->in.u.manage_apbvt_entry.cqp = &iwdev->cqp.sc_cqp; - cqp_info->in.u.manage_apbvt_entry.scratch = (uintptr_t)cqp_request; - status = i40iw_handle_cqp_op(iwdev, cqp_request); - if (status) - i40iw_pr_err("CQP-OP Manage APBVT entry fail"); - - return status; -} - -/** - * i40iw_manage_apbvt - add or delete tcp port - * @iwdev: iwarp device - * @accel_local_port: port for apbvt - * @add_port: add or delete port - */ -enum i40iw_status_code i40iw_manage_apbvt(struct i40iw_device *iwdev, - u16 accel_local_port, - bool add_port) -{ - struct i40iw_cm_core *cm_core = &iwdev->cm_core; - enum i40iw_status_code status; - unsigned long flags; - bool in_use; - - /* apbvt_lock is held across CQP delete APBVT OP (non-waiting) to - * protect against race where add APBVT CQP can race ahead of the delete - * APBVT for same port. - */ - if (add_port) { - spin_lock_irqsave(&cm_core->apbvt_lock, flags); - in_use = __test_and_set_bit(accel_local_port, - cm_core->ports_in_use); - spin_unlock_irqrestore(&cm_core->apbvt_lock, flags); - if (in_use) - return 0; - return i40iw_cqp_manage_abvpt_cmd(iwdev, accel_local_port, - true); - } else { - spin_lock_irqsave(&cm_core->apbvt_lock, flags); - in_use = i40iw_port_in_use(cm_core, accel_local_port); - if (in_use) { - spin_unlock_irqrestore(&cm_core->apbvt_lock, flags); - return 0; - } - __clear_bit(accel_local_port, cm_core->ports_in_use); - status = i40iw_cqp_manage_abvpt_cmd(iwdev, accel_local_port, - false); - spin_unlock_irqrestore(&cm_core->apbvt_lock, flags); - return status; - } -} - -/** - * i40iw_manage_arp_cache - manage hw arp cache - * @iwdev: iwarp device - * @mac_addr: mac address ptr - * @ip_addr: ip addr for arp cache - * @ipv4: flag indicating IPv4 when true - * @action: add, delete or modify - */ -void i40iw_manage_arp_cache(struct i40iw_device *iwdev, - unsigned char *mac_addr, - u32 *ip_addr, - bool ipv4, - u32 action) -{ - struct i40iw_add_arp_cache_entry_info *info; - struct i40iw_cqp_request *cqp_request; - struct cqp_commands_info *cqp_info; - int arp_index; - - arp_index = i40iw_arp_table(iwdev, ip_addr, ipv4, mac_addr, action); - if (arp_index < 0) - return; - cqp_request = i40iw_get_cqp_request(&iwdev->cqp, false); - if (!cqp_request) - return; - - cqp_info = &cqp_request->info; - if (action == I40IW_ARP_ADD) { - cqp_info->cqp_cmd = OP_ADD_ARP_CACHE_ENTRY; - info = &cqp_info->in.u.add_arp_cache_entry.info; - memset(info, 0, sizeof(*info)); - info->arp_index = cpu_to_le16((u16)arp_index); - info->permanent = true; - ether_addr_copy(info->mac_addr, mac_addr); - cqp_info->in.u.add_arp_cache_entry.scratch = (uintptr_t)cqp_request; - cqp_info->in.u.add_arp_cache_entry.cqp = &iwdev->cqp.sc_cqp; - } else { - cqp_info->cqp_cmd = OP_DELETE_ARP_CACHE_ENTRY; - cqp_info->in.u.del_arp_cache_entry.scratch = (uintptr_t)cqp_request; - cqp_info->in.u.del_arp_cache_entry.cqp = &iwdev->cqp.sc_cqp; - cqp_info->in.u.del_arp_cache_entry.arp_index = arp_index; - } - - cqp_info->in.u.add_arp_cache_entry.cqp = &iwdev->cqp.sc_cqp; - cqp_info->in.u.add_arp_cache_entry.scratch = (uintptr_t)cqp_request; - cqp_info->post_sq = 1; - if (i40iw_handle_cqp_op(iwdev, cqp_request)) - i40iw_pr_err("CQP-OP Add/Del Arp Cache entry fail"); -} - -/** - * i40iw_send_syn_cqp_callback - do syn/ack after qhash - * @cqp_request: qhash cqp completion - * @send_ack: flag send ack - */ -static void i40iw_send_syn_cqp_callback(struct i40iw_cqp_request *cqp_request, u32 send_ack) -{ - i40iw_send_syn(cqp_request->param, send_ack); -} - -/** - * i40iw_manage_qhash - add or modify qhash - * @iwdev: iwarp device - * @cminfo: cm info for qhash - * @etype: type (syn or quad) - * @mtype: type of qhash - * @cmnode: cmnode associated with connection - * @wait: wait for completion - */ -enum i40iw_status_code i40iw_manage_qhash(struct i40iw_device *iwdev, - struct i40iw_cm_info *cminfo, - enum i40iw_quad_entry_type etype, - enum i40iw_quad_hash_manage_type mtype, - void *cmnode, - bool wait) -{ - struct i40iw_qhash_table_info *info; - struct i40iw_sc_dev *dev = &iwdev->sc_dev; - struct i40iw_sc_vsi *vsi = &iwdev->vsi; - enum i40iw_status_code status; - struct i40iw_cqp *iwcqp = &iwdev->cqp; - struct i40iw_cqp_request *cqp_request; - struct cqp_commands_info *cqp_info; - - cqp_request = i40iw_get_cqp_request(iwcqp, wait); - if (!cqp_request) - return I40IW_ERR_NO_MEMORY; - cqp_info = &cqp_request->info; - info = &cqp_info->in.u.manage_qhash_table_entry.info; - memset(info, 0, sizeof(*info)); - - info->vsi = &iwdev->vsi; - info->manage = mtype; - info->entry_type = etype; - if (cminfo->vlan_id != 0xFFFF) { - info->vlan_valid = true; - info->vlan_id = cpu_to_le16(cminfo->vlan_id); - } else { - info->vlan_valid = false; - } - - info->ipv4_valid = cminfo->ipv4; - info->user_pri = cminfo->user_pri; - ether_addr_copy(info->mac_addr, iwdev->netdev->dev_addr); - info->qp_num = cpu_to_le32(vsi->ilq->qp_id); - info->dest_port = cpu_to_le16(cminfo->loc_port); - info->dest_ip[0] = cpu_to_le32(cminfo->loc_addr[0]); - info->dest_ip[1] = cpu_to_le32(cminfo->loc_addr[1]); - info->dest_ip[2] = cpu_to_le32(cminfo->loc_addr[2]); - info->dest_ip[3] = cpu_to_le32(cminfo->loc_addr[3]); - if (etype == I40IW_QHASH_TYPE_TCP_ESTABLISHED) { - info->src_port = cpu_to_le16(cminfo->rem_port); - info->src_ip[0] = cpu_to_le32(cminfo->rem_addr[0]); - info->src_ip[1] = cpu_to_le32(cminfo->rem_addr[1]); - info->src_ip[2] = cpu_to_le32(cminfo->rem_addr[2]); - info->src_ip[3] = cpu_to_le32(cminfo->rem_addr[3]); - } - if (cmnode) { - cqp_request->callback_fcn = i40iw_send_syn_cqp_callback; - cqp_request->param = (void *)cmnode; - } - - if (info->ipv4_valid) - i40iw_debug(dev, I40IW_DEBUG_CM, - "%s:%s IP=%pI4, port=%d, mac=%pM, vlan_id=%d\n", - __func__, (!mtype) ? "DELETE" : "ADD", - info->dest_ip, - info->dest_port, info->mac_addr, cminfo->vlan_id); - else - i40iw_debug(dev, I40IW_DEBUG_CM, - "%s:%s IP=%pI6, port=%d, mac=%pM, vlan_id=%d\n", - __func__, (!mtype) ? "DELETE" : "ADD", - info->dest_ip, - info->dest_port, info->mac_addr, cminfo->vlan_id); - cqp_info->in.u.manage_qhash_table_entry.cqp = &iwdev->cqp.sc_cqp; - cqp_info->in.u.manage_qhash_table_entry.scratch = (uintptr_t)cqp_request; - cqp_info->cqp_cmd = OP_MANAGE_QHASH_TABLE_ENTRY; - cqp_info->post_sq = 1; - status = i40iw_handle_cqp_op(iwdev, cqp_request); - if (status) - i40iw_pr_err("CQP-OP Manage Qhash Entry fail"); - return status; -} - -/** - * i40iw_hw_flush_wqes - flush qp's wqe - * @iwdev: iwarp device - * @qp: hardware control qp - * @info: info for flush - * @wait: flag wait for completion - */ -enum i40iw_status_code i40iw_hw_flush_wqes(struct i40iw_device *iwdev, - struct i40iw_sc_qp *qp, - struct i40iw_qp_flush_info *info, - bool wait) -{ - enum i40iw_status_code status; - struct i40iw_qp_flush_info *hw_info; - struct i40iw_cqp_request *cqp_request; - struct cqp_commands_info *cqp_info; - struct i40iw_qp *iwqp = (struct i40iw_qp *)qp->back_qp; - - cqp_request = i40iw_get_cqp_request(&iwdev->cqp, wait); - if (!cqp_request) - return I40IW_ERR_NO_MEMORY; - - cqp_info = &cqp_request->info; - hw_info = &cqp_request->info.in.u.qp_flush_wqes.info; - memcpy(hw_info, info, sizeof(*hw_info)); - - cqp_info->cqp_cmd = OP_QP_FLUSH_WQES; - cqp_info->post_sq = 1; - cqp_info->in.u.qp_flush_wqes.qp = qp; - cqp_info->in.u.qp_flush_wqes.scratch = (uintptr_t)cqp_request; - status = i40iw_handle_cqp_op(iwdev, cqp_request); - if (status) { - i40iw_pr_err("CQP-OP Flush WQE's fail"); - complete(&iwqp->sq_drained); - complete(&iwqp->rq_drained); - return status; - } - if (!cqp_request->compl_info.maj_err_code) { - switch (cqp_request->compl_info.min_err_code) { - case I40IW_CQP_COMPL_RQ_WQE_FLUSHED: - complete(&iwqp->sq_drained); - break; - case I40IW_CQP_COMPL_SQ_WQE_FLUSHED: - complete(&iwqp->rq_drained); - break; - case I40IW_CQP_COMPL_RQ_SQ_WQE_FLUSHED: - break; - default: - complete(&iwqp->sq_drained); - complete(&iwqp->rq_drained); - break; - } - } - - return 0; -} - -/** - * i40iw_gen_ae - generate AE - * @iwdev: iwarp device - * @qp: qp associated with AE - * @info: info for ae - * @wait: wait for completion - */ -void i40iw_gen_ae(struct i40iw_device *iwdev, - struct i40iw_sc_qp *qp, - struct i40iw_gen_ae_info *info, - bool wait) -{ - struct i40iw_gen_ae_info *ae_info; - struct i40iw_cqp_request *cqp_request; - struct cqp_commands_info *cqp_info; - - cqp_request = i40iw_get_cqp_request(&iwdev->cqp, wait); - if (!cqp_request) - return; - - cqp_info = &cqp_request->info; - ae_info = &cqp_request->info.in.u.gen_ae.info; - memcpy(ae_info, info, sizeof(*ae_info)); - - cqp_info->cqp_cmd = OP_GEN_AE; - cqp_info->post_sq = 1; - cqp_info->in.u.gen_ae.qp = qp; - cqp_info->in.u.gen_ae.scratch = (uintptr_t)cqp_request; - if (i40iw_handle_cqp_op(iwdev, cqp_request)) - i40iw_pr_err("CQP OP failed attempting to generate ae_code=0x%x\n", - info->ae_code); -} - -/** - * i40iw_hw_manage_vf_pble_bp - manage vf pbles - * @iwdev: iwarp device - * @info: info for managing pble - * @wait: flag wait for completion - */ -enum i40iw_status_code i40iw_hw_manage_vf_pble_bp(struct i40iw_device *iwdev, - struct i40iw_manage_vf_pble_info *info, - bool wait) -{ - enum i40iw_status_code status; - struct i40iw_manage_vf_pble_info *hw_info; - struct i40iw_cqp_request *cqp_request; - struct cqp_commands_info *cqp_info; - - if ((iwdev->init_state < CCQ_CREATED) && wait) - wait = false; - - cqp_request = i40iw_get_cqp_request(&iwdev->cqp, wait); - if (!cqp_request) - return I40IW_ERR_NO_MEMORY; - - cqp_info = &cqp_request->info; - hw_info = &cqp_request->info.in.u.manage_vf_pble_bp.info; - memcpy(hw_info, info, sizeof(*hw_info)); - - cqp_info->cqp_cmd = OP_MANAGE_VF_PBLE_BP; - cqp_info->post_sq = 1; - cqp_info->in.u.manage_vf_pble_bp.cqp = &iwdev->cqp.sc_cqp; - cqp_info->in.u.manage_vf_pble_bp.scratch = (uintptr_t)cqp_request; - status = i40iw_handle_cqp_op(iwdev, cqp_request); - if (status) - i40iw_pr_err("CQP-OP Manage VF pble_bp fail"); - return status; -} - -/** - * i40iw_get_ib_wc - return change flush code to IB's - * @opcode: iwarp flush code - */ -static enum ib_wc_status i40iw_get_ib_wc(enum i40iw_flush_opcode opcode) -{ - switch (opcode) { - case FLUSH_PROT_ERR: - return IB_WC_LOC_PROT_ERR; - case FLUSH_REM_ACCESS_ERR: - return IB_WC_REM_ACCESS_ERR; - case FLUSH_LOC_QP_OP_ERR: - return IB_WC_LOC_QP_OP_ERR; - case FLUSH_REM_OP_ERR: - return IB_WC_REM_OP_ERR; - case FLUSH_LOC_LEN_ERR: - return IB_WC_LOC_LEN_ERR; - case FLUSH_GENERAL_ERR: - return IB_WC_GENERAL_ERR; - case FLUSH_FATAL_ERR: - default: - return IB_WC_FATAL_ERR; - } -} - -/** - * i40iw_set_flush_info - set flush info - * @pinfo: set flush info - * @min: minor err - * @maj: major err - * @opcode: flush error code - */ -static void i40iw_set_flush_info(struct i40iw_qp_flush_info *pinfo, - u16 *min, - u16 *maj, - enum i40iw_flush_opcode opcode) -{ - *min = (u16)i40iw_get_ib_wc(opcode); - *maj = CQE_MAJOR_DRV; - pinfo->userflushcode = true; -} - -/** - * i40iw_flush_wqes - flush wqe for qp - * @iwdev: iwarp device - * @iwqp: qp to flush wqes - */ -void i40iw_flush_wqes(struct i40iw_device *iwdev, struct i40iw_qp *iwqp) -{ - struct i40iw_qp_flush_info info; - struct i40iw_qp_flush_info *pinfo = &info; - - struct i40iw_sc_qp *qp = &iwqp->sc_qp; - - memset(pinfo, 0, sizeof(*pinfo)); - info.sq = true; - info.rq = true; - if (qp->term_flags) { - i40iw_set_flush_info(pinfo, &pinfo->sq_minor_code, - &pinfo->sq_major_code, qp->flush_code); - i40iw_set_flush_info(pinfo, &pinfo->rq_minor_code, - &pinfo->rq_major_code, qp->flush_code); - } - (void)i40iw_hw_flush_wqes(iwdev, &iwqp->sc_qp, &info, true); -} diff --git a/drivers/infiniband/hw/i40iw/i40iw_main.c b/drivers/infiniband/hw/i40iw/i40iw_main.c deleted file mode 100644 index 364f69cd620f..000000000000 --- a/drivers/infiniband/hw/i40iw/i40iw_main.c +++ /dev/null @@ -1,2064 +0,0 @@ -/******************************************************************************* -* -* Copyright (c) 2015-2016 Intel Corporation. All rights reserved. -* -* This software is available to you under a choice of one of two -* licenses. You may choose to be licensed under the terms of the GNU -* General Public License (GPL) Version 2, available from the file -* COPYING in the main directory of this source tree, or the -* OpenFabrics.org BSD license below: -* -* Redistribution and use in source and binary forms, with or -* without modification, are permitted provided that the following -* conditions are met: -* -* - Redistributions of source code must retain the above -* copyright notice, this list of conditions and the following -* disclaimer. -* -* - Redistributions in binary form must reproduce the above -* copyright notice, this list of conditions and the following -* disclaimer in the documentation and/or other materials -* provided with the distribution. -* -* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS -* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN -* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -* SOFTWARE. -* -*******************************************************************************/ - -#include -#include -#include -#include -#include -#include -#include -#include - -#include "i40iw.h" -#include "i40iw_register.h" -#include -#define CLIENT_IW_INTERFACE_VERSION_MAJOR 0 -#define CLIENT_IW_INTERFACE_VERSION_MINOR 01 -#define CLIENT_IW_INTERFACE_VERSION_BUILD 00 - -#define DRV_VERSION_MAJOR 0 -#define DRV_VERSION_MINOR 5 -#define DRV_VERSION_BUILD 123 -#define DRV_VERSION __stringify(DRV_VERSION_MAJOR) "." \ - __stringify(DRV_VERSION_MINOR) "." __stringify(DRV_VERSION_BUILD) - -static int debug; -module_param(debug, int, 0644); -MODULE_PARM_DESC(debug, "debug flags: 0=disabled (default), 0x7fffffff=all"); - -static int resource_profile; -module_param(resource_profile, int, 0644); -MODULE_PARM_DESC(resource_profile, - "Resource Profile: 0=no VF RDMA support (default), 1=Weighted VF, 2=Even Distribution"); - -static int max_rdma_vfs = 32; -module_param(max_rdma_vfs, int, 0644); -MODULE_PARM_DESC(max_rdma_vfs, "Maximum VF count: 0-32 32=default"); -static int mpa_version = 2; -module_param(mpa_version, int, 0644); -MODULE_PARM_DESC(mpa_version, "MPA version to be used in MPA Req/Resp 1 or 2"); - -MODULE_AUTHOR("Intel Corporation, "); -MODULE_DESCRIPTION("Intel(R) Ethernet Connection X722 iWARP RDMA Driver"); -MODULE_LICENSE("Dual BSD/GPL"); - -static struct i40e_client i40iw_client; -static char i40iw_client_name[I40E_CLIENT_STR_LENGTH] = "i40iw"; - -static LIST_HEAD(i40iw_handlers); -static DEFINE_SPINLOCK(i40iw_handler_lock); - -static enum i40iw_status_code i40iw_virtchnl_send(struct i40iw_sc_dev *dev, - u32 vf_id, u8 *msg, u16 len); - -static struct notifier_block i40iw_inetaddr_notifier = { - .notifier_call = i40iw_inetaddr_event -}; - -static struct notifier_block i40iw_inetaddr6_notifier = { - .notifier_call = i40iw_inet6addr_event -}; - -static struct notifier_block i40iw_net_notifier = { - .notifier_call = i40iw_net_event -}; - -static struct notifier_block i40iw_netdevice_notifier = { - .notifier_call = i40iw_netdevice_event -}; - -/** - * i40iw_find_i40e_handler - find a handler given a client info - * @ldev: pointer to a client info - */ -static struct i40iw_handler *i40iw_find_i40e_handler(struct i40e_info *ldev) -{ - struct i40iw_handler *hdl; - unsigned long flags; - - spin_lock_irqsave(&i40iw_handler_lock, flags); - list_for_each_entry(hdl, &i40iw_handlers, list) { - if (hdl->ldev.netdev == ldev->netdev) { - spin_unlock_irqrestore(&i40iw_handler_lock, flags); - return hdl; - } - } - spin_unlock_irqrestore(&i40iw_handler_lock, flags); - return NULL; -} - -/** - * i40iw_find_netdev - find a handler given a netdev - * @netdev: pointer to net_device - */ -struct i40iw_handler *i40iw_find_netdev(struct net_device *netdev) -{ - struct i40iw_handler *hdl; - unsigned long flags; - - spin_lock_irqsave(&i40iw_handler_lock, flags); - list_for_each_entry(hdl, &i40iw_handlers, list) { - if (hdl->ldev.netdev == netdev) { - spin_unlock_irqrestore(&i40iw_handler_lock, flags); - return hdl; - } - } - spin_unlock_irqrestore(&i40iw_handler_lock, flags); - return NULL; -} - -/** - * i40iw_add_handler - add a handler to the list - * @hdl: handler to be added to the handler list - */ -static void i40iw_add_handler(struct i40iw_handler *hdl) -{ - unsigned long flags; - - spin_lock_irqsave(&i40iw_handler_lock, flags); - list_add(&hdl->list, &i40iw_handlers); - spin_unlock_irqrestore(&i40iw_handler_lock, flags); -} - -/** - * i40iw_del_handler - delete a handler from the list - * @hdl: handler to be deleted from the handler list - */ -static int i40iw_del_handler(struct i40iw_handler *hdl) -{ - unsigned long flags; - - spin_lock_irqsave(&i40iw_handler_lock, flags); - list_del(&hdl->list); - spin_unlock_irqrestore(&i40iw_handler_lock, flags); - return 0; -} - -/** - * i40iw_enable_intr - set up device interrupts - * @dev: hardware control device structure - * @msix_id: id of the interrupt to be enabled - */ -static void i40iw_enable_intr(struct i40iw_sc_dev *dev, u32 msix_id) -{ - u32 val; - - val = I40E_PFINT_DYN_CTLN_INTENA_MASK | - I40E_PFINT_DYN_CTLN_CLEARPBA_MASK | - (3 << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT); - if (dev->is_pf) - i40iw_wr32(dev->hw, I40E_PFINT_DYN_CTLN(msix_id - 1), val); - else - i40iw_wr32(dev->hw, I40E_VFINT_DYN_CTLN1(msix_id - 1), val); -} - -/** - * i40iw_dpc - tasklet for aeq and ceq 0 - * @t: Timer context to fetch pointer to iwarp device - */ -static void i40iw_dpc(struct tasklet_struct *t) -{ - struct i40iw_device *iwdev = from_tasklet(iwdev, t, dpc_tasklet); - - if (iwdev->msix_shared) - i40iw_process_ceq(iwdev, iwdev->ceqlist); - i40iw_process_aeq(iwdev); - i40iw_enable_intr(&iwdev->sc_dev, iwdev->iw_msixtbl[0].idx); -} - -/** - * i40iw_ceq_dpc - dpc handler for CEQ - * @t: Timer context to fetch pointer to CEQ data - */ -static void i40iw_ceq_dpc(struct tasklet_struct *t) -{ - struct i40iw_ceq *iwceq = from_tasklet(iwceq, t, dpc_tasklet); - struct i40iw_device *iwdev = iwceq->iwdev; - - i40iw_process_ceq(iwdev, iwceq); - i40iw_enable_intr(&iwdev->sc_dev, iwceq->msix_idx); -} - -/** - * i40iw_irq_handler - interrupt handler for aeq and ceq0 - * @irq: Interrupt request number - * @data: iwarp device - */ -static irqreturn_t i40iw_irq_handler(int irq, void *data) -{ - struct i40iw_device *iwdev = (struct i40iw_device *)data; - - tasklet_schedule(&iwdev->dpc_tasklet); - return IRQ_HANDLED; -} - -/** - * i40iw_destroy_cqp - destroy control qp - * @iwdev: iwarp device - * @free_hwcqp: 1 if CQP should be destroyed - * - * Issue destroy cqp request and - * free the resources associated with the cqp - */ -static void i40iw_destroy_cqp(struct i40iw_device *iwdev, bool free_hwcqp) -{ - struct i40iw_sc_dev *dev = &iwdev->sc_dev; - struct i40iw_cqp *cqp = &iwdev->cqp; - - if (free_hwcqp) - dev->cqp_ops->cqp_destroy(dev->cqp); - - i40iw_cleanup_pending_cqp_op(iwdev); - - i40iw_free_dma_mem(dev->hw, &cqp->sq); - kfree(cqp->scratch_array); - iwdev->cqp.scratch_array = NULL; - - kfree(cqp->cqp_requests); - cqp->cqp_requests = NULL; -} - -/** - * i40iw_disable_irq - disable device interrupts - * @dev: hardware control device structure - * @msix_vec: msix vector to disable irq - * @dev_id: parameter to pass to free_irq (used during irq setup) - * - * The function is called when destroying aeq/ceq - */ -static void i40iw_disable_irq(struct i40iw_sc_dev *dev, - struct i40iw_msix_vector *msix_vec, - void *dev_id) -{ - if (dev->is_pf) - i40iw_wr32(dev->hw, I40E_PFINT_DYN_CTLN(msix_vec->idx - 1), 0); - else - i40iw_wr32(dev->hw, I40E_VFINT_DYN_CTLN1(msix_vec->idx - 1), 0); - irq_set_affinity_hint(msix_vec->irq, NULL); - free_irq(msix_vec->irq, dev_id); -} - -/** - * i40iw_destroy_aeq - destroy aeq - * @iwdev: iwarp device - * - * Issue a destroy aeq request and - * free the resources associated with the aeq - * The function is called during driver unload - */ -static void i40iw_destroy_aeq(struct i40iw_device *iwdev) -{ - enum i40iw_status_code status = I40IW_ERR_NOT_READY; - struct i40iw_sc_dev *dev = &iwdev->sc_dev; - struct i40iw_aeq *aeq = &iwdev->aeq; - - if (!iwdev->msix_shared) - i40iw_disable_irq(dev, iwdev->iw_msixtbl, (void *)iwdev); - if (iwdev->reset) - goto exit; - - if (!dev->aeq_ops->aeq_destroy(&aeq->sc_aeq, 0, 1)) - status = dev->aeq_ops->aeq_destroy_done(&aeq->sc_aeq); - if (status) - i40iw_pr_err("destroy aeq failed %d\n", status); - -exit: - i40iw_free_dma_mem(dev->hw, &aeq->mem); -} - -/** - * i40iw_destroy_ceq - destroy ceq - * @iwdev: iwarp device - * @iwceq: ceq to be destroyed - * - * Issue a destroy ceq request and - * free the resources associated with the ceq - */ -static void i40iw_destroy_ceq(struct i40iw_device *iwdev, - struct i40iw_ceq *iwceq) -{ - enum i40iw_status_code status; - struct i40iw_sc_dev *dev = &iwdev->sc_dev; - - if (iwdev->reset) - goto exit; - - status = dev->ceq_ops->ceq_destroy(&iwceq->sc_ceq, 0, 1); - if (status) { - i40iw_pr_err("ceq destroy command failed %d\n", status); - goto exit; - } - - status = dev->ceq_ops->cceq_destroy_done(&iwceq->sc_ceq); - if (status) - i40iw_pr_err("ceq destroy completion failed %d\n", status); -exit: - i40iw_free_dma_mem(dev->hw, &iwceq->mem); -} - -/** - * i40iw_dele_ceqs - destroy all ceq's - * @iwdev: iwarp device - * - * Go through all of the device ceq's and for each ceq - * disable the ceq interrupt and destroy the ceq - */ -static void i40iw_dele_ceqs(struct i40iw_device *iwdev) -{ - u32 i = 0; - struct i40iw_sc_dev *dev = &iwdev->sc_dev; - struct i40iw_ceq *iwceq = iwdev->ceqlist; - struct i40iw_msix_vector *msix_vec = iwdev->iw_msixtbl; - - if (iwdev->msix_shared) { - i40iw_disable_irq(dev, msix_vec, (void *)iwdev); - i40iw_destroy_ceq(iwdev, iwceq); - iwceq++; - i++; - } - - for (msix_vec++; i < iwdev->ceqs_count; i++, msix_vec++, iwceq++) { - i40iw_disable_irq(dev, msix_vec, (void *)iwceq); - i40iw_destroy_ceq(iwdev, iwceq); - } - - iwdev->sc_dev.ceq_valid = false; -} - -/** - * i40iw_destroy_ccq - destroy control cq - * @iwdev: iwarp device - * - * Issue destroy ccq request and - * free the resources associated with the ccq - */ -static void i40iw_destroy_ccq(struct i40iw_device *iwdev) -{ - struct i40iw_sc_dev *dev = &iwdev->sc_dev; - struct i40iw_ccq *ccq = &iwdev->ccq; - enum i40iw_status_code status = 0; - - if (!iwdev->reset) - status = dev->ccq_ops->ccq_destroy(dev->ccq, 0, true); - if (status) - i40iw_pr_err("ccq destroy failed %d\n", status); - i40iw_free_dma_mem(dev->hw, &ccq->mem_cq); -} - -/* types of hmc objects */ -static enum i40iw_hmc_rsrc_type iw_hmc_obj_types[] = { - I40IW_HMC_IW_QP, - I40IW_HMC_IW_CQ, - I40IW_HMC_IW_HTE, - I40IW_HMC_IW_ARP, - I40IW_HMC_IW_APBVT_ENTRY, - I40IW_HMC_IW_MR, - I40IW_HMC_IW_XF, - I40IW_HMC_IW_XFFL, - I40IW_HMC_IW_Q1, - I40IW_HMC_IW_Q1FL, - I40IW_HMC_IW_TIMER, -}; - -/** - * i40iw_close_hmc_objects_type - delete hmc objects of a given type - * @dev: iwarp device - * @obj_type: the hmc object type to be deleted - * @hmc_info: pointer to the HMC configuration information - * @is_pf: true if the function is PF otherwise false - * @reset: true if called before reset - */ -static void i40iw_close_hmc_objects_type(struct i40iw_sc_dev *dev, - enum i40iw_hmc_rsrc_type obj_type, - struct i40iw_hmc_info *hmc_info, - bool is_pf, - bool reset) -{ - struct i40iw_hmc_del_obj_info info; - - memset(&info, 0, sizeof(info)); - info.hmc_info = hmc_info; - info.rsrc_type = obj_type; - info.count = hmc_info->hmc_obj[obj_type].cnt; - info.is_pf = is_pf; - if (dev->hmc_ops->del_hmc_object(dev, &info, reset)) - i40iw_pr_err("del obj of type %d failed\n", obj_type); -} - -/** - * i40iw_del_hmc_objects - remove all device hmc objects - * @dev: iwarp device - * @hmc_info: hmc_info to free - * @is_pf: true if hmc_info belongs to PF, not vf nor allocated - * by PF on behalf of VF - * @reset: true if called before reset - */ -static void i40iw_del_hmc_objects(struct i40iw_sc_dev *dev, - struct i40iw_hmc_info *hmc_info, - bool is_pf, - bool reset) -{ - unsigned int i; - - for (i = 0; i < IW_HMC_OBJ_TYPE_NUM; i++) - i40iw_close_hmc_objects_type(dev, iw_hmc_obj_types[i], hmc_info, is_pf, reset); -} - -/** - * i40iw_ceq_handler - interrupt handler for ceq - * @irq: interrupt request number - * @data: ceq pointer - */ -static irqreturn_t i40iw_ceq_handler(int irq, void *data) -{ - struct i40iw_ceq *iwceq = (struct i40iw_ceq *)data; - - if (iwceq->irq != irq) - i40iw_pr_err("expected irq = %d received irq = %d\n", iwceq->irq, irq); - tasklet_schedule(&iwceq->dpc_tasklet); - return IRQ_HANDLED; -} - -/** - * i40iw_create_hmc_obj_type - create hmc object of a given type - * @dev: hardware control device structure - * @info: information for the hmc object to create - */ -static enum i40iw_status_code i40iw_create_hmc_obj_type(struct i40iw_sc_dev *dev, - struct i40iw_hmc_create_obj_info *info) -{ - return dev->hmc_ops->create_hmc_object(dev, info); -} - -/** - * i40iw_create_hmc_objs - create all hmc objects for the device - * @iwdev: iwarp device - * @is_pf: true if the function is PF otherwise false - * - * Create the device hmc objects and allocate hmc pages - * Return 0 if successful, otherwise clean up and return error - */ -static enum i40iw_status_code i40iw_create_hmc_objs(struct i40iw_device *iwdev, - bool is_pf) -{ - struct i40iw_sc_dev *dev = &iwdev->sc_dev; - struct i40iw_hmc_create_obj_info info; - enum i40iw_status_code status; - int i; - - memset(&info, 0, sizeof(info)); - info.hmc_info = dev->hmc_info; - info.is_pf = is_pf; - info.entry_type = iwdev->sd_type; - for (i = 0; i < IW_HMC_OBJ_TYPE_NUM; i++) { - info.rsrc_type = iw_hmc_obj_types[i]; - info.count = dev->hmc_info->hmc_obj[info.rsrc_type].cnt; - info.add_sd_cnt = 0; - status = i40iw_create_hmc_obj_type(dev, &info); - if (status) { - i40iw_pr_err("create obj type %d status = %d\n", - iw_hmc_obj_types[i], status); - break; - } - } - if (!status) - return (dev->cqp_misc_ops->static_hmc_pages_allocated(dev->cqp, 0, - dev->hmc_fn_id, - true, true)); - - while (i) { - i--; - /* destroy the hmc objects of a given type */ - i40iw_close_hmc_objects_type(dev, - iw_hmc_obj_types[i], - dev->hmc_info, - is_pf, - false); - } - return status; -} - -/** - * i40iw_obj_aligned_mem - get aligned memory from device allocated memory - * @iwdev: iwarp device - * @memptr: points to the memory addresses - * @size: size of memory needed - * @mask: mask for the aligned memory - * - * Get aligned memory of the requested size and - * update the memptr to point to the new aligned memory - * Return 0 if successful, otherwise return no memory error - */ -enum i40iw_status_code i40iw_obj_aligned_mem(struct i40iw_device *iwdev, - struct i40iw_dma_mem *memptr, - u32 size, - u32 mask) -{ - unsigned long va, newva; - unsigned long extra; - - va = (unsigned long)iwdev->obj_next.va; - newva = va; - if (mask) - newva = ALIGN(va, (mask + 1)); - extra = newva - va; - memptr->va = (u8 *)va + extra; - memptr->pa = iwdev->obj_next.pa + extra; - memptr->size = size; - if ((memptr->va + size) > (iwdev->obj_mem.va + iwdev->obj_mem.size)) - return I40IW_ERR_NO_MEMORY; - - iwdev->obj_next.va = memptr->va + size; - iwdev->obj_next.pa = memptr->pa + size; - return 0; -} - -/** - * i40iw_create_cqp - create control qp - * @iwdev: iwarp device - * - * Return 0, if the cqp and all the resources associated with it - * are successfully created, otherwise return error - */ -static enum i40iw_status_code i40iw_create_cqp(struct i40iw_device *iwdev) -{ - enum i40iw_status_code status; - u32 sqsize = I40IW_CQP_SW_SQSIZE_2048; - struct i40iw_dma_mem mem; - struct i40iw_sc_dev *dev = &iwdev->sc_dev; - struct i40iw_cqp_init_info cqp_init_info; - struct i40iw_cqp *cqp = &iwdev->cqp; - u16 maj_err, min_err; - int i; - - cqp->cqp_requests = kcalloc(sqsize, sizeof(*cqp->cqp_requests), GFP_KERNEL); - if (!cqp->cqp_requests) - return I40IW_ERR_NO_MEMORY; - cqp->scratch_array = kcalloc(sqsize, sizeof(*cqp->scratch_array), GFP_KERNEL); - if (!cqp->scratch_array) { - kfree(cqp->cqp_requests); - return I40IW_ERR_NO_MEMORY; - } - dev->cqp = &cqp->sc_cqp; - dev->cqp->dev = dev; - memset(&cqp_init_info, 0, sizeof(cqp_init_info)); - status = i40iw_allocate_dma_mem(dev->hw, &cqp->sq, - (sizeof(struct i40iw_cqp_sq_wqe) * sqsize), - I40IW_CQP_ALIGNMENT); - if (status) - goto exit; - status = i40iw_obj_aligned_mem(iwdev, &mem, sizeof(struct i40iw_cqp_ctx), - I40IW_HOST_CTX_ALIGNMENT_MASK); - if (status) - goto exit; - dev->cqp->host_ctx_pa = mem.pa; - dev->cqp->host_ctx = mem.va; - /* populate the cqp init info */ - cqp_init_info.dev = dev; - cqp_init_info.sq_size = sqsize; - cqp_init_info.sq = cqp->sq.va; - cqp_init_info.sq_pa = cqp->sq.pa; - cqp_init_info.host_ctx_pa = mem.pa; - cqp_init_info.host_ctx = mem.va; - cqp_init_info.hmc_profile = iwdev->resource_profile; - cqp_init_info.enabled_vf_count = iwdev->max_rdma_vfs; - cqp_init_info.scratch_array = cqp->scratch_array; - status = dev->cqp_ops->cqp_init(dev->cqp, &cqp_init_info); - if (status) { - i40iw_pr_err("cqp init status %d\n", status); - goto exit; - } - status = dev->cqp_ops->cqp_create(dev->cqp, &maj_err, &min_err); - if (status) { - i40iw_pr_err("cqp create status %d maj_err %d min_err %d\n", - status, maj_err, min_err); - goto exit; - } - spin_lock_init(&cqp->req_lock); - INIT_LIST_HEAD(&cqp->cqp_avail_reqs); - INIT_LIST_HEAD(&cqp->cqp_pending_reqs); - /* init the waitq of the cqp_requests and add them to the list */ - for (i = 0; i < sqsize; i++) { - init_waitqueue_head(&cqp->cqp_requests[i].waitq); - list_add_tail(&cqp->cqp_requests[i].list, &cqp->cqp_avail_reqs); - } - return 0; -exit: - /* clean up the created resources */ - i40iw_destroy_cqp(iwdev, false); - return status; -} - -/** - * i40iw_create_ccq - create control cq - * @iwdev: iwarp device - * - * Return 0, if the ccq and the resources associated with it - * are successfully created, otherwise return error - */ -static enum i40iw_status_code i40iw_create_ccq(struct i40iw_device *iwdev) -{ - struct i40iw_sc_dev *dev = &iwdev->sc_dev; - struct i40iw_dma_mem mem; - enum i40iw_status_code status; - struct i40iw_ccq_init_info info; - struct i40iw_ccq *ccq = &iwdev->ccq; - - memset(&info, 0, sizeof(info)); - dev->ccq = &ccq->sc_cq; - dev->ccq->dev = dev; - info.dev = dev; - ccq->shadow_area.size = sizeof(struct i40iw_cq_shadow_area); - ccq->mem_cq.size = sizeof(struct i40iw_cqe) * IW_CCQ_SIZE; - status = i40iw_allocate_dma_mem(dev->hw, &ccq->mem_cq, - ccq->mem_cq.size, I40IW_CQ0_ALIGNMENT); - if (status) - goto exit; - status = i40iw_obj_aligned_mem(iwdev, &mem, ccq->shadow_area.size, - I40IW_SHADOWAREA_MASK); - if (status) - goto exit; - ccq->sc_cq.back_cq = (void *)ccq; - /* populate the ccq init info */ - info.cq_base = ccq->mem_cq.va; - info.cq_pa = ccq->mem_cq.pa; - info.num_elem = IW_CCQ_SIZE; - info.shadow_area = mem.va; - info.shadow_area_pa = mem.pa; - info.ceqe_mask = false; - info.ceq_id_valid = true; - info.shadow_read_threshold = 16; - status = dev->ccq_ops->ccq_init(dev->ccq, &info); - if (!status) - status = dev->ccq_ops->ccq_create(dev->ccq, 0, true, true); -exit: - if (status) - i40iw_free_dma_mem(dev->hw, &ccq->mem_cq); - return status; -} - -/** - * i40iw_configure_ceq_vector - set up the msix interrupt vector for ceq - * @iwdev: iwarp device - * @msix_vec: interrupt vector information - * @iwceq: ceq associated with the vector - * @ceq_id: the id number of the iwceq - * - * Allocate interrupt resources and enable irq handling - * Return 0 if successful, otherwise return error - */ -static enum i40iw_status_code i40iw_configure_ceq_vector(struct i40iw_device *iwdev, - struct i40iw_ceq *iwceq, - u32 ceq_id, - struct i40iw_msix_vector *msix_vec) -{ - enum i40iw_status_code status; - - if (iwdev->msix_shared && !ceq_id) { - tasklet_setup(&iwdev->dpc_tasklet, i40iw_dpc); - status = request_irq(msix_vec->irq, i40iw_irq_handler, 0, "AEQCEQ", iwdev); - } else { - tasklet_setup(&iwceq->dpc_tasklet, i40iw_ceq_dpc); - status = request_irq(msix_vec->irq, i40iw_ceq_handler, 0, "CEQ", iwceq); - } - - cpumask_clear(&msix_vec->mask); - cpumask_set_cpu(msix_vec->cpu_affinity, &msix_vec->mask); - irq_set_affinity_hint(msix_vec->irq, &msix_vec->mask); - - if (status) { - i40iw_pr_err("ceq irq config fail\n"); - return I40IW_ERR_CONFIG; - } - msix_vec->ceq_id = ceq_id; - - return 0; -} - -/** - * i40iw_create_ceq - create completion event queue - * @iwdev: iwarp device - * @iwceq: pointer to the ceq resources to be created - * @ceq_id: the id number of the iwceq - * - * Return 0, if the ceq and the resources associated with it - * are successfully created, otherwise return error - */ -static enum i40iw_status_code i40iw_create_ceq(struct i40iw_device *iwdev, - struct i40iw_ceq *iwceq, - u32 ceq_id) -{ - enum i40iw_status_code status; - struct i40iw_ceq_init_info info; - struct i40iw_sc_dev *dev = &iwdev->sc_dev; - u64 scratch; - - memset(&info, 0, sizeof(info)); - info.ceq_id = ceq_id; - iwceq->iwdev = iwdev; - iwceq->mem.size = sizeof(struct i40iw_ceqe) * - iwdev->sc_dev.hmc_info->hmc_obj[I40IW_HMC_IW_CQ].cnt; - status = i40iw_allocate_dma_mem(dev->hw, &iwceq->mem, iwceq->mem.size, - I40IW_CEQ_ALIGNMENT); - if (status) - goto exit; - info.ceq_id = ceq_id; - info.ceqe_base = iwceq->mem.va; - info.ceqe_pa = iwceq->mem.pa; - - info.elem_cnt = iwdev->sc_dev.hmc_info->hmc_obj[I40IW_HMC_IW_CQ].cnt; - iwceq->sc_ceq.ceq_id = ceq_id; - info.dev = dev; - scratch = (uintptr_t)&iwdev->cqp.sc_cqp; - status = dev->ceq_ops->ceq_init(&iwceq->sc_ceq, &info); - if (!status) - status = dev->ceq_ops->cceq_create(&iwceq->sc_ceq, scratch); - -exit: - if (status) - i40iw_free_dma_mem(dev->hw, &iwceq->mem); - return status; -} - -void i40iw_request_reset(struct i40iw_device *iwdev) -{ - struct i40e_info *ldev = iwdev->ldev; - - ldev->ops->request_reset(ldev, iwdev->client, 1); -} - -/** - * i40iw_setup_ceqs - manage the device ceq's and their interrupt resources - * @iwdev: iwarp device - * @ldev: i40e lan device - * - * Allocate a list for all device completion event queues - * Create the ceq's and configure their msix interrupt vectors - * Return 0, if at least one ceq is successfully set up, otherwise return error - */ -static enum i40iw_status_code i40iw_setup_ceqs(struct i40iw_device *iwdev, - struct i40e_info *ldev) -{ - u32 i; - u32 ceq_id; - struct i40iw_ceq *iwceq; - struct i40iw_msix_vector *msix_vec; - enum i40iw_status_code status = 0; - u32 num_ceqs; - - if (ldev && ldev->ops && ldev->ops->setup_qvlist) { - status = ldev->ops->setup_qvlist(ldev, &i40iw_client, - iwdev->iw_qvlist); - if (status) - goto exit; - } else { - status = I40IW_ERR_BAD_PTR; - goto exit; - } - - num_ceqs = min(iwdev->msix_count, iwdev->sc_dev.hmc_fpm_misc.max_ceqs); - iwdev->ceqlist = kcalloc(num_ceqs, sizeof(*iwdev->ceqlist), GFP_KERNEL); - if (!iwdev->ceqlist) { - status = I40IW_ERR_NO_MEMORY; - goto exit; - } - i = (iwdev->msix_shared) ? 0 : 1; - for (ceq_id = 0; i < num_ceqs; i++, ceq_id++) { - iwceq = &iwdev->ceqlist[ceq_id]; - status = i40iw_create_ceq(iwdev, iwceq, ceq_id); - if (status) { - i40iw_pr_err("create ceq status = %d\n", status); - break; - } - - msix_vec = &iwdev->iw_msixtbl[i]; - iwceq->irq = msix_vec->irq; - iwceq->msix_idx = msix_vec->idx; - status = i40iw_configure_ceq_vector(iwdev, iwceq, ceq_id, msix_vec); - if (status) { - i40iw_destroy_ceq(iwdev, iwceq); - break; - } - i40iw_enable_intr(&iwdev->sc_dev, msix_vec->idx); - iwdev->ceqs_count++; - } -exit: - if (status && !iwdev->ceqs_count) { - kfree(iwdev->ceqlist); - iwdev->ceqlist = NULL; - return status; - } else { - iwdev->sc_dev.ceq_valid = true; - return 0; - } - -} - -/** - * i40iw_configure_aeq_vector - set up the msix vector for aeq - * @iwdev: iwarp device - * - * Allocate interrupt resources and enable irq handling - * Return 0 if successful, otherwise return error - */ -static enum i40iw_status_code i40iw_configure_aeq_vector(struct i40iw_device *iwdev) -{ - struct i40iw_msix_vector *msix_vec = iwdev->iw_msixtbl; - u32 ret = 0; - - if (!iwdev->msix_shared) { - tasklet_setup(&iwdev->dpc_tasklet, i40iw_dpc); - ret = request_irq(msix_vec->irq, i40iw_irq_handler, 0, "i40iw", iwdev); - } - if (ret) { - i40iw_pr_err("aeq irq config fail\n"); - return I40IW_ERR_CONFIG; - } - - return 0; -} - -/** - * i40iw_create_aeq - create async event queue - * @iwdev: iwarp device - * - * Return 0, if the aeq and the resources associated with it - * are successfully created, otherwise return error - */ -static enum i40iw_status_code i40iw_create_aeq(struct i40iw_device *iwdev) -{ - enum i40iw_status_code status; - struct i40iw_aeq_init_info info; - struct i40iw_sc_dev *dev = &iwdev->sc_dev; - struct i40iw_aeq *aeq = &iwdev->aeq; - u64 scratch = 0; - u32 aeq_size; - - aeq_size = 2 * iwdev->sc_dev.hmc_info->hmc_obj[I40IW_HMC_IW_QP].cnt + - iwdev->sc_dev.hmc_info->hmc_obj[I40IW_HMC_IW_CQ].cnt; - memset(&info, 0, sizeof(info)); - aeq->mem.size = sizeof(struct i40iw_sc_aeqe) * aeq_size; - status = i40iw_allocate_dma_mem(dev->hw, &aeq->mem, aeq->mem.size, - I40IW_AEQ_ALIGNMENT); - if (status) - goto exit; - - info.aeqe_base = aeq->mem.va; - info.aeq_elem_pa = aeq->mem.pa; - info.elem_cnt = aeq_size; - info.dev = dev; - status = dev->aeq_ops->aeq_init(&aeq->sc_aeq, &info); - if (status) - goto exit; - status = dev->aeq_ops->aeq_create(&aeq->sc_aeq, scratch, 1); - if (!status) - status = dev->aeq_ops->aeq_create_done(&aeq->sc_aeq); -exit: - if (status) - i40iw_free_dma_mem(dev->hw, &aeq->mem); - return status; -} - -/** - * i40iw_setup_aeq - set up the device aeq - * @iwdev: iwarp device - * - * Create the aeq and configure its msix interrupt vector - * Return 0 if successful, otherwise return error - */ -static enum i40iw_status_code i40iw_setup_aeq(struct i40iw_device *iwdev) -{ - struct i40iw_sc_dev *dev = &iwdev->sc_dev; - enum i40iw_status_code status; - - status = i40iw_create_aeq(iwdev); - if (status) - return status; - - status = i40iw_configure_aeq_vector(iwdev); - if (status) { - i40iw_destroy_aeq(iwdev); - return status; - } - - if (!iwdev->msix_shared) - i40iw_enable_intr(dev, iwdev->iw_msixtbl[0].idx); - return 0; -} - -/** - * i40iw_initialize_ilq - create iwarp local queue for cm - * @iwdev: iwarp device - * - * Return 0 if successful, otherwise return error - */ -static enum i40iw_status_code i40iw_initialize_ilq(struct i40iw_device *iwdev) -{ - struct i40iw_puda_rsrc_info info; - enum i40iw_status_code status; - - memset(&info, 0, sizeof(info)); - info.type = I40IW_PUDA_RSRC_TYPE_ILQ; - info.cq_id = 1; - info.qp_id = 0; - info.count = 1; - info.pd_id = 1; - info.sq_size = 8192; - info.rq_size = 8192; - info.buf_size = 1024; - info.tx_buf_cnt = 16384; - info.receive = i40iw_receive_ilq; - info.xmit_complete = i40iw_free_sqbuf; - status = i40iw_puda_create_rsrc(&iwdev->vsi, &info); - if (status) - i40iw_pr_err("ilq create fail\n"); - return status; -} - -/** - * i40iw_initialize_ieq - create iwarp exception queue - * @iwdev: iwarp device - * - * Return 0 if successful, otherwise return error - */ -static enum i40iw_status_code i40iw_initialize_ieq(struct i40iw_device *iwdev) -{ - struct i40iw_puda_rsrc_info info; - enum i40iw_status_code status; - - memset(&info, 0, sizeof(info)); - info.type = I40IW_PUDA_RSRC_TYPE_IEQ; - info.cq_id = 2; - info.qp_id = iwdev->vsi.exception_lan_queue; - info.count = 1; - info.pd_id = 2; - info.sq_size = 8192; - info.rq_size = 8192; - info.buf_size = iwdev->vsi.mtu + VLAN_ETH_HLEN; - info.tx_buf_cnt = 4096; - status = i40iw_puda_create_rsrc(&iwdev->vsi, &info); - if (status) - i40iw_pr_err("ieq create fail\n"); - return status; -} - -/** - * i40iw_reinitialize_ieq - destroy and re-create ieq - * @dev: iwarp device - */ -void i40iw_reinitialize_ieq(struct i40iw_sc_dev *dev) -{ - struct i40iw_device *iwdev = (struct i40iw_device *)dev->back_dev; - - i40iw_puda_dele_resources(&iwdev->vsi, I40IW_PUDA_RSRC_TYPE_IEQ, false); - if (i40iw_initialize_ieq(iwdev)) { - iwdev->reset = true; - i40iw_request_reset(iwdev); - } -} - -/** - * i40iw_hmc_setup - create hmc objects for the device - * @iwdev: iwarp device - * - * Set up the device private memory space for the number and size of - * the hmc objects and create the objects - * Return 0 if successful, otherwise return error - */ -static enum i40iw_status_code i40iw_hmc_setup(struct i40iw_device *iwdev) -{ - enum i40iw_status_code status; - - iwdev->sd_type = I40IW_SD_TYPE_DIRECT; - status = i40iw_config_fpm_values(&iwdev->sc_dev, IW_CFG_FPM_QP_COUNT); - if (status) - goto exit; - status = i40iw_create_hmc_objs(iwdev, true); - if (status) - goto exit; - iwdev->init_state = HMC_OBJS_CREATED; -exit: - return status; -} - -/** - * i40iw_del_init_mem - deallocate memory resources - * @iwdev: iwarp device - */ -static void i40iw_del_init_mem(struct i40iw_device *iwdev) -{ - struct i40iw_sc_dev *dev = &iwdev->sc_dev; - - i40iw_free_dma_mem(&iwdev->hw, &iwdev->obj_mem); - kfree(dev->hmc_info->sd_table.sd_entry); - dev->hmc_info->sd_table.sd_entry = NULL; - kfree(iwdev->mem_resources); - iwdev->mem_resources = NULL; - kfree(iwdev->ceqlist); - iwdev->ceqlist = NULL; - kfree(iwdev->iw_msixtbl); - iwdev->iw_msixtbl = NULL; - kfree(iwdev->hmc_info_mem); - iwdev->hmc_info_mem = NULL; -} - -/** - * i40iw_del_macip_entry - remove a mac ip address entry from the hw table - * @iwdev: iwarp device - * @idx: the index of the mac ip address to delete - */ -static void i40iw_del_macip_entry(struct i40iw_device *iwdev, u8 idx) -{ - struct i40iw_cqp *iwcqp = &iwdev->cqp; - struct i40iw_cqp_request *cqp_request; - struct cqp_commands_info *cqp_info; - enum i40iw_status_code status = 0; - - cqp_request = i40iw_get_cqp_request(iwcqp, true); - if (!cqp_request) { - i40iw_pr_err("cqp_request memory failed\n"); - return; - } - cqp_info = &cqp_request->info; - cqp_info->cqp_cmd = OP_DELETE_LOCAL_MAC_IPADDR_ENTRY; - cqp_info->post_sq = 1; - cqp_info->in.u.del_local_mac_ipaddr_entry.cqp = &iwcqp->sc_cqp; - cqp_info->in.u.del_local_mac_ipaddr_entry.scratch = (uintptr_t)cqp_request; - cqp_info->in.u.del_local_mac_ipaddr_entry.entry_idx = idx; - cqp_info->in.u.del_local_mac_ipaddr_entry.ignore_ref_count = 0; - status = i40iw_handle_cqp_op(iwdev, cqp_request); - if (status) - i40iw_pr_err("CQP-OP Del MAC Ip entry fail"); -} - -/** - * i40iw_add_mac_ipaddr_entry - add a mac ip address entry to the hw table - * @iwdev: iwarp device - * @mac_addr: pointer to mac address - * @idx: the index of the mac ip address to add - */ -static enum i40iw_status_code i40iw_add_mac_ipaddr_entry(struct i40iw_device *iwdev, - u8 *mac_addr, - u8 idx) -{ - struct i40iw_local_mac_ipaddr_entry_info *info; - struct i40iw_cqp *iwcqp = &iwdev->cqp; - struct i40iw_cqp_request *cqp_request; - struct cqp_commands_info *cqp_info; - enum i40iw_status_code status = 0; - - cqp_request = i40iw_get_cqp_request(iwcqp, true); - if (!cqp_request) { - i40iw_pr_err("cqp_request memory failed\n"); - return I40IW_ERR_NO_MEMORY; - } - - cqp_info = &cqp_request->info; - - cqp_info->post_sq = 1; - info = &cqp_info->in.u.add_local_mac_ipaddr_entry.info; - ether_addr_copy(info->mac_addr, mac_addr); - info->entry_idx = idx; - cqp_info->in.u.add_local_mac_ipaddr_entry.scratch = (uintptr_t)cqp_request; - cqp_info->cqp_cmd = OP_ADD_LOCAL_MAC_IPADDR_ENTRY; - cqp_info->in.u.add_local_mac_ipaddr_entry.cqp = &iwcqp->sc_cqp; - cqp_info->in.u.add_local_mac_ipaddr_entry.scratch = (uintptr_t)cqp_request; - status = i40iw_handle_cqp_op(iwdev, cqp_request); - if (status) - i40iw_pr_err("CQP-OP Add MAC Ip entry fail"); - return status; -} - -/** - * i40iw_alloc_local_mac_ipaddr_entry - allocate a mac ip address entry - * @iwdev: iwarp device - * @mac_ip_tbl_idx: the index of the new mac ip address - * - * Allocate a mac ip address entry and update the mac_ip_tbl_idx - * to hold the index of the newly created mac ip address - * Return 0 if successful, otherwise return error - */ -static enum i40iw_status_code i40iw_alloc_local_mac_ipaddr_entry(struct i40iw_device *iwdev, - u16 *mac_ip_tbl_idx) -{ - struct i40iw_cqp *iwcqp = &iwdev->cqp; - struct i40iw_cqp_request *cqp_request; - struct cqp_commands_info *cqp_info; - enum i40iw_status_code status = 0; - - cqp_request = i40iw_get_cqp_request(iwcqp, true); - if (!cqp_request) { - i40iw_pr_err("cqp_request memory failed\n"); - return I40IW_ERR_NO_MEMORY; - } - - /* increment refcount, because we need the cqp request ret value */ - atomic_inc(&cqp_request->refcount); - - cqp_info = &cqp_request->info; - cqp_info->cqp_cmd = OP_ALLOC_LOCAL_MAC_IPADDR_ENTRY; - cqp_info->post_sq = 1; - cqp_info->in.u.alloc_local_mac_ipaddr_entry.cqp = &iwcqp->sc_cqp; - cqp_info->in.u.alloc_local_mac_ipaddr_entry.scratch = (uintptr_t)cqp_request; - status = i40iw_handle_cqp_op(iwdev, cqp_request); - if (!status) - *mac_ip_tbl_idx = cqp_request->compl_info.op_ret_val; - else - i40iw_pr_err("CQP-OP Alloc MAC Ip entry fail"); - /* decrement refcount and free the cqp request, if no longer used */ - i40iw_put_cqp_request(iwcqp, cqp_request); - return status; -} - -/** - * i40iw_alloc_set_mac_ipaddr - set up a mac ip address table entry - * @iwdev: iwarp device - * @macaddr: pointer to mac address - * - * Allocate a mac ip address entry and add it to the hw table - * Return 0 if successful, otherwise return error - */ -static enum i40iw_status_code i40iw_alloc_set_mac_ipaddr(struct i40iw_device *iwdev, - u8 *macaddr) -{ - enum i40iw_status_code status; - - status = i40iw_alloc_local_mac_ipaddr_entry(iwdev, &iwdev->mac_ip_table_idx); - if (!status) { - status = i40iw_add_mac_ipaddr_entry(iwdev, macaddr, - (u8)iwdev->mac_ip_table_idx); - if (status) - i40iw_del_macip_entry(iwdev, (u8)iwdev->mac_ip_table_idx); - } - return status; -} - -/** - * i40iw_add_ipv6_addr - add ipv6 address to the hw arp table - * @iwdev: iwarp device - */ -static void i40iw_add_ipv6_addr(struct i40iw_device *iwdev) -{ - struct net_device *ip_dev; - struct inet6_dev *idev; - struct inet6_ifaddr *ifp, *tmp; - u32 local_ipaddr6[4]; - - rcu_read_lock(); - for_each_netdev_rcu(&init_net, ip_dev) { - if ((((rdma_vlan_dev_vlan_id(ip_dev) < 0xFFFF) && - (rdma_vlan_dev_real_dev(ip_dev) == iwdev->netdev)) || - (ip_dev == iwdev->netdev)) && (ip_dev->flags & IFF_UP)) { - idev = __in6_dev_get(ip_dev); - if (!idev) { - i40iw_pr_err("ipv6 inet device not found\n"); - break; - } - list_for_each_entry_safe(ifp, tmp, &idev->addr_list, if_list) { - i40iw_pr_info("IP=%pI6, vlan_id=%d, MAC=%pM\n", &ifp->addr, - rdma_vlan_dev_vlan_id(ip_dev), ip_dev->dev_addr); - i40iw_copy_ip_ntohl(local_ipaddr6, - ifp->addr.in6_u.u6_addr32); - i40iw_manage_arp_cache(iwdev, - ip_dev->dev_addr, - local_ipaddr6, - false, - I40IW_ARP_ADD); - } - } - } - rcu_read_unlock(); -} - -/** - * i40iw_add_ipv4_addr - add ipv4 address to the hw arp table - * @iwdev: iwarp device - */ -static void i40iw_add_ipv4_addr(struct i40iw_device *iwdev) -{ - struct net_device *dev; - struct in_device *idev; - u32 ip_addr; - - rcu_read_lock(); - for_each_netdev_rcu(&init_net, dev) { - if ((((rdma_vlan_dev_vlan_id(dev) < 0xFFFF) && - (rdma_vlan_dev_real_dev(dev) == iwdev->netdev)) || - (dev == iwdev->netdev)) && (READ_ONCE(dev->flags) & IFF_UP)) { - const struct in_ifaddr *ifa; - - idev = __in_dev_get_rcu(dev); - if (!idev) - continue; - in_dev_for_each_ifa_rcu(ifa, idev) { - i40iw_debug(&iwdev->sc_dev, I40IW_DEBUG_CM, - "IP=%pI4, vlan_id=%d, MAC=%pM\n", &ifa->ifa_address, - rdma_vlan_dev_vlan_id(dev), dev->dev_addr); - - ip_addr = ntohl(ifa->ifa_address); - i40iw_manage_arp_cache(iwdev, - dev->dev_addr, - &ip_addr, - true, - I40IW_ARP_ADD); - } - } - } - rcu_read_unlock(); -} - -/** - * i40iw_add_mac_ip - add mac and ip addresses - * @iwdev: iwarp device - * - * Create and add a mac ip address entry to the hw table and - * ipv4/ipv6 addresses to the arp cache - * Return 0 if successful, otherwise return error - */ -static enum i40iw_status_code i40iw_add_mac_ip(struct i40iw_device *iwdev) -{ - struct net_device *netdev = iwdev->netdev; - enum i40iw_status_code status; - - status = i40iw_alloc_set_mac_ipaddr(iwdev, (u8 *)netdev->dev_addr); - if (status) - return status; - i40iw_add_ipv4_addr(iwdev); - i40iw_add_ipv6_addr(iwdev); - return 0; -} - -/** - * i40iw_wait_pe_ready - Check if firmware is ready - * @hw: provides access to registers - */ -static void i40iw_wait_pe_ready(struct i40iw_hw *hw) -{ - u32 statusfw; - u32 statuscpu0; - u32 statuscpu1; - u32 statuscpu2; - u32 retrycount = 0; - - do { - statusfw = i40iw_rd32(hw, I40E_GLPE_FWLDSTATUS); - i40iw_pr_info("[%04d] fm load status[x%04X]\n", __LINE__, statusfw); - statuscpu0 = i40iw_rd32(hw, I40E_GLPE_CPUSTATUS0); - i40iw_pr_info("[%04d] CSR_CQP status[x%04X]\n", __LINE__, statuscpu0); - statuscpu1 = i40iw_rd32(hw, I40E_GLPE_CPUSTATUS1); - i40iw_pr_info("[%04d] I40E_GLPE_CPUSTATUS1 status[x%04X]\n", - __LINE__, statuscpu1); - statuscpu2 = i40iw_rd32(hw, I40E_GLPE_CPUSTATUS2); - i40iw_pr_info("[%04d] I40E_GLPE_CPUSTATUS2 status[x%04X]\n", - __LINE__, statuscpu2); - if ((statuscpu0 == 0x80) && (statuscpu1 == 0x80) && (statuscpu2 == 0x80)) - break; /* SUCCESS */ - msleep(1000); - retrycount++; - } while (retrycount < 14); - i40iw_wr32(hw, 0xb4040, 0x4C104C5); -} - -/** - * i40iw_initialize_dev - initialize device - * @iwdev: iwarp device - * @ldev: lan device information - * - * Allocate memory for the hmc objects and initialize iwdev - * Return 0 if successful, otherwise clean up the resources - * and return error - */ -static enum i40iw_status_code i40iw_initialize_dev(struct i40iw_device *iwdev, - struct i40e_info *ldev) -{ - enum i40iw_status_code status; - struct i40iw_sc_dev *dev = &iwdev->sc_dev; - struct i40iw_device_init_info info; - struct i40iw_vsi_init_info vsi_info; - struct i40iw_dma_mem mem; - struct i40iw_l2params l2params; - u32 size; - struct i40iw_vsi_stats_info stats_info; - u16 last_qset = I40IW_NO_QSET; - u16 qset; - u32 i; - - memset(&l2params, 0, sizeof(l2params)); - memset(&info, 0, sizeof(info)); - size = sizeof(struct i40iw_hmc_pble_rsrc) + sizeof(struct i40iw_hmc_info) + - (sizeof(struct i40iw_hmc_obj_info) * I40IW_HMC_IW_MAX); - iwdev->hmc_info_mem = kzalloc(size, GFP_KERNEL); - if (!iwdev->hmc_info_mem) - return I40IW_ERR_NO_MEMORY; - - iwdev->pble_rsrc = (struct i40iw_hmc_pble_rsrc *)iwdev->hmc_info_mem; - dev->hmc_info = &iwdev->hw.hmc; - dev->hmc_info->hmc_obj = (struct i40iw_hmc_obj_info *)(iwdev->pble_rsrc + 1); - status = i40iw_obj_aligned_mem(iwdev, &mem, I40IW_QUERY_FPM_BUF_SIZE, - I40IW_FPM_QUERY_BUF_ALIGNMENT_MASK); - if (status) - goto error; - info.fpm_query_buf_pa = mem.pa; - info.fpm_query_buf = mem.va; - status = i40iw_obj_aligned_mem(iwdev, &mem, I40IW_COMMIT_FPM_BUF_SIZE, - I40IW_FPM_COMMIT_BUF_ALIGNMENT_MASK); - if (status) - goto error; - info.fpm_commit_buf_pa = mem.pa; - info.fpm_commit_buf = mem.va; - info.hmc_fn_id = ldev->fid; - info.is_pf = (ldev->ftype) ? false : true; - info.bar0 = ldev->hw_addr; - info.hw = &iwdev->hw; - info.debug_mask = debug; - l2params.mtu = - (ldev->params.mtu) ? ldev->params.mtu : I40IW_DEFAULT_MTU; - for (i = 0; i < I40E_CLIENT_MAX_USER_PRIORITY; i++) { - qset = ldev->params.qos.prio_qos[i].qs_handle; - l2params.qs_handle_list[i] = qset; - if (last_qset == I40IW_NO_QSET) - last_qset = qset; - else if ((qset != last_qset) && (qset != I40IW_NO_QSET)) - iwdev->dcb = true; - } - i40iw_pr_info("DCB is set/clear = %d\n", iwdev->dcb); - info.vchnl_send = i40iw_virtchnl_send; - status = i40iw_device_init(&iwdev->sc_dev, &info); - - if (status) - goto error; - memset(&vsi_info, 0, sizeof(vsi_info)); - vsi_info.dev = &iwdev->sc_dev; - vsi_info.back_vsi = (void *)iwdev; - vsi_info.params = &l2params; - vsi_info.exception_lan_queue = 1; - i40iw_sc_vsi_init(&iwdev->vsi, &vsi_info); - - if (dev->is_pf) { - memset(&stats_info, 0, sizeof(stats_info)); - stats_info.fcn_id = ldev->fid; - stats_info.pestat = kzalloc(sizeof(*stats_info.pestat), GFP_KERNEL); - if (!stats_info.pestat) { - status = I40IW_ERR_NO_MEMORY; - goto error; - } - stats_info.stats_initialize = true; - if (stats_info.pestat) - i40iw_vsi_stats_init(&iwdev->vsi, &stats_info); - } - return status; -error: - kfree(iwdev->hmc_info_mem); - iwdev->hmc_info_mem = NULL; - return status; -} - -/** - * i40iw_register_notifiers - register tcp ip notifiers - */ -static void i40iw_register_notifiers(void) -{ - register_inetaddr_notifier(&i40iw_inetaddr_notifier); - register_inet6addr_notifier(&i40iw_inetaddr6_notifier); - register_netevent_notifier(&i40iw_net_notifier); - register_netdevice_notifier(&i40iw_netdevice_notifier); -} - -/** - * i40iw_unregister_notifiers - unregister tcp ip notifiers - */ - -static void i40iw_unregister_notifiers(void) -{ - unregister_netevent_notifier(&i40iw_net_notifier); - unregister_inetaddr_notifier(&i40iw_inetaddr_notifier); - unregister_inet6addr_notifier(&i40iw_inetaddr6_notifier); - unregister_netdevice_notifier(&i40iw_netdevice_notifier); -} - -/** - * i40iw_save_msix_info - copy msix vector information to iwarp device - * @iwdev: iwarp device - * @ldev: lan device information - * - * Allocate iwdev msix table and copy the ldev msix info to the table - * Return 0 if successful, otherwise return error - */ -static enum i40iw_status_code i40iw_save_msix_info(struct i40iw_device *iwdev, - struct i40e_info *ldev) -{ - struct i40e_qvlist_info *iw_qvlist; - struct i40e_qv_info *iw_qvinfo; - u32 ceq_idx; - u32 i; - size_t size; - - if (!ldev->msix_count) { - i40iw_pr_err("No MSI-X vectors\n"); - return I40IW_ERR_CONFIG; - } - - iwdev->msix_count = ldev->msix_count; - - size = sizeof(struct i40iw_msix_vector) * iwdev->msix_count; - size += struct_size(iw_qvlist, qv_info, iwdev->msix_count); - iwdev->iw_msixtbl = kzalloc(size, GFP_KERNEL); - - if (!iwdev->iw_msixtbl) - return I40IW_ERR_NO_MEMORY; - iwdev->iw_qvlist = (struct i40e_qvlist_info *)(&iwdev->iw_msixtbl[iwdev->msix_count]); - iw_qvlist = iwdev->iw_qvlist; - iw_qvinfo = iw_qvlist->qv_info; - iw_qvlist->num_vectors = iwdev->msix_count; - if (iwdev->msix_count <= num_online_cpus()) - iwdev->msix_shared = true; - for (i = 0, ceq_idx = 0; i < iwdev->msix_count; i++, iw_qvinfo++) { - iwdev->iw_msixtbl[i].idx = ldev->msix_entries[i].entry; - iwdev->iw_msixtbl[i].irq = ldev->msix_entries[i].vector; - iwdev->iw_msixtbl[i].cpu_affinity = ceq_idx; - if (i == 0) { - iw_qvinfo->aeq_idx = 0; - if (iwdev->msix_shared) - iw_qvinfo->ceq_idx = ceq_idx++; - else - iw_qvinfo->ceq_idx = I40E_QUEUE_INVALID_IDX; - } else { - iw_qvinfo->aeq_idx = I40E_QUEUE_INVALID_IDX; - iw_qvinfo->ceq_idx = ceq_idx++; - } - iw_qvinfo->itr_idx = 3; - iw_qvinfo->v_idx = iwdev->iw_msixtbl[i].idx; - } - return 0; -} - -/** - * i40iw_deinit_device - clean up the device resources - * @iwdev: iwarp device - * - * Destroy the ib device interface, remove the mac ip entry and ipv4/ipv6 addresses, - * destroy the device queues and free the pble and the hmc objects - */ -static void i40iw_deinit_device(struct i40iw_device *iwdev) -{ - struct i40e_info *ldev = iwdev->ldev; - - struct i40iw_sc_dev *dev = &iwdev->sc_dev; - - i40iw_pr_info("state = %d\n", iwdev->init_state); - if (iwdev->param_wq) - destroy_workqueue(iwdev->param_wq); - - switch (iwdev->init_state) { - case RDMA_DEV_REGISTERED: - iwdev->iw_status = 0; - i40iw_port_ibevent(iwdev); - i40iw_destroy_rdma_device(iwdev->iwibdev); - fallthrough; - case IP_ADDR_REGISTERED: - if (!iwdev->reset) - i40iw_del_macip_entry(iwdev, (u8)iwdev->mac_ip_table_idx); - fallthrough; - case PBLE_CHUNK_MEM: - i40iw_destroy_pble_pool(dev, iwdev->pble_rsrc); - fallthrough; - case CEQ_CREATED: - i40iw_dele_ceqs(iwdev); - fallthrough; - case AEQ_CREATED: - i40iw_destroy_aeq(iwdev); - fallthrough; - case IEQ_CREATED: - i40iw_puda_dele_resources(&iwdev->vsi, I40IW_PUDA_RSRC_TYPE_IEQ, iwdev->reset); - fallthrough; - case ILQ_CREATED: - i40iw_puda_dele_resources(&iwdev->vsi, I40IW_PUDA_RSRC_TYPE_ILQ, iwdev->reset); - fallthrough; - case CCQ_CREATED: - i40iw_destroy_ccq(iwdev); - fallthrough; - case HMC_OBJS_CREATED: - i40iw_del_hmc_objects(dev, dev->hmc_info, true, iwdev->reset); - fallthrough; - case CQP_CREATED: - i40iw_destroy_cqp(iwdev, true); - fallthrough; - case INITIAL_STATE: - i40iw_cleanup_cm_core(&iwdev->cm_core); - if (iwdev->vsi.pestat) { - i40iw_vsi_stats_free(&iwdev->vsi); - kfree(iwdev->vsi.pestat); - } - i40iw_del_init_mem(iwdev); - break; - case INVALID_STATE: - default: - i40iw_pr_err("bad init_state = %d\n", iwdev->init_state); - break; - } - - i40iw_del_handler(i40iw_find_i40e_handler(ldev)); - kfree(iwdev->hdl); -} - -/** - * i40iw_setup_init_state - set up the initial device struct - * @hdl: handler for iwarp device - one per instance - * @ldev: lan device information - * @client: iwarp client information, provided during registration - * - * Initialize the iwarp device and its hdl information - * using the ldev and client information - * Return 0 if successful, otherwise return error - */ -static enum i40iw_status_code i40iw_setup_init_state(struct i40iw_handler *hdl, - struct i40e_info *ldev, - struct i40e_client *client) -{ - struct i40iw_device *iwdev = &hdl->device; - struct i40iw_sc_dev *dev = &iwdev->sc_dev; - enum i40iw_status_code status; - - memcpy(&hdl->ldev, ldev, sizeof(*ldev)); - - iwdev->mpa_version = mpa_version; - iwdev->resource_profile = (resource_profile < I40IW_HMC_PROFILE_EQUAL) ? - (u8)resource_profile + I40IW_HMC_PROFILE_DEFAULT : - I40IW_HMC_PROFILE_DEFAULT; - iwdev->max_rdma_vfs = - (iwdev->resource_profile != I40IW_HMC_PROFILE_DEFAULT) ? max_rdma_vfs : 0; - iwdev->max_enabled_vfs = iwdev->max_rdma_vfs; - iwdev->netdev = ldev->netdev; - hdl->client = client; - if (!ldev->ftype) - iwdev->db_start = pci_resource_start(ldev->pcidev, 0) + I40IW_DB_ADDR_OFFSET; - else - iwdev->db_start = pci_resource_start(ldev->pcidev, 0) + I40IW_VF_DB_ADDR_OFFSET; - - status = i40iw_save_msix_info(iwdev, ldev); - if (status) - return status; - iwdev->hw.pcidev = ldev->pcidev; - iwdev->hw.hw_addr = ldev->hw_addr; - status = i40iw_allocate_dma_mem(&iwdev->hw, - &iwdev->obj_mem, 8192, 4096); - if (status) - goto exit; - iwdev->obj_next = iwdev->obj_mem; - - init_waitqueue_head(&iwdev->vchnl_waitq); - init_waitqueue_head(&dev->vf_reqs); - init_waitqueue_head(&iwdev->close_wq); - - status = i40iw_initialize_dev(iwdev, ldev); -exit: - if (status) { - kfree(iwdev->iw_msixtbl); - i40iw_free_dma_mem(dev->hw, &iwdev->obj_mem); - iwdev->iw_msixtbl = NULL; - } - return status; -} - -/** - * i40iw_get_used_rsrc - determine resources used internally - * @iwdev: iwarp device - * - * Called after internal allocations - */ -static void i40iw_get_used_rsrc(struct i40iw_device *iwdev) -{ - iwdev->used_pds = find_next_zero_bit(iwdev->allocated_pds, iwdev->max_pd, 0); - iwdev->used_qps = find_next_zero_bit(iwdev->allocated_qps, iwdev->max_qp, 0); - iwdev->used_cqs = find_next_zero_bit(iwdev->allocated_cqs, iwdev->max_cq, 0); - iwdev->used_mrs = find_next_zero_bit(iwdev->allocated_mrs, iwdev->max_mr, 0); -} - -/** - * i40iw_open - client interface operation open for iwarp/uda device - * @ldev: lan device information - * @client: iwarp client information, provided during registration - * - * Called by the lan driver during the processing of client register - * Create device resources, set up queues, pble and hmc objects and - * register the device with the ib verbs interface - * Return 0 if successful, otherwise return error - */ -static int i40iw_open(struct i40e_info *ldev, struct i40e_client *client) -{ - struct i40iw_device *iwdev; - struct i40iw_sc_dev *dev; - enum i40iw_status_code status; - struct i40iw_handler *hdl; - - hdl = i40iw_find_netdev(ldev->netdev); - if (hdl) - return 0; - - hdl = kzalloc(sizeof(*hdl), GFP_KERNEL); - if (!hdl) - return -ENOMEM; - iwdev = &hdl->device; - iwdev->hdl = hdl; - dev = &iwdev->sc_dev; - if (i40iw_setup_cm_core(iwdev)) { - kfree(iwdev->hdl); - return -ENOMEM; - } - - dev->back_dev = (void *)iwdev; - iwdev->ldev = &hdl->ldev; - iwdev->client = client; - mutex_init(&iwdev->pbl_mutex); - i40iw_add_handler(hdl); - - do { - status = i40iw_setup_init_state(hdl, ldev, client); - if (status) - break; - iwdev->init_state = INITIAL_STATE; - if (dev->is_pf) - i40iw_wait_pe_ready(dev->hw); - status = i40iw_create_cqp(iwdev); - if (status) - break; - iwdev->init_state = CQP_CREATED; - status = i40iw_hmc_setup(iwdev); - if (status) - break; - status = i40iw_create_ccq(iwdev); - if (status) - break; - iwdev->init_state = CCQ_CREATED; - status = i40iw_initialize_ilq(iwdev); - if (status) - break; - iwdev->init_state = ILQ_CREATED; - status = i40iw_initialize_ieq(iwdev); - if (status) - break; - iwdev->init_state = IEQ_CREATED; - status = i40iw_setup_aeq(iwdev); - if (status) - break; - iwdev->init_state = AEQ_CREATED; - status = i40iw_setup_ceqs(iwdev, ldev); - if (status) - break; - - status = i40iw_get_rdma_features(dev); - if (status) - dev->feature_info[I40IW_FEATURE_FW_INFO] = - I40IW_FW_VER_DEFAULT; - - iwdev->init_state = CEQ_CREATED; - status = i40iw_initialize_hw_resources(iwdev); - if (status) - break; - i40iw_get_used_rsrc(iwdev); - dev->ccq_ops->ccq_arm(dev->ccq); - status = i40iw_hmc_init_pble(&iwdev->sc_dev, iwdev->pble_rsrc); - if (status) - break; - iwdev->init_state = PBLE_CHUNK_MEM; - iwdev->virtchnl_wq = alloc_ordered_workqueue("iwvch", WQ_MEM_RECLAIM); - status = i40iw_add_mac_ip(iwdev); - if (status) - break; - iwdev->init_state = IP_ADDR_REGISTERED; - if (i40iw_register_rdma_device(iwdev)) { - i40iw_pr_err("register rdma device fail\n"); - break; - }; - - iwdev->init_state = RDMA_DEV_REGISTERED; - iwdev->iw_status = 1; - i40iw_port_ibevent(iwdev); - iwdev->param_wq = alloc_ordered_workqueue("l2params", WQ_MEM_RECLAIM); - if(iwdev->param_wq == NULL) - break; - i40iw_pr_info("i40iw_open completed\n"); - return 0; - } while (0); - - i40iw_pr_err("status = %d last completion = %d\n", status, iwdev->init_state); - i40iw_deinit_device(iwdev); - return -ERESTART; -} - -/** - * i40iw_l2params_worker - worker for l2 params change - * @work: work pointer for l2 params - */ -static void i40iw_l2params_worker(struct work_struct *work) -{ - struct l2params_work *dwork = - container_of(work, struct l2params_work, work); - struct i40iw_device *iwdev = dwork->iwdev; - - i40iw_change_l2params(&iwdev->vsi, &dwork->l2params); - atomic_dec(&iwdev->params_busy); - kfree(work); -} - -/** - * i40iw_l2param_change - handle qs handles for qos and mss change - * @ldev: lan device information - * @client: client for paramater change - * @params: new parameters from L2 - */ -static void i40iw_l2param_change(struct i40e_info *ldev, struct i40e_client *client, - struct i40e_params *params) -{ - struct i40iw_handler *hdl; - struct i40iw_l2params *l2params; - struct l2params_work *work; - struct i40iw_device *iwdev; - int i; - - hdl = i40iw_find_i40e_handler(ldev); - if (!hdl) - return; - - iwdev = &hdl->device; - - if (atomic_read(&iwdev->params_busy)) - return; - - - work = kzalloc(sizeof(*work), GFP_KERNEL); - if (!work) - return; - - atomic_inc(&iwdev->params_busy); - - work->iwdev = iwdev; - l2params = &work->l2params; - for (i = 0; i < I40E_CLIENT_MAX_USER_PRIORITY; i++) - l2params->qs_handle_list[i] = params->qos.prio_qos[i].qs_handle; - - l2params->mtu = (params->mtu) ? params->mtu : iwdev->vsi.mtu; - - INIT_WORK(&work->work, i40iw_l2params_worker); - queue_work(iwdev->param_wq, &work->work); -} - -/** - * i40iw_close - client interface operation close for iwarp/uda device - * @ldev: lan device information - * @reset: true if called before reset - * @client: client to close - * - * Called by the lan driver during the processing of client unregister - * Destroy and clean up the driver resources - */ -static void i40iw_close(struct i40e_info *ldev, struct i40e_client *client, bool reset) -{ - struct i40iw_device *iwdev; - struct i40iw_handler *hdl; - - hdl = i40iw_find_i40e_handler(ldev); - if (!hdl) - return; - - iwdev = &hdl->device; - iwdev->closing = true; - - if (reset) - iwdev->reset = true; - - i40iw_cm_teardown_connections(iwdev, NULL, NULL, true); - destroy_workqueue(iwdev->virtchnl_wq); - i40iw_deinit_device(iwdev); -} - -/** - * i40iw_vf_reset - process VF reset - * @ldev: lan device information - * @client: client interface instance - * @vf_id: virtual function id - * - * Called when a VF is reset by the PF - * Destroy and clean up the VF resources - */ -static void i40iw_vf_reset(struct i40e_info *ldev, struct i40e_client *client, u32 vf_id) -{ - struct i40iw_handler *hdl; - struct i40iw_sc_dev *dev; - struct i40iw_hmc_fcn_info hmc_fcn_info; - struct i40iw_virt_mem vf_dev_mem; - struct i40iw_vfdev *tmp_vfdev; - unsigned int i; - unsigned long flags; - struct i40iw_device *iwdev; - - hdl = i40iw_find_i40e_handler(ldev); - if (!hdl) - return; - - dev = &hdl->device.sc_dev; - iwdev = (struct i40iw_device *)dev->back_dev; - - for (i = 0; i < I40IW_MAX_PE_ENABLED_VF_COUNT; i++) { - if (!dev->vf_dev[i] || (dev->vf_dev[i]->vf_id != vf_id)) - continue; - /* free all resources allocated on behalf of vf */ - tmp_vfdev = dev->vf_dev[i]; - spin_lock_irqsave(&iwdev->vsi.pestat->lock, flags); - dev->vf_dev[i] = NULL; - spin_unlock_irqrestore(&iwdev->vsi.pestat->lock, flags); - i40iw_del_hmc_objects(dev, &tmp_vfdev->hmc_info, false, false); - /* remove vf hmc function */ - memset(&hmc_fcn_info, 0, sizeof(hmc_fcn_info)); - hmc_fcn_info.vf_id = vf_id; - hmc_fcn_info.iw_vf_idx = tmp_vfdev->iw_vf_idx; - hmc_fcn_info.free_fcn = true; - i40iw_cqp_manage_hmc_fcn_cmd(dev, &hmc_fcn_info); - /* free vf_dev */ - vf_dev_mem.va = tmp_vfdev; - vf_dev_mem.size = sizeof(struct i40iw_vfdev) + - sizeof(struct i40iw_hmc_obj_info) * I40IW_HMC_IW_MAX; - i40iw_free_virt_mem(dev->hw, &vf_dev_mem); - break; - } -} - -/** - * i40iw_vf_enable - enable a number of VFs - * @ldev: lan device information - * @client: client interface instance - * @num_vfs: number of VFs for the PF - * - * Called when the number of VFs changes - */ -static void i40iw_vf_enable(struct i40e_info *ldev, - struct i40e_client *client, - u32 num_vfs) -{ - struct i40iw_handler *hdl; - - hdl = i40iw_find_i40e_handler(ldev); - if (!hdl) - return; - - if (num_vfs > I40IW_MAX_PE_ENABLED_VF_COUNT) - hdl->device.max_enabled_vfs = I40IW_MAX_PE_ENABLED_VF_COUNT; - else - hdl->device.max_enabled_vfs = num_vfs; -} - -/** - * i40iw_vf_capable - check if VF capable - * @ldev: lan device information - * @client: client interface instance - * @vf_id: virtual function id - * - * Return 1 if a VF slot is available or if VF is already RDMA enabled - * Return 0 otherwise - */ -static int i40iw_vf_capable(struct i40e_info *ldev, - struct i40e_client *client, - u32 vf_id) -{ - struct i40iw_handler *hdl; - struct i40iw_sc_dev *dev; - unsigned int i; - - hdl = i40iw_find_i40e_handler(ldev); - if (!hdl) - return 0; - - dev = &hdl->device.sc_dev; - - for (i = 0; i < hdl->device.max_enabled_vfs; i++) { - if (!dev->vf_dev[i] || (dev->vf_dev[i]->vf_id == vf_id)) - return 1; - } - - return 0; -} - -/** - * i40iw_virtchnl_receive - receive a message through the virtual channel - * @ldev: lan device information - * @client: client interface instance - * @vf_id: virtual function id associated with the message - * @msg: message buffer pointer - * @len: length of the message - * - * Invoke virtual channel receive operation for the given msg - * Return 0 if successful, otherwise return error - */ -static int i40iw_virtchnl_receive(struct i40e_info *ldev, - struct i40e_client *client, - u32 vf_id, - u8 *msg, - u16 len) -{ - struct i40iw_handler *hdl; - struct i40iw_sc_dev *dev; - struct i40iw_device *iwdev; - int ret_code = I40IW_NOT_SUPPORTED; - - if (!len || !msg) - return I40IW_ERR_PARAM; - - hdl = i40iw_find_i40e_handler(ldev); - if (!hdl) - return I40IW_ERR_PARAM; - - dev = &hdl->device.sc_dev; - iwdev = dev->back_dev; - - if (dev->vchnl_if.vchnl_recv) { - ret_code = dev->vchnl_if.vchnl_recv(dev, vf_id, msg, len); - if (!dev->is_pf) { - atomic_dec(&iwdev->vchnl_msgs); - wake_up(&iwdev->vchnl_waitq); - } - } - return ret_code; -} - -/** - * i40iw_vf_clear_to_send - wait to send virtual channel message - * @dev: iwarp device * - * Wait for until virtual channel is clear - * before sending the next message - * - * Returns false if error - * Returns true if clear to send - */ -bool i40iw_vf_clear_to_send(struct i40iw_sc_dev *dev) -{ - struct i40iw_device *iwdev; - wait_queue_entry_t wait; - - iwdev = dev->back_dev; - - if (!wq_has_sleeper(&dev->vf_reqs) && - (atomic_read(&iwdev->vchnl_msgs) == 0)) - return true; /* virtual channel is clear */ - - init_wait(&wait); - add_wait_queue_exclusive(&dev->vf_reqs, &wait); - - if (!wait_event_timeout(dev->vf_reqs, - (atomic_read(&iwdev->vchnl_msgs) == 0), - I40IW_VCHNL_EVENT_TIMEOUT)) - dev->vchnl_up = false; - - remove_wait_queue(&dev->vf_reqs, &wait); - - return dev->vchnl_up; -} - -/** - * i40iw_virtchnl_send - send a message through the virtual channel - * @dev: iwarp device - * @vf_id: virtual function id associated with the message - * @msg: virtual channel message buffer pointer - * @len: length of the message - * - * Invoke virtual channel send operation for the given msg - * Return 0 if successful, otherwise return error - */ -static enum i40iw_status_code i40iw_virtchnl_send(struct i40iw_sc_dev *dev, - u32 vf_id, - u8 *msg, - u16 len) -{ - struct i40iw_device *iwdev; - struct i40e_info *ldev; - - if (!dev || !dev->back_dev) - return I40IW_ERR_BAD_PTR; - - iwdev = dev->back_dev; - ldev = iwdev->ldev; - - if (ldev && ldev->ops && ldev->ops->virtchnl_send) - return ldev->ops->virtchnl_send(ldev, &i40iw_client, vf_id, msg, len); - return I40IW_ERR_BAD_PTR; -} - -/* client interface functions */ -static const struct i40e_client_ops i40e_ops = { - .open = i40iw_open, - .close = i40iw_close, - .l2_param_change = i40iw_l2param_change, - .virtchnl_receive = i40iw_virtchnl_receive, - .vf_reset = i40iw_vf_reset, - .vf_enable = i40iw_vf_enable, - .vf_capable = i40iw_vf_capable -}; - -/** - * i40iw_init_module - driver initialization function - * - * First function to call when the driver is loaded - * Register the driver as i40e client and port mapper client - */ -static int __init i40iw_init_module(void) -{ - int ret; - - memset(&i40iw_client, 0, sizeof(i40iw_client)); - i40iw_client.version.major = CLIENT_IW_INTERFACE_VERSION_MAJOR; - i40iw_client.version.minor = CLIENT_IW_INTERFACE_VERSION_MINOR; - i40iw_client.version.build = CLIENT_IW_INTERFACE_VERSION_BUILD; - i40iw_client.ops = &i40e_ops; - memcpy(i40iw_client.name, i40iw_client_name, I40E_CLIENT_STR_LENGTH); - i40iw_client.type = I40E_CLIENT_IWARP; - ret = i40e_register_client(&i40iw_client); - i40iw_register_notifiers(); - - return ret; -} - -/** - * i40iw_exit_module - driver exit clean up function - * - * The function is called just before the driver is unloaded - * Unregister the driver as i40e client and port mapper client - */ -static void __exit i40iw_exit_module(void) -{ - i40iw_unregister_notifiers(); - i40e_unregister_client(&i40iw_client); -} - -module_init(i40iw_init_module); -module_exit(i40iw_exit_module); diff --git a/drivers/infiniband/hw/i40iw/i40iw_osdep.h b/drivers/infiniband/hw/i40iw/i40iw_osdep.h deleted file mode 100644 index d938ccb195b1..000000000000 --- a/drivers/infiniband/hw/i40iw/i40iw_osdep.h +++ /dev/null @@ -1,195 +0,0 @@ -/******************************************************************************* -* -* Copyright (c) 2015-2016 Intel Corporation. All rights reserved. -* -* This software is available to you under a choice of one of two -* licenses. You may choose to be licensed under the terms of the GNU -* General Public License (GPL) Version 2, available from the file -* COPYING in the main directory of this source tree, or the -* OpenFabrics.org BSD license below: -* -* Redistribution and use in source and binary forms, with or -* without modification, are permitted provided that the following -* conditions are met: -* -* - Redistributions of source code must retain the above -* copyright notice, this list of conditions and the following -* disclaimer. -* -* - Redistributions in binary form must reproduce the above -* copyright notice, this list of conditions and the following -* disclaimer in the documentation and/or other materials -* provided with the distribution. -* -* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS -* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN -* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -* SOFTWARE. -* -*******************************************************************************/ - -#ifndef I40IW_OSDEP_H -#define I40IW_OSDEP_H - -#include -#include -#include -#include -#include -/* get readq/writeq support for 32 bit kernels, use the low-first version */ -#include - -#define STATS_TIMER_DELAY 1000 - -static inline void set_64bit_val(u64 *wqe_words, u32 byte_index, u64 value) -{ - wqe_words[byte_index >> 3] = value; -} - -/** - * get_64bit_val - read 64 bit value from wqe - * @wqe_words: wqe addr - * @byte_index: index to read from - * @value: read value - **/ -static inline void get_64bit_val(u64 *wqe_words, u32 byte_index, u64 *value) -{ - *value = wqe_words[byte_index >> 3]; -} - -struct i40iw_dma_mem { - void *va; - dma_addr_t pa; - u32 size; -} __packed; - -struct i40iw_virt_mem { - void *va; - u32 size; -} __packed; - -#define i40iw_debug(h, m, s, ...) \ -do { \ - if (((m) & (h)->debug_mask)) \ - pr_info("i40iw " s, ##__VA_ARGS__); \ -} while (0) - -#define i40iw_flush(a) readl((a)->hw_addr + I40E_GLGEN_STAT) - -#define I40E_GLHMC_VFSDCMD(_i) (0x000C8000 + ((_i) * 4)) \ - /* _i=0...31 */ -#define I40E_GLHMC_VFSDCMD_MAX_INDEX 31 -#define I40E_GLHMC_VFSDCMD_PMSDIDX_SHIFT 0 -#define I40E_GLHMC_VFSDCMD_PMSDIDX_MASK (0xFFF \ - << I40E_GLHMC_VFSDCMD_PMSDIDX_SHIFT) -#define I40E_GLHMC_VFSDCMD_PF_SHIFT 16 -#define I40E_GLHMC_VFSDCMD_PF_MASK (0xF << I40E_GLHMC_VFSDCMD_PF_SHIFT) -#define I40E_GLHMC_VFSDCMD_VF_SHIFT 20 -#define I40E_GLHMC_VFSDCMD_VF_MASK (0x1FF << I40E_GLHMC_VFSDCMD_VF_SHIFT) -#define I40E_GLHMC_VFSDCMD_PMF_TYPE_SHIFT 29 -#define I40E_GLHMC_VFSDCMD_PMF_TYPE_MASK (0x3 \ - << I40E_GLHMC_VFSDCMD_PMF_TYPE_SHIFT) -#define I40E_GLHMC_VFSDCMD_PMSDWR_SHIFT 31 -#define I40E_GLHMC_VFSDCMD_PMSDWR_MASK (0x1 << I40E_GLHMC_VFSDCMD_PMSDWR_SHIFT) - -#define I40E_GLHMC_VFSDDATAHIGH(_i) (0x000C8200 + ((_i) * 4)) \ - /* _i=0...31 */ -#define I40E_GLHMC_VFSDDATAHIGH_MAX_INDEX 31 -#define I40E_GLHMC_VFSDDATAHIGH_PMSDDATAHIGH_SHIFT 0 -#define I40E_GLHMC_VFSDDATAHIGH_PMSDDATAHIGH_MASK (0xFFFFFFFF \ - << I40E_GLHMC_VFSDDATAHIGH_PMSDDATAHIGH_SHIFT) - -#define I40E_GLHMC_VFSDDATALOW(_i) (0x000C8100 + ((_i) * 4)) \ - /* _i=0...31 */ -#define I40E_GLHMC_VFSDDATALOW_MAX_INDEX 31 -#define I40E_GLHMC_VFSDDATALOW_PMSDVALID_SHIFT 0 -#define I40E_GLHMC_VFSDDATALOW_PMSDVALID_MASK (0x1 \ - << I40E_GLHMC_VFSDDATALOW_PMSDVALID_SHIFT) -#define I40E_GLHMC_VFSDDATALOW_PMSDTYPE_SHIFT 1 -#define I40E_GLHMC_VFSDDATALOW_PMSDTYPE_MASK (0x1 \ - << I40E_GLHMC_VFSDDATALOW_PMSDTYPE_SHIFT) -#define I40E_GLHMC_VFSDDATALOW_PMSDBPCOUNT_SHIFT 2 -#define I40E_GLHMC_VFSDDATALOW_PMSDBPCOUNT_MASK (0x3FF \ - << I40E_GLHMC_VFSDDATALOW_PMSDBPCOUNT_SHIFT) -#define I40E_GLHMC_VFSDDATALOW_PMSDDATALOW_SHIFT 12 -#define I40E_GLHMC_VFSDDATALOW_PMSDDATALOW_MASK (0xFFFFF \ - << I40E_GLHMC_VFSDDATALOW_PMSDDATALOW_SHIFT) - -#define I40E_GLPE_FWLDSTATUS 0x0000D200 -#define I40E_GLPE_FWLDSTATUS_LOAD_REQUESTED_SHIFT 0 -#define I40E_GLPE_FWLDSTATUS_LOAD_REQUESTED_MASK (0x1 \ - << I40E_GLPE_FWLDSTATUS_LOAD_REQUESTED_SHIFT) -#define I40E_GLPE_FWLDSTATUS_DONE_SHIFT 1 -#define I40E_GLPE_FWLDSTATUS_DONE_MASK (0x1 << I40E_GLPE_FWLDSTATUS_DONE_SHIFT) -#define I40E_GLPE_FWLDSTATUS_CQP_FAIL_SHIFT 2 -#define I40E_GLPE_FWLDSTATUS_CQP_FAIL_MASK (0x1 \ - << I40E_GLPE_FWLDSTATUS_CQP_FAIL_SHIFT) -#define I40E_GLPE_FWLDSTATUS_TEP_FAIL_SHIFT 3 -#define I40E_GLPE_FWLDSTATUS_TEP_FAIL_MASK (0x1 \ - << I40E_GLPE_FWLDSTATUS_TEP_FAIL_SHIFT) -#define I40E_GLPE_FWLDSTATUS_OOP_FAIL_SHIFT 4 -#define I40E_GLPE_FWLDSTATUS_OOP_FAIL_MASK (0x1 \ - << I40E_GLPE_FWLDSTATUS_OOP_FAIL_SHIFT) - -struct i40iw_sc_dev; -struct i40iw_sc_qp; -struct i40iw_puda_buf; -struct i40iw_puda_completion_info; -struct i40iw_update_sds_info; -struct i40iw_hmc_fcn_info; -struct i40iw_virtchnl_work_info; -struct i40iw_manage_vf_pble_info; -struct i40iw_device; -struct i40iw_hmc_info; -struct i40iw_hw; - -u8 __iomem *i40iw_get_hw_addr(void *dev); -void i40iw_ieq_mpa_crc_ae(struct i40iw_sc_dev *dev, struct i40iw_sc_qp *qp); -enum i40iw_status_code i40iw_vf_wait_vchnl_resp(struct i40iw_sc_dev *dev); -bool i40iw_vf_clear_to_send(struct i40iw_sc_dev *dev); -enum i40iw_status_code i40iw_ieq_check_mpacrc(struct shash_desc *desc, void *addr, - u32 length, u32 value); -struct i40iw_sc_qp *i40iw_ieq_get_qp(struct i40iw_sc_dev *dev, struct i40iw_puda_buf *buf); -void i40iw_ieq_update_tcpip_info(struct i40iw_puda_buf *buf, u16 length, u32 seqnum); -void i40iw_free_hash_desc(struct shash_desc *); -enum i40iw_status_code i40iw_init_hash_desc(struct shash_desc **); -enum i40iw_status_code i40iw_puda_get_tcpip_info(struct i40iw_puda_completion_info *info, - struct i40iw_puda_buf *buf); -enum i40iw_status_code i40iw_cqp_sds_cmd(struct i40iw_sc_dev *dev, - struct i40iw_update_sds_info *info); -enum i40iw_status_code i40iw_cqp_manage_hmc_fcn_cmd(struct i40iw_sc_dev *dev, - struct i40iw_hmc_fcn_info *hmcfcninfo); -enum i40iw_status_code i40iw_cqp_query_fpm_values_cmd(struct i40iw_sc_dev *dev, - struct i40iw_dma_mem *values_mem, - u8 hmc_fn_id); -enum i40iw_status_code i40iw_cqp_commit_fpm_values_cmd(struct i40iw_sc_dev *dev, - struct i40iw_dma_mem *values_mem, - u8 hmc_fn_id); -enum i40iw_status_code i40iw_alloc_query_fpm_buf(struct i40iw_sc_dev *dev, - struct i40iw_dma_mem *mem); -enum i40iw_status_code i40iw_cqp_manage_vf_pble_bp(struct i40iw_sc_dev *dev, - struct i40iw_manage_vf_pble_info *info); -void i40iw_cqp_spawn_worker(struct i40iw_sc_dev *dev, - struct i40iw_virtchnl_work_info *work_info, u32 iw_vf_idx); -void *i40iw_remove_head(struct list_head *list); -void i40iw_qp_suspend_resume(struct i40iw_sc_dev *dev, struct i40iw_sc_qp *qp, bool suspend); - -void i40iw_term_modify_qp(struct i40iw_sc_qp *qp, u8 next_state, u8 term, u8 term_len); -void i40iw_terminate_done(struct i40iw_sc_qp *qp, int timeout_occurred); -void i40iw_terminate_start_timer(struct i40iw_sc_qp *qp); -void i40iw_terminate_del_timer(struct i40iw_sc_qp *qp); - -enum i40iw_status_code i40iw_hw_manage_vf_pble_bp(struct i40iw_device *iwdev, - struct i40iw_manage_vf_pble_info *info, - bool wait); -struct i40iw_sc_vsi; -void i40iw_hw_stats_start_timer(struct i40iw_sc_vsi *vsi); -void i40iw_hw_stats_stop_timer(struct i40iw_sc_vsi *vsi); -#define i40iw_mmiowb() do { } while (0) -void i40iw_wr32(struct i40iw_hw *hw, u32 reg, u32 value); -u32 i40iw_rd32(struct i40iw_hw *hw, u32 reg); -#endif /* _I40IW_OSDEP_H_ */ diff --git a/drivers/infiniband/hw/i40iw/i40iw_p.h b/drivers/infiniband/hw/i40iw/i40iw_p.h deleted file mode 100644 index 4c429567bbb4..000000000000 --- a/drivers/infiniband/hw/i40iw/i40iw_p.h +++ /dev/null @@ -1,129 +0,0 @@ -/******************************************************************************* -* -* Copyright (c) 2015-2016 Intel Corporation. All rights reserved. -* -* This software is available to you under a choice of one of two -* licenses. You may choose to be licensed under the terms of the GNU -* General Public License (GPL) Version 2, available from the file -* COPYING in the main directory of this source tree, or the -* OpenFabrics.org BSD license below: -* -* Redistribution and use in source and binary forms, with or -* without modification, are permitted provided that the following -* conditions are met: -* -* - Redistributions of source code must retain the above -* copyright notice, this list of conditions and the following -* disclaimer. -* -* - Redistributions in binary form must reproduce the above -* copyright notice, this list of conditions and the following -* disclaimer in the documentation and/or other materials -* provided with the distribution. -* -* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS -* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN -* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -* SOFTWARE. -* -*******************************************************************************/ - -#ifndef I40IW_P_H -#define I40IW_P_H - -#define PAUSE_TIMER_VALUE 0xFFFF -#define REFRESH_THRESHOLD 0x7FFF -#define HIGH_THRESHOLD 0x800 -#define LOW_THRESHOLD 0x200 -#define ALL_TC2PFC 0xFF -#define CQP_COMPL_WAIT_TIME 0x3E8 -#define CQP_TIMEOUT_THRESHOLD 5 - -void i40iw_debug_buf(struct i40iw_sc_dev *dev, enum i40iw_debug_flag mask, - char *desc, u64 *buf, u32 size); -/* init operations */ -enum i40iw_status_code i40iw_device_init(struct i40iw_sc_dev *dev, - struct i40iw_device_init_info *info); - -void i40iw_sc_cqp_post_sq(struct i40iw_sc_cqp *cqp); - -u64 *i40iw_sc_cqp_get_next_send_wqe(struct i40iw_sc_cqp *cqp, u64 scratch); - -void i40iw_check_cqp_progress(struct i40iw_cqp_timeout *cqp_timeout, struct i40iw_sc_dev *dev); - -enum i40iw_status_code i40iw_sc_mr_fast_register(struct i40iw_sc_qp *qp, - struct i40iw_fast_reg_stag_info *info, - bool post_sq); - -void i40iw_insert_wqe_hdr(u64 *wqe, u64 header); - -/* HMC/FPM functions */ -enum i40iw_status_code i40iw_sc_init_iw_hmc(struct i40iw_sc_dev *dev, - u8 hmc_fn_id); - -enum i40iw_status_code i40iw_pf_init_vfhmc(struct i40iw_sc_dev *dev, u8 vf_hmc_fn_id, - u32 *vf_cnt_array); - -/* stats functions */ -void i40iw_hw_stats_refresh_all(struct i40iw_vsi_pestat *stats); -void i40iw_hw_stats_read_all(struct i40iw_vsi_pestat *stats, struct i40iw_dev_hw_stats *stats_values); -void i40iw_hw_stats_read_32(struct i40iw_vsi_pestat *stats, - enum i40iw_hw_stats_index_32b index, - u64 *value); -void i40iw_hw_stats_read_64(struct i40iw_vsi_pestat *stats, - enum i40iw_hw_stats_index_64b index, - u64 *value); -void i40iw_hw_stats_init(struct i40iw_vsi_pestat *stats, u8 index, bool is_pf); - -/* vsi misc functions */ -enum i40iw_status_code i40iw_vsi_stats_init(struct i40iw_sc_vsi *vsi, struct i40iw_vsi_stats_info *info); -void i40iw_vsi_stats_free(struct i40iw_sc_vsi *vsi); -void i40iw_sc_vsi_init(struct i40iw_sc_vsi *vsi, struct i40iw_vsi_init_info *info); - -void i40iw_change_l2params(struct i40iw_sc_vsi *vsi, struct i40iw_l2params *l2params); -void i40iw_qp_add_qos(struct i40iw_sc_qp *qp); -void i40iw_qp_rem_qos(struct i40iw_sc_qp *qp); -void i40iw_terminate_send_fin(struct i40iw_sc_qp *qp); - -void i40iw_terminate_connection(struct i40iw_sc_qp *qp, struct i40iw_aeqe_info *info); - -void i40iw_terminate_received(struct i40iw_sc_qp *qp, struct i40iw_aeqe_info *info); - -enum i40iw_status_code i40iw_sc_suspend_qp(struct i40iw_sc_cqp *cqp, - struct i40iw_sc_qp *qp, u64 scratch); - -enum i40iw_status_code i40iw_sc_resume_qp(struct i40iw_sc_cqp *cqp, - struct i40iw_sc_qp *qp, u64 scratch); - -enum i40iw_status_code i40iw_sc_static_hmc_pages_allocated(struct i40iw_sc_cqp *cqp, - u64 scratch, u8 hmc_fn_id, - bool post_sq, - bool poll_registers); - -enum i40iw_status_code i40iw_config_fpm_values(struct i40iw_sc_dev *dev, u32 qp_count); -enum i40iw_status_code i40iw_get_rdma_features(struct i40iw_sc_dev *dev); - -void free_sd_mem(struct i40iw_sc_dev *dev); - -enum i40iw_status_code i40iw_process_cqp_cmd(struct i40iw_sc_dev *dev, - struct cqp_commands_info *pcmdinfo); - -enum i40iw_status_code i40iw_process_bh(struct i40iw_sc_dev *dev); - -/* prototype for functions used for dynamic memory allocation */ -enum i40iw_status_code i40iw_allocate_dma_mem(struct i40iw_hw *hw, - struct i40iw_dma_mem *mem, u64 size, - u32 alignment); -void i40iw_free_dma_mem(struct i40iw_hw *hw, struct i40iw_dma_mem *mem); -enum i40iw_status_code i40iw_allocate_virt_mem(struct i40iw_hw *hw, - struct i40iw_virt_mem *mem, u32 size); -enum i40iw_status_code i40iw_free_virt_mem(struct i40iw_hw *hw, - struct i40iw_virt_mem *mem); -u8 i40iw_get_encoded_wqe_size(u32 wqsize, bool cqpsq); -void i40iw_reinitialize_ieq(struct i40iw_sc_dev *dev); - -#endif diff --git a/drivers/infiniband/hw/i40iw/i40iw_pble.c b/drivers/infiniband/hw/i40iw/i40iw_pble.c deleted file mode 100644 index 146a4148219b..000000000000 --- a/drivers/infiniband/hw/i40iw/i40iw_pble.c +++ /dev/null @@ -1,611 +0,0 @@ -/******************************************************************************* -* -* Copyright (c) 2015-2016 Intel Corporation. All rights reserved. -* -* This software is available to you under a choice of one of two -* licenses. You may choose to be licensed under the terms of the GNU -* General Public License (GPL) Version 2, available from the file -* COPYING in the main directory of this source tree, or the -* OpenFabrics.org BSD license below: -* -* Redistribution and use in source and binary forms, with or -* without modification, are permitted provided that the following -* conditions are met: -* -* - Redistributions of source code must retain the above -* copyright notice, this list of conditions and the following -* disclaimer. -* -* - Redistributions in binary form must reproduce the above -* copyright notice, this list of conditions and the following -* disclaimer in the documentation and/or other materials -* provided with the distribution. -* -* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS -* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN -* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -* SOFTWARE. -* -*******************************************************************************/ - -#include "i40iw_status.h" -#include "i40iw_osdep.h" -#include "i40iw_register.h" -#include "i40iw_hmc.h" - -#include "i40iw_d.h" -#include "i40iw_type.h" -#include "i40iw_p.h" - -#include -#include -#include -#include "i40iw_pble.h" -#include "i40iw.h" - -struct i40iw_device; -static enum i40iw_status_code add_pble_pool(struct i40iw_sc_dev *dev, - struct i40iw_hmc_pble_rsrc *pble_rsrc); -static void i40iw_free_vmalloc_mem(struct i40iw_hw *hw, struct i40iw_chunk *chunk); - -/** - * i40iw_destroy_pble_pool - destroy pool during module unload - * @dev: i40iw_sc_dev struct - * @pble_rsrc: pble resources - */ -void i40iw_destroy_pble_pool(struct i40iw_sc_dev *dev, struct i40iw_hmc_pble_rsrc *pble_rsrc) -{ - struct list_head *clist; - struct list_head *tlist; - struct i40iw_chunk *chunk; - struct i40iw_pble_pool *pinfo = &pble_rsrc->pinfo; - - if (pinfo->pool) { - list_for_each_safe(clist, tlist, &pinfo->clist) { - chunk = list_entry(clist, struct i40iw_chunk, list); - if (chunk->type == I40IW_VMALLOC) - i40iw_free_vmalloc_mem(dev->hw, chunk); - kfree(chunk); - } - gen_pool_destroy(pinfo->pool); - } -} - -/** - * i40iw_hmc_init_pble - Initialize pble resources during module load - * @dev: i40iw_sc_dev struct - * @pble_rsrc: pble resources - */ -enum i40iw_status_code i40iw_hmc_init_pble(struct i40iw_sc_dev *dev, - struct i40iw_hmc_pble_rsrc *pble_rsrc) -{ - struct i40iw_hmc_info *hmc_info; - u32 fpm_idx = 0; - - hmc_info = dev->hmc_info; - pble_rsrc->fpm_base_addr = hmc_info->hmc_obj[I40IW_HMC_IW_PBLE].base; - /* Now start the pble' on 4k boundary */ - if (pble_rsrc->fpm_base_addr & 0xfff) - fpm_idx = (PAGE_SIZE - (pble_rsrc->fpm_base_addr & 0xfff)) >> 3; - - pble_rsrc->unallocated_pble = - hmc_info->hmc_obj[I40IW_HMC_IW_PBLE].cnt - fpm_idx; - pble_rsrc->next_fpm_addr = pble_rsrc->fpm_base_addr + (fpm_idx << 3); - - pble_rsrc->pinfo.pool_shift = POOL_SHIFT; - pble_rsrc->pinfo.pool = gen_pool_create(pble_rsrc->pinfo.pool_shift, -1); - INIT_LIST_HEAD(&pble_rsrc->pinfo.clist); - if (!pble_rsrc->pinfo.pool) - goto error; - - if (add_pble_pool(dev, pble_rsrc)) - goto error; - - return 0; - - error:i40iw_destroy_pble_pool(dev, pble_rsrc); - return I40IW_ERR_NO_MEMORY; -} - -/** - * get_sd_pd_idx - Returns sd index, pd index and rel_pd_idx from fpm address - * @pble_rsrc: structure containing fpm address - * @idx: where to return indexes - */ -static inline void get_sd_pd_idx(struct i40iw_hmc_pble_rsrc *pble_rsrc, - struct sd_pd_idx *idx) -{ - idx->sd_idx = (u32)(pble_rsrc->next_fpm_addr) / I40IW_HMC_DIRECT_BP_SIZE; - idx->pd_idx = (u32)(pble_rsrc->next_fpm_addr) / I40IW_HMC_PAGED_BP_SIZE; - idx->rel_pd_idx = (idx->pd_idx % I40IW_HMC_PD_CNT_IN_SD); -} - -/** - * add_sd_direct - add sd direct for pble - * @dev: hardware control device structure - * @pble_rsrc: pble resource ptr - * @info: page info for sd - */ -static enum i40iw_status_code add_sd_direct(struct i40iw_sc_dev *dev, - struct i40iw_hmc_pble_rsrc *pble_rsrc, - struct i40iw_add_page_info *info) -{ - enum i40iw_status_code ret_code = 0; - struct sd_pd_idx *idx = &info->idx; - struct i40iw_chunk *chunk = info->chunk; - struct i40iw_hmc_info *hmc_info = info->hmc_info; - struct i40iw_hmc_sd_entry *sd_entry = info->sd_entry; - u32 offset = 0; - - if (!sd_entry->valid) { - if (dev->is_pf) { - ret_code = i40iw_add_sd_table_entry(dev->hw, hmc_info, - info->idx.sd_idx, - I40IW_SD_TYPE_DIRECT, - I40IW_HMC_DIRECT_BP_SIZE); - if (ret_code) - return ret_code; - chunk->type = I40IW_DMA_COHERENT; - } - } - offset = idx->rel_pd_idx << I40IW_HMC_PAGED_BP_SHIFT; - chunk->size = info->pages << I40IW_HMC_PAGED_BP_SHIFT; - chunk->vaddr = ((u8 *)sd_entry->u.bp.addr.va + offset); - chunk->fpm_addr = pble_rsrc->next_fpm_addr; - i40iw_debug(dev, I40IW_DEBUG_PBLE, "chunk_size[%d] = 0x%x vaddr=%p fpm_addr = %llx\n", - chunk->size, chunk->size, chunk->vaddr, chunk->fpm_addr); - return 0; -} - -/** - * i40iw_free_vmalloc_mem - free vmalloc during close - * @hw: hw struct - * @chunk: chunk information for vmalloc - */ -static void i40iw_free_vmalloc_mem(struct i40iw_hw *hw, struct i40iw_chunk *chunk) -{ - struct pci_dev *pcidev = hw->pcidev; - int i; - - if (!chunk->pg_cnt) - goto done; - for (i = 0; i < chunk->pg_cnt; i++) - dma_unmap_page(&pcidev->dev, chunk->dmaaddrs[i], PAGE_SIZE, DMA_BIDIRECTIONAL); - - done: - kfree(chunk->dmaaddrs); - chunk->dmaaddrs = NULL; - vfree(chunk->vaddr); - chunk->vaddr = NULL; - chunk->type = 0; -} - -/** - * i40iw_get_vmalloc_mem - get 2M page for sd - * @hw: hardware address - * @chunk: chunk to adf - * @pg_cnt: #of 4 K pages - */ -static enum i40iw_status_code i40iw_get_vmalloc_mem(struct i40iw_hw *hw, - struct i40iw_chunk *chunk, - int pg_cnt) -{ - struct pci_dev *pcidev = hw->pcidev; - struct page *page; - u8 *addr; - u32 size; - int i; - - chunk->dmaaddrs = kzalloc(pg_cnt << 3, GFP_KERNEL); - if (!chunk->dmaaddrs) - return I40IW_ERR_NO_MEMORY; - size = PAGE_SIZE * pg_cnt; - chunk->vaddr = vmalloc(size); - if (!chunk->vaddr) { - kfree(chunk->dmaaddrs); - chunk->dmaaddrs = NULL; - return I40IW_ERR_NO_MEMORY; - } - chunk->size = size; - addr = (u8 *)chunk->vaddr; - for (i = 0; i < pg_cnt; i++) { - page = vmalloc_to_page((void *)addr); - if (!page) - break; - chunk->dmaaddrs[i] = dma_map_page(&pcidev->dev, page, 0, - PAGE_SIZE, DMA_BIDIRECTIONAL); - if (dma_mapping_error(&pcidev->dev, chunk->dmaaddrs[i])) - break; - addr += PAGE_SIZE; - } - - chunk->pg_cnt = i; - chunk->type = I40IW_VMALLOC; - if (i == pg_cnt) - return 0; - - i40iw_free_vmalloc_mem(hw, chunk); - return I40IW_ERR_NO_MEMORY; -} - -/** - * fpm_to_idx - given fpm address, get pble index - * @pble_rsrc: pble resource management - * @addr: fpm address for index - */ -static inline u32 fpm_to_idx(struct i40iw_hmc_pble_rsrc *pble_rsrc, u64 addr) -{ - return (addr - (pble_rsrc->fpm_base_addr)) >> 3; -} - -/** - * add_bp_pages - add backing pages for sd - * @dev: hardware control device structure - * @pble_rsrc: pble resource management - * @info: page info for sd - */ -static enum i40iw_status_code add_bp_pages(struct i40iw_sc_dev *dev, - struct i40iw_hmc_pble_rsrc *pble_rsrc, - struct i40iw_add_page_info *info) -{ - u8 *addr; - struct i40iw_dma_mem mem; - struct i40iw_hmc_pd_entry *pd_entry; - struct i40iw_hmc_sd_entry *sd_entry = info->sd_entry; - struct i40iw_hmc_info *hmc_info = info->hmc_info; - struct i40iw_chunk *chunk = info->chunk; - struct i40iw_manage_vf_pble_info vf_pble_info; - enum i40iw_status_code status = 0; - u32 rel_pd_idx = info->idx.rel_pd_idx; - u32 pd_idx = info->idx.pd_idx; - u32 i; - - status = i40iw_get_vmalloc_mem(dev->hw, chunk, info->pages); - if (status) - return I40IW_ERR_NO_MEMORY; - status = i40iw_add_sd_table_entry(dev->hw, hmc_info, - info->idx.sd_idx, I40IW_SD_TYPE_PAGED, - I40IW_HMC_DIRECT_BP_SIZE); - if (status) - goto error; - if (!dev->is_pf) { - status = i40iw_vchnl_vf_add_hmc_objs(dev, I40IW_HMC_IW_PBLE, - fpm_to_idx(pble_rsrc, - pble_rsrc->next_fpm_addr), - (info->pages << PBLE_512_SHIFT)); - if (status) { - i40iw_pr_err("allocate PBLEs in the PF. Error %i\n", status); - goto error; - } - } - addr = chunk->vaddr; - for (i = 0; i < info->pages; i++) { - mem.pa = chunk->dmaaddrs[i]; - mem.size = PAGE_SIZE; - mem.va = (void *)(addr); - pd_entry = &sd_entry->u.pd_table.pd_entry[rel_pd_idx++]; - if (!pd_entry->valid) { - status = i40iw_add_pd_table_entry(dev->hw, hmc_info, pd_idx++, &mem); - if (status) - goto error; - addr += PAGE_SIZE; - } else { - i40iw_pr_err("pd entry is valid expecting to be invalid\n"); - } - } - if (!dev->is_pf) { - vf_pble_info.first_pd_index = info->idx.rel_pd_idx; - vf_pble_info.inv_pd_ent = false; - vf_pble_info.pd_entry_cnt = PBLE_PER_PAGE; - vf_pble_info.pd_pl_pba = sd_entry->u.pd_table.pd_page_addr.pa; - vf_pble_info.sd_index = info->idx.sd_idx; - status = i40iw_hw_manage_vf_pble_bp(dev->back_dev, - &vf_pble_info, true); - if (status) { - i40iw_pr_err("CQP manage VF PBLE BP failed. %i\n", status); - goto error; - } - } - chunk->fpm_addr = pble_rsrc->next_fpm_addr; - return 0; -error: - i40iw_free_vmalloc_mem(dev->hw, chunk); - return status; -} - -/** - * add_pble_pool - add a sd entry for pble resoure - * @dev: hardware control device structure - * @pble_rsrc: pble resource management - */ -static enum i40iw_status_code add_pble_pool(struct i40iw_sc_dev *dev, - struct i40iw_hmc_pble_rsrc *pble_rsrc) -{ - struct i40iw_hmc_sd_entry *sd_entry; - struct i40iw_hmc_info *hmc_info; - struct i40iw_chunk *chunk; - struct i40iw_add_page_info info; - struct sd_pd_idx *idx = &info.idx; - enum i40iw_status_code ret_code = 0; - enum i40iw_sd_entry_type sd_entry_type; - u64 sd_reg_val = 0; - u32 pages; - - if (pble_rsrc->unallocated_pble < PBLE_PER_PAGE) - return I40IW_ERR_NO_MEMORY; - if (pble_rsrc->next_fpm_addr & 0xfff) { - i40iw_pr_err("next fpm_addr %llx\n", pble_rsrc->next_fpm_addr); - return I40IW_ERR_INVALID_PAGE_DESC_INDEX; - } - chunk = kzalloc(sizeof(*chunk), GFP_KERNEL); - if (!chunk) - return I40IW_ERR_NO_MEMORY; - hmc_info = dev->hmc_info; - chunk->fpm_addr = pble_rsrc->next_fpm_addr; - get_sd_pd_idx(pble_rsrc, idx); - sd_entry = &hmc_info->sd_table.sd_entry[idx->sd_idx]; - pages = (idx->rel_pd_idx) ? (I40IW_HMC_PD_CNT_IN_SD - - idx->rel_pd_idx) : I40IW_HMC_PD_CNT_IN_SD; - pages = min(pages, pble_rsrc->unallocated_pble >> PBLE_512_SHIFT); - info.chunk = chunk; - info.hmc_info = hmc_info; - info.pages = pages; - info.sd_entry = sd_entry; - if (!sd_entry->valid) { - sd_entry_type = (!idx->rel_pd_idx && - (pages == I40IW_HMC_PD_CNT_IN_SD) && - dev->is_pf) ? I40IW_SD_TYPE_DIRECT : I40IW_SD_TYPE_PAGED; - } else { - sd_entry_type = sd_entry->entry_type; - } - i40iw_debug(dev, I40IW_DEBUG_PBLE, - "pages = %d, unallocated_pble[%u] current_fpm_addr = %llx\n", - pages, pble_rsrc->unallocated_pble, pble_rsrc->next_fpm_addr); - i40iw_debug(dev, I40IW_DEBUG_PBLE, "sd_entry_type = %d sd_entry valid = %d\n", - sd_entry_type, sd_entry->valid); - - if (sd_entry_type == I40IW_SD_TYPE_DIRECT) - ret_code = add_sd_direct(dev, pble_rsrc, &info); - if (ret_code) - sd_entry_type = I40IW_SD_TYPE_PAGED; - else - pble_rsrc->stats_direct_sds++; - - if (sd_entry_type == I40IW_SD_TYPE_PAGED) { - ret_code = add_bp_pages(dev, pble_rsrc, &info); - if (ret_code) - goto error; - else - pble_rsrc->stats_paged_sds++; - } - - if (gen_pool_add_virt(pble_rsrc->pinfo.pool, (unsigned long)chunk->vaddr, - (phys_addr_t)chunk->fpm_addr, chunk->size, -1)) { - i40iw_pr_err("could not allocate memory by gen_pool_addr_virt()\n"); - ret_code = I40IW_ERR_NO_MEMORY; - goto error; - } - pble_rsrc->next_fpm_addr += chunk->size; - i40iw_debug(dev, I40IW_DEBUG_PBLE, "next_fpm_addr = %llx chunk_size[%u] = 0x%x\n", - pble_rsrc->next_fpm_addr, chunk->size, chunk->size); - pble_rsrc->unallocated_pble -= (chunk->size >> 3); - sd_reg_val = (sd_entry_type == I40IW_SD_TYPE_PAGED) ? - sd_entry->u.pd_table.pd_page_addr.pa : sd_entry->u.bp.addr.pa; - if (dev->is_pf && !sd_entry->valid) { - ret_code = i40iw_hmc_sd_one(dev, hmc_info->hmc_fn_id, - sd_reg_val, idx->sd_idx, - sd_entry->entry_type, true); - if (ret_code) { - i40iw_pr_err("cqp cmd failed for sd (pbles)\n"); - goto error; - } - } - - sd_entry->valid = true; - list_add(&chunk->list, &pble_rsrc->pinfo.clist); - return 0; - error: - kfree(chunk); - return ret_code; -} - -/** - * free_lvl2 - fee level 2 pble - * @pble_rsrc: pble resource management - * @palloc: level 2 pble allocation - */ -static void free_lvl2(struct i40iw_hmc_pble_rsrc *pble_rsrc, - struct i40iw_pble_alloc *palloc) -{ - u32 i; - struct gen_pool *pool; - struct i40iw_pble_level2 *lvl2 = &palloc->level2; - struct i40iw_pble_info *root = &lvl2->root; - struct i40iw_pble_info *leaf = lvl2->leaf; - - pool = pble_rsrc->pinfo.pool; - - for (i = 0; i < lvl2->leaf_cnt; i++, leaf++) { - if (leaf->addr) - gen_pool_free(pool, leaf->addr, (leaf->cnt << 3)); - else - break; - } - - if (root->addr) - gen_pool_free(pool, root->addr, (root->cnt << 3)); - - kfree(lvl2->leaf); - lvl2->leaf = NULL; -} - -/** - * get_lvl2_pble - get level 2 pble resource - * @pble_rsrc: pble resource management - * @palloc: level 2 pble allocation - * @pool: pool pointer - */ -static enum i40iw_status_code get_lvl2_pble(struct i40iw_hmc_pble_rsrc *pble_rsrc, - struct i40iw_pble_alloc *palloc, - struct gen_pool *pool) -{ - u32 lf4k, lflast, total, i; - u32 pblcnt = PBLE_PER_PAGE; - u64 *addr; - struct i40iw_pble_level2 *lvl2 = &palloc->level2; - struct i40iw_pble_info *root = &lvl2->root; - struct i40iw_pble_info *leaf; - - /* number of full 512 (4K) leafs) */ - lf4k = palloc->total_cnt >> 9; - lflast = palloc->total_cnt % PBLE_PER_PAGE; - total = (lflast == 0) ? lf4k : lf4k + 1; - lvl2->leaf_cnt = total; - - leaf = kzalloc((sizeof(*leaf) * total), GFP_ATOMIC); - if (!leaf) - return I40IW_ERR_NO_MEMORY; - lvl2->leaf = leaf; - /* allocate pbles for the root */ - root->addr = gen_pool_alloc(pool, (total << 3)); - if (!root->addr) { - kfree(lvl2->leaf); - lvl2->leaf = NULL; - return I40IW_ERR_NO_MEMORY; - } - root->idx = fpm_to_idx(pble_rsrc, - (u64)gen_pool_virt_to_phys(pool, root->addr)); - root->cnt = total; - addr = (u64 *)root->addr; - for (i = 0; i < total; i++, leaf++) { - pblcnt = (lflast && ((i + 1) == total)) ? lflast : PBLE_PER_PAGE; - leaf->addr = gen_pool_alloc(pool, (pblcnt << 3)); - if (!leaf->addr) - goto error; - leaf->idx = fpm_to_idx(pble_rsrc, (u64)gen_pool_virt_to_phys(pool, leaf->addr)); - - leaf->cnt = pblcnt; - *addr = (u64)leaf->idx; - addr++; - } - palloc->level = I40IW_LEVEL_2; - pble_rsrc->stats_lvl2++; - return 0; - error: - free_lvl2(pble_rsrc, palloc); - return I40IW_ERR_NO_MEMORY; -} - -/** - * get_lvl1_pble - get level 1 pble resource - * @dev: hardware control device structure - * @pble_rsrc: pble resource management - * @palloc: level 1 pble allocation - */ -static enum i40iw_status_code get_lvl1_pble(struct i40iw_sc_dev *dev, - struct i40iw_hmc_pble_rsrc *pble_rsrc, - struct i40iw_pble_alloc *palloc) -{ - u64 *addr; - struct gen_pool *pool; - struct i40iw_pble_info *lvl1 = &palloc->level1; - - pool = pble_rsrc->pinfo.pool; - addr = (u64 *)gen_pool_alloc(pool, (palloc->total_cnt << 3)); - - if (!addr) - return I40IW_ERR_NO_MEMORY; - - palloc->level = I40IW_LEVEL_1; - lvl1->addr = (unsigned long)addr; - lvl1->idx = fpm_to_idx(pble_rsrc, (u64)gen_pool_virt_to_phys(pool, - (unsigned long)addr)); - lvl1->cnt = palloc->total_cnt; - pble_rsrc->stats_lvl1++; - return 0; -} - -/** - * get_lvl1_lvl2_pble - calls get_lvl1 and get_lvl2 pble routine - * @dev: i40iw_sc_dev struct - * @pble_rsrc: pble resources - * @palloc: contains all inforamtion regarding pble (idx + pble addr) - * @pool: pointer to general purpose special memory pool descriptor - */ -static inline enum i40iw_status_code get_lvl1_lvl2_pble(struct i40iw_sc_dev *dev, - struct i40iw_hmc_pble_rsrc *pble_rsrc, - struct i40iw_pble_alloc *palloc, - struct gen_pool *pool) -{ - enum i40iw_status_code status = 0; - - status = get_lvl1_pble(dev, pble_rsrc, palloc); - if (status && (palloc->total_cnt > PBLE_PER_PAGE)) - status = get_lvl2_pble(pble_rsrc, palloc, pool); - return status; -} - -/** - * i40iw_get_pble - allocate pbles from the pool - * @dev: i40iw_sc_dev struct - * @pble_rsrc: pble resources - * @palloc: contains all inforamtion regarding pble (idx + pble addr) - * @pble_cnt: #of pbles requested - */ -enum i40iw_status_code i40iw_get_pble(struct i40iw_sc_dev *dev, - struct i40iw_hmc_pble_rsrc *pble_rsrc, - struct i40iw_pble_alloc *palloc, - u32 pble_cnt) -{ - struct gen_pool *pool; - enum i40iw_status_code status = 0; - u32 max_sds = 0; - int i; - - pool = pble_rsrc->pinfo.pool; - palloc->total_cnt = pble_cnt; - palloc->level = I40IW_LEVEL_0; - /*check first to see if we can get pble's without acquiring additional sd's */ - status = get_lvl1_lvl2_pble(dev, pble_rsrc, palloc, pool); - if (!status) - goto exit; - max_sds = (palloc->total_cnt >> 18) + 1; - for (i = 0; i < max_sds; i++) { - status = add_pble_pool(dev, pble_rsrc); - if (status) - break; - status = get_lvl1_lvl2_pble(dev, pble_rsrc, palloc, pool); - if (!status) - break; - } -exit: - if (!status) - pble_rsrc->stats_alloc_ok++; - else - pble_rsrc->stats_alloc_fail++; - - return status; -} - -/** - * i40iw_free_pble - put pbles back into pool - * @pble_rsrc: pble resources - * @palloc: contains all inforamtion regarding pble resource being freed - */ -void i40iw_free_pble(struct i40iw_hmc_pble_rsrc *pble_rsrc, - struct i40iw_pble_alloc *palloc) -{ - struct gen_pool *pool; - - pool = pble_rsrc->pinfo.pool; - if (palloc->level == I40IW_LEVEL_2) - free_lvl2(pble_rsrc, palloc); - else - gen_pool_free(pool, palloc->level1.addr, - (palloc->level1.cnt << 3)); - pble_rsrc->stats_alloc_freed++; -} diff --git a/drivers/infiniband/hw/i40iw/i40iw_pble.h b/drivers/infiniband/hw/i40iw/i40iw_pble.h deleted file mode 100644 index 7b1851d21cc0..000000000000 --- a/drivers/infiniband/hw/i40iw/i40iw_pble.h +++ /dev/null @@ -1,131 +0,0 @@ -/******************************************************************************* -* -* Copyright (c) 2015-2016 Intel Corporation. All rights reserved. -* -* This software is available to you under a choice of one of two -* licenses. You may choose to be licensed under the terms of the GNU -* General Public License (GPL) Version 2, available from the file -* COPYING in the main directory of this source tree, or the -* OpenFabrics.org BSD license below: -* -* Redistribution and use in source and binary forms, with or -* without modification, are permitted provided that the following -* conditions are met: -* -* - Redistributions of source code must retain the above -* copyright notice, this list of conditions and the following -* disclaimer. -* -* - Redistributions in binary form must reproduce the above -* copyright notice, this list of conditions and the following -* disclaimer in the documentation and/or other materials -* provided with the distribution. -* -* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS -* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN -* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -* SOFTWARE. -* -*******************************************************************************/ - -#ifndef I40IW_PBLE_H -#define I40IW_PBLE_H - -#define POOL_SHIFT 6 -#define PBLE_PER_PAGE 512 -#define I40IW_HMC_PAGED_BP_SHIFT 12 -#define PBLE_512_SHIFT 9 - -enum i40iw_pble_level { - I40IW_LEVEL_0 = 0, - I40IW_LEVEL_1 = 1, - I40IW_LEVEL_2 = 2 -}; - -enum i40iw_alloc_type { - I40IW_NO_ALLOC = 0, - I40IW_DMA_COHERENT = 1, - I40IW_VMALLOC = 2 -}; - -struct i40iw_pble_info { - unsigned long addr; - u32 idx; - u32 cnt; -}; - -struct i40iw_pble_level2 { - struct i40iw_pble_info root; - struct i40iw_pble_info *leaf; - u32 leaf_cnt; -}; - -struct i40iw_pble_alloc { - u32 total_cnt; - enum i40iw_pble_level level; - union { - struct i40iw_pble_info level1; - struct i40iw_pble_level2 level2; - }; -}; - -struct sd_pd_idx { - u32 sd_idx; - u32 pd_idx; - u32 rel_pd_idx; -}; - -struct i40iw_add_page_info { - struct i40iw_chunk *chunk; - struct i40iw_hmc_sd_entry *sd_entry; - struct i40iw_hmc_info *hmc_info; - struct sd_pd_idx idx; - u32 pages; -}; - -struct i40iw_chunk { - struct list_head list; - u32 size; - void *vaddr; - u64 fpm_addr; - u32 pg_cnt; - dma_addr_t *dmaaddrs; - enum i40iw_alloc_type type; -}; - -struct i40iw_pble_pool { - struct gen_pool *pool; - struct list_head clist; - u32 total_pble_alloc; - u32 free_pble_cnt; - u32 pool_shift; -}; - -struct i40iw_hmc_pble_rsrc { - u32 unallocated_pble; - u64 fpm_base_addr; - u64 next_fpm_addr; - struct i40iw_pble_pool pinfo; - - u32 stats_direct_sds; - u32 stats_paged_sds; - u64 stats_alloc_ok; - u64 stats_alloc_fail; - u64 stats_alloc_freed; - u64 stats_lvl1; - u64 stats_lvl2; -}; - -void i40iw_destroy_pble_pool(struct i40iw_sc_dev *dev, struct i40iw_hmc_pble_rsrc *pble_rsrc); -enum i40iw_status_code i40iw_hmc_init_pble(struct i40iw_sc_dev *dev, - struct i40iw_hmc_pble_rsrc *pble_rsrc); -void i40iw_free_pble(struct i40iw_hmc_pble_rsrc *pble_rsrc, struct i40iw_pble_alloc *palloc); -enum i40iw_status_code i40iw_get_pble(struct i40iw_sc_dev *dev, - struct i40iw_hmc_pble_rsrc *pble_rsrc, - struct i40iw_pble_alloc *palloc, - u32 pble_cnt); -#endif diff --git a/drivers/infiniband/hw/i40iw/i40iw_puda.c b/drivers/infiniband/hw/i40iw/i40iw_puda.c deleted file mode 100644 index 88fb68e866ba..000000000000 --- a/drivers/infiniband/hw/i40iw/i40iw_puda.c +++ /dev/null @@ -1,1496 +0,0 @@ -/******************************************************************************* -* -* Copyright (c) 2015-2016 Intel Corporation. All rights reserved. -* -* This software is available to you under a choice of one of two -* licenses. You may choose to be licensed under the terms of the GNU -* General Public License (GPL) Version 2, available from the file -* COPYING in the main directory of this source tree, or the -* OpenFabrics.org BSD license below: -* -* Redistribution and use in source and binary forms, with or -* without modification, are permitted provided that the following -* conditions are met: -* -* - Redistributions of source code must retain the above -* copyright notice, this list of conditions and the following -* disclaimer. -* -* - Redistributions in binary form must reproduce the above -* copyright notice, this list of conditions and the following -* disclaimer in the documentation and/or other materials -* provided with the distribution. -* -* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS -* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN -* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -* SOFTWARE. -* -*******************************************************************************/ - -#include "i40iw_osdep.h" -#include "i40iw_register.h" -#include "i40iw_status.h" -#include "i40iw_hmc.h" - -#include "i40iw_d.h" -#include "i40iw_type.h" -#include "i40iw_p.h" -#include "i40iw_puda.h" - -static void i40iw_ieq_receive(struct i40iw_sc_vsi *vsi, - struct i40iw_puda_buf *buf); -static void i40iw_ieq_tx_compl(struct i40iw_sc_vsi *vsi, void *sqwrid); -static void i40iw_ilq_putback_rcvbuf(struct i40iw_sc_qp *qp, u32 wqe_idx); -static enum i40iw_status_code i40iw_puda_replenish_rq(struct i40iw_puda_rsrc - *rsrc, bool initial); -/** - * i40iw_puda_get_listbuf - get buffer from puda list - * @list: list to use for buffers (ILQ or IEQ) - */ -static struct i40iw_puda_buf *i40iw_puda_get_listbuf(struct list_head *list) -{ - struct i40iw_puda_buf *buf = NULL; - - if (!list_empty(list)) { - buf = (struct i40iw_puda_buf *)list->next; - list_del((struct list_head *)&buf->list); - } - return buf; -} - -/** - * i40iw_puda_get_bufpool - return buffer from resource - * @rsrc: resource to use for buffer - */ -struct i40iw_puda_buf *i40iw_puda_get_bufpool(struct i40iw_puda_rsrc *rsrc) -{ - struct i40iw_puda_buf *buf = NULL; - struct list_head *list = &rsrc->bufpool; - unsigned long flags; - - spin_lock_irqsave(&rsrc->bufpool_lock, flags); - buf = i40iw_puda_get_listbuf(list); - if (buf) - rsrc->avail_buf_count--; - else - rsrc->stats_buf_alloc_fail++; - spin_unlock_irqrestore(&rsrc->bufpool_lock, flags); - return buf; -} - -/** - * i40iw_puda_ret_bufpool - return buffer to rsrc list - * @rsrc: resource to use for buffer - * @buf: buffe to return to resouce - */ -void i40iw_puda_ret_bufpool(struct i40iw_puda_rsrc *rsrc, - struct i40iw_puda_buf *buf) -{ - unsigned long flags; - - spin_lock_irqsave(&rsrc->bufpool_lock, flags); - list_add(&buf->list, &rsrc->bufpool); - spin_unlock_irqrestore(&rsrc->bufpool_lock, flags); - rsrc->avail_buf_count++; -} - -/** - * i40iw_puda_post_recvbuf - set wqe for rcv buffer - * @rsrc: resource ptr - * @wqe_idx: wqe index to use - * @buf: puda buffer for rcv q - * @initial: flag if during init time - */ -static void i40iw_puda_post_recvbuf(struct i40iw_puda_rsrc *rsrc, u32 wqe_idx, - struct i40iw_puda_buf *buf, bool initial) -{ - u64 *wqe; - struct i40iw_sc_qp *qp = &rsrc->qp; - u64 offset24 = 0; - - qp->qp_uk.rq_wrid_array[wqe_idx] = (uintptr_t)buf; - wqe = qp->qp_uk.rq_base[wqe_idx].elem; - i40iw_debug(rsrc->dev, I40IW_DEBUG_PUDA, - "%s: wqe_idx= %d buf = %p wqe = %p\n", __func__, - wqe_idx, buf, wqe); - if (!initial) - get_64bit_val(wqe, 24, &offset24); - - offset24 = (offset24) ? 0 : LS_64(1, I40IWQPSQ_VALID); - - set_64bit_val(wqe, 0, buf->mem.pa); - set_64bit_val(wqe, 8, - LS_64(buf->mem.size, I40IWQPSQ_FRAG_LEN)); - i40iw_insert_wqe_hdr(wqe, offset24); -} - -/** - * i40iw_puda_replenish_rq - post rcv buffers - * @rsrc: resource to use for buffer - * @initial: flag if during init time - */ -static enum i40iw_status_code i40iw_puda_replenish_rq(struct i40iw_puda_rsrc *rsrc, - bool initial) -{ - u32 i; - u32 invalid_cnt = rsrc->rxq_invalid_cnt; - struct i40iw_puda_buf *buf = NULL; - - for (i = 0; i < invalid_cnt; i++) { - buf = i40iw_puda_get_bufpool(rsrc); - if (!buf) - return I40IW_ERR_list_empty; - i40iw_puda_post_recvbuf(rsrc, rsrc->rx_wqe_idx, buf, - initial); - rsrc->rx_wqe_idx = - ((rsrc->rx_wqe_idx + 1) % rsrc->rq_size); - rsrc->rxq_invalid_cnt--; - } - return 0; -} - -/** - * i40iw_puda_alloc_buf - allocate mem for buffer - * @dev: iwarp device - * @length: length of buffer - */ -static struct i40iw_puda_buf *i40iw_puda_alloc_buf(struct i40iw_sc_dev *dev, - u32 length) -{ - struct i40iw_puda_buf *buf = NULL; - struct i40iw_virt_mem buf_mem; - enum i40iw_status_code ret; - - ret = i40iw_allocate_virt_mem(dev->hw, &buf_mem, - sizeof(struct i40iw_puda_buf)); - if (ret) { - i40iw_debug(dev, I40IW_DEBUG_PUDA, - "%s: error mem for buf\n", __func__); - return NULL; - } - buf = (struct i40iw_puda_buf *)buf_mem.va; - ret = i40iw_allocate_dma_mem(dev->hw, &buf->mem, length, 1); - if (ret) { - i40iw_debug(dev, I40IW_DEBUG_PUDA, - "%s: error dma mem for buf\n", __func__); - i40iw_free_virt_mem(dev->hw, &buf_mem); - return NULL; - } - buf->buf_mem.va = buf_mem.va; - buf->buf_mem.size = buf_mem.size; - return buf; -} - -/** - * i40iw_puda_dele_buf - delete buffer back to system - * @dev: iwarp device - * @buf: buffer to free - */ -static void i40iw_puda_dele_buf(struct i40iw_sc_dev *dev, - struct i40iw_puda_buf *buf) -{ - i40iw_free_dma_mem(dev->hw, &buf->mem); - i40iw_free_virt_mem(dev->hw, &buf->buf_mem); -} - -/** - * i40iw_puda_get_next_send_wqe - return next wqe for processing - * @qp: puda qp for wqe - * @wqe_idx: wqe index for caller - */ -static u64 *i40iw_puda_get_next_send_wqe(struct i40iw_qp_uk *qp, u32 *wqe_idx) -{ - u64 *wqe = NULL; - enum i40iw_status_code ret_code = 0; - - *wqe_idx = I40IW_RING_GETCURRENT_HEAD(qp->sq_ring); - if (!*wqe_idx) - qp->swqe_polarity = !qp->swqe_polarity; - I40IW_RING_MOVE_HEAD(qp->sq_ring, ret_code); - if (ret_code) - return wqe; - wqe = qp->sq_base[*wqe_idx].elem; - - return wqe; -} - -/** - * i40iw_puda_poll_info - poll cq for completion - * @cq: cq for poll - * @info: info return for successful completion - */ -static enum i40iw_status_code i40iw_puda_poll_info(struct i40iw_sc_cq *cq, - struct i40iw_puda_completion_info *info) -{ - u64 qword0, qword2, qword3; - u64 *cqe; - u64 comp_ctx; - bool valid_bit; - u32 major_err, minor_err; - bool error; - - cqe = (u64 *)I40IW_GET_CURRENT_CQ_ELEMENT(&cq->cq_uk); - get_64bit_val(cqe, 24, &qword3); - valid_bit = (bool)RS_64(qword3, I40IW_CQ_VALID); - - if (valid_bit != cq->cq_uk.polarity) - return I40IW_ERR_QUEUE_EMPTY; - - i40iw_debug_buf(cq->dev, I40IW_DEBUG_PUDA, "PUDA CQE", cqe, 32); - error = (bool)RS_64(qword3, I40IW_CQ_ERROR); - if (error) { - i40iw_debug(cq->dev, I40IW_DEBUG_PUDA, "%s receive error\n", __func__); - major_err = (u32)(RS_64(qword3, I40IW_CQ_MAJERR)); - minor_err = (u32)(RS_64(qword3, I40IW_CQ_MINERR)); - info->compl_error = major_err << 16 | minor_err; - return I40IW_ERR_CQ_COMPL_ERROR; - } - - get_64bit_val(cqe, 0, &qword0); - get_64bit_val(cqe, 16, &qword2); - - info->q_type = (u8)RS_64(qword3, I40IW_CQ_SQ); - info->qp_id = (u32)RS_64(qword2, I40IWCQ_QPID); - - get_64bit_val(cqe, 8, &comp_ctx); - info->qp = (struct i40iw_qp_uk *)(unsigned long)comp_ctx; - info->wqe_idx = (u32)RS_64(qword3, I40IW_CQ_WQEIDX); - - if (info->q_type == I40IW_CQE_QTYPE_RQ) { - info->vlan_valid = (bool)RS_64(qword3, I40IW_VLAN_TAG_VALID); - info->l4proto = (u8)RS_64(qword2, I40IW_UDA_L4PROTO); - info->l3proto = (u8)RS_64(qword2, I40IW_UDA_L3PROTO); - info->payload_len = (u16)RS_64(qword0, I40IW_UDA_PAYLOADLEN); - } - - return 0; -} - -/** - * i40iw_puda_poll_completion - processes completion for cq - * @dev: iwarp device - * @cq: cq getting interrupt - * @compl_err: return any completion err - */ -enum i40iw_status_code i40iw_puda_poll_completion(struct i40iw_sc_dev *dev, - struct i40iw_sc_cq *cq, u32 *compl_err) -{ - struct i40iw_qp_uk *qp; - struct i40iw_cq_uk *cq_uk = &cq->cq_uk; - struct i40iw_puda_completion_info info; - enum i40iw_status_code ret = 0; - struct i40iw_puda_buf *buf; - struct i40iw_puda_rsrc *rsrc; - void *sqwrid; - u8 cq_type = cq->cq_type; - unsigned long flags; - - if ((cq_type == I40IW_CQ_TYPE_ILQ) || (cq_type == I40IW_CQ_TYPE_IEQ)) { - rsrc = (cq_type == I40IW_CQ_TYPE_ILQ) ? cq->vsi->ilq : cq->vsi->ieq; - } else { - i40iw_debug(dev, I40IW_DEBUG_PUDA, "%s qp_type error\n", __func__); - return I40IW_ERR_BAD_PTR; - } - memset(&info, 0, sizeof(info)); - ret = i40iw_puda_poll_info(cq, &info); - *compl_err = info.compl_error; - if (ret == I40IW_ERR_QUEUE_EMPTY) - return ret; - if (ret) - goto done; - - qp = info.qp; - if (!qp || !rsrc) { - ret = I40IW_ERR_BAD_PTR; - goto done; - } - - if (qp->qp_id != rsrc->qp_id) { - ret = I40IW_ERR_BAD_PTR; - goto done; - } - - if (info.q_type == I40IW_CQE_QTYPE_RQ) { - buf = (struct i40iw_puda_buf *)(uintptr_t)qp->rq_wrid_array[info.wqe_idx]; - /* Get all the tcpip information in the buf header */ - ret = i40iw_puda_get_tcpip_info(&info, buf); - if (ret) { - rsrc->stats_rcvd_pkt_err++; - if (cq_type == I40IW_CQ_TYPE_ILQ) { - i40iw_ilq_putback_rcvbuf(&rsrc->qp, - info.wqe_idx); - } else { - i40iw_puda_ret_bufpool(rsrc, buf); - i40iw_puda_replenish_rq(rsrc, false); - } - goto done; - } - - rsrc->stats_pkt_rcvd++; - rsrc->compl_rxwqe_idx = info.wqe_idx; - i40iw_debug(dev, I40IW_DEBUG_PUDA, "%s RQ completion\n", __func__); - rsrc->receive(rsrc->vsi, buf); - if (cq_type == I40IW_CQ_TYPE_ILQ) - i40iw_ilq_putback_rcvbuf(&rsrc->qp, info.wqe_idx); - else - i40iw_puda_replenish_rq(rsrc, false); - - } else { - i40iw_debug(dev, I40IW_DEBUG_PUDA, "%s SQ completion\n", __func__); - sqwrid = (void *)(uintptr_t)qp->sq_wrtrk_array[info.wqe_idx].wrid; - I40IW_RING_SET_TAIL(qp->sq_ring, info.wqe_idx); - rsrc->xmit_complete(rsrc->vsi, sqwrid); - spin_lock_irqsave(&rsrc->bufpool_lock, flags); - rsrc->tx_wqe_avail_cnt++; - spin_unlock_irqrestore(&rsrc->bufpool_lock, flags); - if (!list_empty(&rsrc->txpend)) - i40iw_puda_send_buf(rsrc, NULL); - } - -done: - I40IW_RING_MOVE_HEAD(cq_uk->cq_ring, ret); - if (I40IW_RING_GETCURRENT_HEAD(cq_uk->cq_ring) == 0) - cq_uk->polarity = !cq_uk->polarity; - /* update cq tail in cq shadow memory also */ - I40IW_RING_MOVE_TAIL(cq_uk->cq_ring); - set_64bit_val(cq_uk->shadow_area, 0, - I40IW_RING_GETCURRENT_HEAD(cq_uk->cq_ring)); - return 0; -} - -/** - * i40iw_puda_send - complete send wqe for transmit - * @qp: puda qp for send - * @info: buffer information for transmit - */ -enum i40iw_status_code i40iw_puda_send(struct i40iw_sc_qp *qp, - struct i40iw_puda_send_info *info) -{ - u64 *wqe; - u32 iplen, l4len; - u64 header[2]; - u32 wqe_idx; - u8 iipt; - - /* number of 32 bits DWORDS in header */ - l4len = info->tcplen >> 2; - if (info->ipv4) { - iipt = 3; - iplen = 5; - } else { - iipt = 1; - iplen = 10; - } - - wqe = i40iw_puda_get_next_send_wqe(&qp->qp_uk, &wqe_idx); - if (!wqe) - return I40IW_ERR_QP_TOOMANY_WRS_POSTED; - qp->qp_uk.sq_wrtrk_array[wqe_idx].wrid = (uintptr_t)info->scratch; - /* Third line of WQE descriptor */ - /* maclen is in words */ - header[0] = LS_64((info->maclen >> 1), I40IW_UDA_QPSQ_MACLEN) | - LS_64(iplen, I40IW_UDA_QPSQ_IPLEN) | LS_64(1, I40IW_UDA_QPSQ_L4T) | - LS_64(iipt, I40IW_UDA_QPSQ_IIPT) | - LS_64(l4len, I40IW_UDA_QPSQ_L4LEN); - /* Forth line of WQE descriptor */ - header[1] = LS_64(I40IW_OP_TYPE_SEND, I40IW_UDA_QPSQ_OPCODE) | - LS_64(1, I40IW_UDA_QPSQ_SIGCOMPL) | - LS_64(info->doloopback, I40IW_UDA_QPSQ_DOLOOPBACK) | - LS_64(qp->qp_uk.swqe_polarity, I40IW_UDA_QPSQ_VALID); - - set_64bit_val(wqe, 0, info->paddr); - set_64bit_val(wqe, 8, LS_64(info->len, I40IWQPSQ_FRAG_LEN)); - set_64bit_val(wqe, 16, header[0]); - - i40iw_insert_wqe_hdr(wqe, header[1]); - - i40iw_debug_buf(qp->dev, I40IW_DEBUG_PUDA, "PUDA SEND WQE", wqe, 32); - i40iw_qp_post_wr(&qp->qp_uk); - return 0; -} - -/** - * i40iw_puda_send_buf - transmit puda buffer - * @rsrc: resource to use for buffer - * @buf: puda buffer to transmit - */ -void i40iw_puda_send_buf(struct i40iw_puda_rsrc *rsrc, struct i40iw_puda_buf *buf) -{ - struct i40iw_puda_send_info info; - enum i40iw_status_code ret = 0; - unsigned long flags; - - spin_lock_irqsave(&rsrc->bufpool_lock, flags); - /* if no wqe available or not from a completion and we have - * pending buffers, we must queue new buffer - */ - if (!rsrc->tx_wqe_avail_cnt || (buf && !list_empty(&rsrc->txpend))) { - list_add_tail(&buf->list, &rsrc->txpend); - spin_unlock_irqrestore(&rsrc->bufpool_lock, flags); - rsrc->stats_sent_pkt_q++; - if (rsrc->type == I40IW_PUDA_RSRC_TYPE_ILQ) - i40iw_debug(rsrc->dev, I40IW_DEBUG_PUDA, - "%s: adding to txpend\n", __func__); - return; - } - rsrc->tx_wqe_avail_cnt--; - /* if we are coming from a completion and have pending buffers - * then Get one from pending list - */ - if (!buf) { - buf = i40iw_puda_get_listbuf(&rsrc->txpend); - if (!buf) - goto done; - } - - info.scratch = (void *)buf; - info.paddr = buf->mem.pa; - info.len = buf->totallen; - info.tcplen = buf->tcphlen; - info.maclen = buf->maclen; - info.ipv4 = buf->ipv4; - info.doloopback = (rsrc->type == I40IW_PUDA_RSRC_TYPE_IEQ); - - ret = i40iw_puda_send(&rsrc->qp, &info); - if (ret) { - rsrc->tx_wqe_avail_cnt++; - rsrc->stats_sent_pkt_q++; - list_add(&buf->list, &rsrc->txpend); - if (rsrc->type == I40IW_PUDA_RSRC_TYPE_ILQ) - i40iw_debug(rsrc->dev, I40IW_DEBUG_PUDA, - "%s: adding to puda_send\n", __func__); - } else { - rsrc->stats_pkt_sent++; - } -done: - spin_unlock_irqrestore(&rsrc->bufpool_lock, flags); -} - -/** - * i40iw_puda_qp_setctx - during init, set qp's context - * @rsrc: qp's resource - */ -static void i40iw_puda_qp_setctx(struct i40iw_puda_rsrc *rsrc) -{ - struct i40iw_sc_qp *qp = &rsrc->qp; - u64 *qp_ctx = qp->hw_host_ctx; - - set_64bit_val(qp_ctx, 8, qp->sq_pa); - set_64bit_val(qp_ctx, 16, qp->rq_pa); - - set_64bit_val(qp_ctx, 24, - LS_64(qp->hw_rq_size, I40IWQPC_RQSIZE) | - LS_64(qp->hw_sq_size, I40IWQPC_SQSIZE)); - - set_64bit_val(qp_ctx, 48, LS_64(rsrc->buf_size, I40IW_UDA_QPC_MAXFRAMESIZE)); - set_64bit_val(qp_ctx, 56, 0); - set_64bit_val(qp_ctx, 64, 1); - - set_64bit_val(qp_ctx, 136, - LS_64(rsrc->cq_id, I40IWQPC_TXCQNUM) | - LS_64(rsrc->cq_id, I40IWQPC_RXCQNUM)); - - set_64bit_val(qp_ctx, 160, LS_64(1, I40IWQPC_PRIVEN)); - - set_64bit_val(qp_ctx, 168, - LS_64((uintptr_t)qp, I40IWQPC_QPCOMPCTX)); - - set_64bit_val(qp_ctx, 176, - LS_64(qp->sq_tph_val, I40IWQPC_SQTPHVAL) | - LS_64(qp->rq_tph_val, I40IWQPC_RQTPHVAL) | - LS_64(qp->qs_handle, I40IWQPC_QSHANDLE)); - - i40iw_debug_buf(rsrc->dev, I40IW_DEBUG_PUDA, "PUDA QP CONTEXT", - qp_ctx, I40IW_QP_CTX_SIZE); -} - -/** - * i40iw_puda_qp_wqe - setup wqe for qp create - * @dev: iwarp device - * @qp: resource for qp - */ -static enum i40iw_status_code i40iw_puda_qp_wqe(struct i40iw_sc_dev *dev, struct i40iw_sc_qp *qp) -{ - struct i40iw_sc_cqp *cqp; - u64 *wqe; - u64 header; - struct i40iw_ccq_cqe_info compl_info; - enum i40iw_status_code status = 0; - - cqp = dev->cqp; - wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, 0); - if (!wqe) - return I40IW_ERR_RING_FULL; - - set_64bit_val(wqe, 16, qp->hw_host_ctx_pa); - set_64bit_val(wqe, 40, qp->shadow_area_pa); - header = qp->qp_uk.qp_id | - LS_64(I40IW_CQP_OP_CREATE_QP, I40IW_CQPSQ_OPCODE) | - LS_64(I40IW_QP_TYPE_UDA, I40IW_CQPSQ_QP_QPTYPE) | - LS_64(1, I40IW_CQPSQ_QP_CQNUMVALID) | - LS_64(2, I40IW_CQPSQ_QP_NEXTIWSTATE) | - LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID); - - i40iw_insert_wqe_hdr(wqe, header); - - i40iw_debug_buf(cqp->dev, I40IW_DEBUG_PUDA, "PUDA CQE", wqe, 32); - i40iw_sc_cqp_post_sq(cqp); - status = dev->cqp_ops->poll_for_cqp_op_done(dev->cqp, - I40IW_CQP_OP_CREATE_QP, - &compl_info); - return status; -} - -/** - * i40iw_puda_qp_create - create qp for resource - * @rsrc: resource to use for buffer - */ -static enum i40iw_status_code i40iw_puda_qp_create(struct i40iw_puda_rsrc *rsrc) -{ - struct i40iw_sc_qp *qp = &rsrc->qp; - struct i40iw_qp_uk *ukqp = &qp->qp_uk; - enum i40iw_status_code ret = 0; - u32 sq_size, rq_size, t_size; - struct i40iw_dma_mem *mem; - - sq_size = rsrc->sq_size * I40IW_QP_WQE_MIN_SIZE; - rq_size = rsrc->rq_size * I40IW_QP_WQE_MIN_SIZE; - t_size = (sq_size + rq_size + (I40IW_SHADOW_AREA_SIZE << 3) + - I40IW_QP_CTX_SIZE); - /* Get page aligned memory */ - ret = - i40iw_allocate_dma_mem(rsrc->dev->hw, &rsrc->qpmem, t_size, - I40IW_HW_PAGE_SIZE); - if (ret) { - i40iw_debug(rsrc->dev, I40IW_DEBUG_PUDA, "%s: error dma mem\n", __func__); - return ret; - } - - mem = &rsrc->qpmem; - memset(mem->va, 0, t_size); - qp->hw_sq_size = i40iw_get_encoded_wqe_size(rsrc->sq_size, false); - qp->hw_rq_size = i40iw_get_encoded_wqe_size(rsrc->rq_size, false); - qp->pd = &rsrc->sc_pd; - qp->qp_type = I40IW_QP_TYPE_UDA; - qp->dev = rsrc->dev; - qp->back_qp = (void *)rsrc; - qp->sq_pa = mem->pa; - qp->rq_pa = qp->sq_pa + sq_size; - qp->vsi = rsrc->vsi; - ukqp->sq_base = mem->va; - ukqp->rq_base = &ukqp->sq_base[rsrc->sq_size]; - ukqp->shadow_area = ukqp->rq_base[rsrc->rq_size].elem; - qp->shadow_area_pa = qp->rq_pa + rq_size; - qp->hw_host_ctx = ukqp->shadow_area + I40IW_SHADOW_AREA_SIZE; - qp->hw_host_ctx_pa = - qp->shadow_area_pa + (I40IW_SHADOW_AREA_SIZE << 3); - ukqp->qp_id = rsrc->qp_id; - ukqp->sq_wrtrk_array = rsrc->sq_wrtrk_array; - ukqp->rq_wrid_array = rsrc->rq_wrid_array; - - ukqp->qp_id = rsrc->qp_id; - ukqp->sq_size = rsrc->sq_size; - ukqp->rq_size = rsrc->rq_size; - - I40IW_RING_INIT(ukqp->sq_ring, ukqp->sq_size); - I40IW_RING_INIT(ukqp->initial_ring, ukqp->sq_size); - I40IW_RING_INIT(ukqp->rq_ring, ukqp->rq_size); - - if (qp->pd->dev->is_pf) - ukqp->wqe_alloc_reg = (u32 __iomem *)(i40iw_get_hw_addr(qp->pd->dev) + - I40E_PFPE_WQEALLOC); - else - ukqp->wqe_alloc_reg = (u32 __iomem *)(i40iw_get_hw_addr(qp->pd->dev) + - I40E_VFPE_WQEALLOC1); - - qp->user_pri = 0; - i40iw_qp_add_qos(qp); - i40iw_puda_qp_setctx(rsrc); - if (rsrc->dev->ceq_valid) - ret = i40iw_cqp_qp_create_cmd(rsrc->dev, qp); - else - ret = i40iw_puda_qp_wqe(rsrc->dev, qp); - if (ret) { - i40iw_qp_rem_qos(qp); - i40iw_free_dma_mem(rsrc->dev->hw, &rsrc->qpmem); - } - return ret; -} - -/** - * i40iw_puda_cq_wqe - setup wqe for cq create - * @dev: iwarp device - * @cq: cq to setup - */ -static enum i40iw_status_code i40iw_puda_cq_wqe(struct i40iw_sc_dev *dev, struct i40iw_sc_cq *cq) -{ - u64 *wqe; - struct i40iw_sc_cqp *cqp; - u64 header; - struct i40iw_ccq_cqe_info compl_info; - enum i40iw_status_code status = 0; - - cqp = dev->cqp; - wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, 0); - if (!wqe) - return I40IW_ERR_RING_FULL; - - set_64bit_val(wqe, 0, cq->cq_uk.cq_size); - set_64bit_val(wqe, 8, RS_64_1(cq, 1)); - set_64bit_val(wqe, 16, - LS_64(cq->shadow_read_threshold, - I40IW_CQPSQ_CQ_SHADOW_READ_THRESHOLD)); - set_64bit_val(wqe, 32, cq->cq_pa); - - set_64bit_val(wqe, 40, cq->shadow_area_pa); - - header = cq->cq_uk.cq_id | - LS_64(I40IW_CQP_OP_CREATE_CQ, I40IW_CQPSQ_OPCODE) | - LS_64(1, I40IW_CQPSQ_CQ_CHKOVERFLOW) | - LS_64(1, I40IW_CQPSQ_CQ_ENCEQEMASK) | - LS_64(1, I40IW_CQPSQ_CQ_CEQIDVALID) | - LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID); - i40iw_insert_wqe_hdr(wqe, header); - - i40iw_debug_buf(dev, I40IW_DEBUG_PUDA, "PUDA CQE", - wqe, I40IW_CQP_WQE_SIZE * 8); - - i40iw_sc_cqp_post_sq(dev->cqp); - status = dev->cqp_ops->poll_for_cqp_op_done(dev->cqp, - I40IW_CQP_OP_CREATE_CQ, - &compl_info); - return status; -} - -/** - * i40iw_puda_cq_create - create cq for resource - * @rsrc: resource for which cq to create - */ -static enum i40iw_status_code i40iw_puda_cq_create(struct i40iw_puda_rsrc *rsrc) -{ - struct i40iw_sc_dev *dev = rsrc->dev; - struct i40iw_sc_cq *cq = &rsrc->cq; - enum i40iw_status_code ret = 0; - u32 tsize, cqsize; - struct i40iw_dma_mem *mem; - struct i40iw_cq_init_info info; - struct i40iw_cq_uk_init_info *init_info = &info.cq_uk_init_info; - - cq->vsi = rsrc->vsi; - cqsize = rsrc->cq_size * (sizeof(struct i40iw_cqe)); - tsize = cqsize + sizeof(struct i40iw_cq_shadow_area); - ret = i40iw_allocate_dma_mem(dev->hw, &rsrc->cqmem, tsize, - I40IW_CQ0_ALIGNMENT); - if (ret) - return ret; - - mem = &rsrc->cqmem; - memset(&info, 0, sizeof(info)); - info.dev = dev; - info.type = (rsrc->type == I40IW_PUDA_RSRC_TYPE_ILQ) ? - I40IW_CQ_TYPE_ILQ : I40IW_CQ_TYPE_IEQ; - info.shadow_read_threshold = rsrc->cq_size >> 2; - info.ceq_id_valid = true; - info.cq_base_pa = mem->pa; - info.shadow_area_pa = mem->pa + cqsize; - init_info->cq_base = mem->va; - init_info->shadow_area = (u64 *)((u8 *)mem->va + cqsize); - init_info->cq_size = rsrc->cq_size; - init_info->cq_id = rsrc->cq_id; - info.ceqe_mask = true; - info.ceq_id_valid = true; - ret = dev->iw_priv_cq_ops->cq_init(cq, &info); - if (ret) - goto error; - if (rsrc->dev->ceq_valid) - ret = i40iw_cqp_cq_create_cmd(dev, cq); - else - ret = i40iw_puda_cq_wqe(dev, cq); -error: - if (ret) - i40iw_free_dma_mem(dev->hw, &rsrc->cqmem); - return ret; -} - -/** - * i40iw_puda_free_qp - free qp for resource - * @rsrc: resource for which qp to free - */ -static void i40iw_puda_free_qp(struct i40iw_puda_rsrc *rsrc) -{ - enum i40iw_status_code ret; - struct i40iw_ccq_cqe_info compl_info; - struct i40iw_sc_dev *dev = rsrc->dev; - - if (rsrc->dev->ceq_valid) { - i40iw_cqp_qp_destroy_cmd(dev, &rsrc->qp); - return; - } - - ret = dev->iw_priv_qp_ops->qp_destroy(&rsrc->qp, - 0, false, true, true); - if (ret) - i40iw_debug(dev, I40IW_DEBUG_PUDA, - "%s error puda qp destroy wqe\n", - __func__); - - if (!ret) { - ret = dev->cqp_ops->poll_for_cqp_op_done(dev->cqp, - I40IW_CQP_OP_DESTROY_QP, - &compl_info); - if (ret) - i40iw_debug(dev, I40IW_DEBUG_PUDA, - "%s error puda qp destroy failed\n", - __func__); - } -} - -/** - * i40iw_puda_free_cq - free cq for resource - * @rsrc: resource for which cq to free - */ -static void i40iw_puda_free_cq(struct i40iw_puda_rsrc *rsrc) -{ - enum i40iw_status_code ret; - struct i40iw_ccq_cqe_info compl_info; - struct i40iw_sc_dev *dev = rsrc->dev; - - if (rsrc->dev->ceq_valid) { - i40iw_cqp_cq_destroy_cmd(dev, &rsrc->cq); - return; - } - ret = dev->iw_priv_cq_ops->cq_destroy(&rsrc->cq, 0, true); - - if (ret) - i40iw_debug(dev, I40IW_DEBUG_PUDA, - "%s error ieq cq destroy\n", - __func__); - - if (!ret) { - ret = dev->cqp_ops->poll_for_cqp_op_done(dev->cqp, - I40IW_CQP_OP_DESTROY_CQ, - &compl_info); - if (ret) - i40iw_debug(dev, I40IW_DEBUG_PUDA, - "%s error ieq qp destroy done\n", - __func__); - } -} - -/** - * i40iw_puda_dele_resources - delete all resources during close - * @vsi: pointer to vsi structure - * @type: type of resource to dele - * @reset: true if reset chip - */ -void i40iw_puda_dele_resources(struct i40iw_sc_vsi *vsi, - enum puda_resource_type type, - bool reset) -{ - struct i40iw_sc_dev *dev = vsi->dev; - struct i40iw_puda_rsrc *rsrc; - struct i40iw_puda_buf *buf = NULL; - struct i40iw_puda_buf *nextbuf = NULL; - struct i40iw_virt_mem *vmem; - - switch (type) { - case I40IW_PUDA_RSRC_TYPE_ILQ: - rsrc = vsi->ilq; - vmem = &vsi->ilq_mem; - break; - case I40IW_PUDA_RSRC_TYPE_IEQ: - rsrc = vsi->ieq; - vmem = &vsi->ieq_mem; - break; - default: - i40iw_debug(dev, I40IW_DEBUG_PUDA, "%s: error resource type = 0x%x\n", - __func__, type); - return; - } - - switch (rsrc->completion) { - case PUDA_HASH_CRC_COMPLETE: - i40iw_free_hash_desc(rsrc->hash_desc); - fallthrough; - case PUDA_QP_CREATED: - if (!reset) - i40iw_puda_free_qp(rsrc); - - i40iw_free_dma_mem(dev->hw, &rsrc->qpmem); - fallthrough; - case PUDA_CQ_CREATED: - if (!reset) - i40iw_puda_free_cq(rsrc); - - i40iw_free_dma_mem(dev->hw, &rsrc->cqmem); - break; - default: - i40iw_debug(rsrc->dev, I40IW_DEBUG_PUDA, "%s error no resources\n", __func__); - break; - } - /* Free all allocated puda buffers for both tx and rx */ - buf = rsrc->alloclist; - while (buf) { - nextbuf = buf->next; - i40iw_puda_dele_buf(dev, buf); - buf = nextbuf; - rsrc->alloc_buf_count--; - } - i40iw_free_virt_mem(dev->hw, vmem); -} - -/** - * i40iw_puda_allocbufs - allocate buffers for resource - * @rsrc: resource for buffer allocation - * @count: number of buffers to create - */ -static enum i40iw_status_code i40iw_puda_allocbufs(struct i40iw_puda_rsrc *rsrc, - u32 count) -{ - u32 i; - struct i40iw_puda_buf *buf; - struct i40iw_puda_buf *nextbuf; - - for (i = 0; i < count; i++) { - buf = i40iw_puda_alloc_buf(rsrc->dev, rsrc->buf_size); - if (!buf) { - rsrc->stats_buf_alloc_fail++; - return I40IW_ERR_NO_MEMORY; - } - i40iw_puda_ret_bufpool(rsrc, buf); - rsrc->alloc_buf_count++; - if (!rsrc->alloclist) { - rsrc->alloclist = buf; - } else { - nextbuf = rsrc->alloclist; - rsrc->alloclist = buf; - buf->next = nextbuf; - } - } - rsrc->avail_buf_count = rsrc->alloc_buf_count; - return 0; -} - -/** - * i40iw_puda_create_rsrc - create resouce (ilq or ieq) - * @vsi: pointer to vsi structure - * @info: resource information - */ -enum i40iw_status_code i40iw_puda_create_rsrc(struct i40iw_sc_vsi *vsi, - struct i40iw_puda_rsrc_info *info) -{ - struct i40iw_sc_dev *dev = vsi->dev; - enum i40iw_status_code ret = 0; - struct i40iw_puda_rsrc *rsrc; - u32 pudasize; - u32 sqwridsize, rqwridsize; - struct i40iw_virt_mem *vmem; - - info->count = 1; - pudasize = sizeof(struct i40iw_puda_rsrc); - sqwridsize = info->sq_size * sizeof(struct i40iw_sq_uk_wr_trk_info); - rqwridsize = info->rq_size * 8; - switch (info->type) { - case I40IW_PUDA_RSRC_TYPE_ILQ: - vmem = &vsi->ilq_mem; - break; - case I40IW_PUDA_RSRC_TYPE_IEQ: - vmem = &vsi->ieq_mem; - break; - default: - return I40IW_NOT_SUPPORTED; - } - ret = - i40iw_allocate_virt_mem(dev->hw, vmem, - pudasize + sqwridsize + rqwridsize); - if (ret) - return ret; - rsrc = (struct i40iw_puda_rsrc *)vmem->va; - spin_lock_init(&rsrc->bufpool_lock); - if (info->type == I40IW_PUDA_RSRC_TYPE_ILQ) { - vsi->ilq = (struct i40iw_puda_rsrc *)vmem->va; - vsi->ilq_count = info->count; - rsrc->receive = info->receive; - rsrc->xmit_complete = info->xmit_complete; - } else { - vmem = &vsi->ieq_mem; - vsi->ieq_count = info->count; - vsi->ieq = (struct i40iw_puda_rsrc *)vmem->va; - rsrc->receive = i40iw_ieq_receive; - rsrc->xmit_complete = i40iw_ieq_tx_compl; - } - - rsrc->type = info->type; - rsrc->sq_wrtrk_array = (struct i40iw_sq_uk_wr_trk_info *)((u8 *)vmem->va + pudasize); - rsrc->rq_wrid_array = (u64 *)((u8 *)vmem->va + pudasize + sqwridsize); - /* Initialize all ieq lists */ - INIT_LIST_HEAD(&rsrc->bufpool); - INIT_LIST_HEAD(&rsrc->txpend); - - rsrc->tx_wqe_avail_cnt = info->sq_size - 1; - dev->iw_pd_ops->pd_init(dev, &rsrc->sc_pd, info->pd_id, -1); - rsrc->qp_id = info->qp_id; - rsrc->cq_id = info->cq_id; - rsrc->sq_size = info->sq_size; - rsrc->rq_size = info->rq_size; - rsrc->cq_size = info->rq_size + info->sq_size; - rsrc->buf_size = info->buf_size; - rsrc->dev = dev; - rsrc->vsi = vsi; - - ret = i40iw_puda_cq_create(rsrc); - if (!ret) { - rsrc->completion = PUDA_CQ_CREATED; - ret = i40iw_puda_qp_create(rsrc); - } - if (ret) { - i40iw_debug(dev, I40IW_DEBUG_PUDA, "[%s] error qp_create\n", - __func__); - goto error; - } - rsrc->completion = PUDA_QP_CREATED; - - ret = i40iw_puda_allocbufs(rsrc, info->tx_buf_cnt + info->rq_size); - if (ret) { - i40iw_debug(dev, I40IW_DEBUG_PUDA, "[%s] error alloc_buf\n", - __func__); - goto error; - } - - rsrc->rxq_invalid_cnt = info->rq_size; - ret = i40iw_puda_replenish_rq(rsrc, true); - if (ret) - goto error; - - if (info->type == I40IW_PUDA_RSRC_TYPE_IEQ) { - if (!i40iw_init_hash_desc(&rsrc->hash_desc)) { - rsrc->check_crc = true; - rsrc->completion = PUDA_HASH_CRC_COMPLETE; - ret = 0; - } - } - - dev->ccq_ops->ccq_arm(&rsrc->cq); - return ret; - error: - i40iw_puda_dele_resources(vsi, info->type, false); - - return ret; -} - -/** - * i40iw_ilq_putback_rcvbuf - ilq buffer to put back on rq - * @qp: ilq's qp resource - * @wqe_idx: wqe index of completed rcvbuf - */ -static void i40iw_ilq_putback_rcvbuf(struct i40iw_sc_qp *qp, u32 wqe_idx) -{ - u64 *wqe; - u64 offset24; - - wqe = qp->qp_uk.rq_base[wqe_idx].elem; - get_64bit_val(wqe, 24, &offset24); - offset24 = (offset24) ? 0 : LS_64(1, I40IWQPSQ_VALID); - set_64bit_val(wqe, 24, offset24); -} - -/** - * i40iw_ieq_get_fpdu_length - given length return fpdu length - * @length: length if fpdu - */ -static u16 i40iw_ieq_get_fpdu_length(u16 length) -{ - u16 fpdu_len; - - fpdu_len = length + I40IW_IEQ_MPA_FRAMING; - fpdu_len = (fpdu_len + 3) & 0xfffffffc; - return fpdu_len; -} - -/** - * i40iw_ieq_copy_to_txbuf - copydata from rcv buf to tx buf - * @buf: rcv buffer with partial - * @txbuf: tx buffer for sendign back - * @buf_offset: rcv buffer offset to copy from - * @txbuf_offset: at offset in tx buf to copy - * @length: length of data to copy - */ -static void i40iw_ieq_copy_to_txbuf(struct i40iw_puda_buf *buf, - struct i40iw_puda_buf *txbuf, - u16 buf_offset, u32 txbuf_offset, - u32 length) -{ - void *mem1 = (u8 *)buf->mem.va + buf_offset; - void *mem2 = (u8 *)txbuf->mem.va + txbuf_offset; - - memcpy(mem2, mem1, length); -} - -/** - * i40iw_ieq_setup_tx_buf - setup tx buffer for partial handling - * @buf: reeive buffer with partial - * @txbuf: buffer to prepare - */ -static void i40iw_ieq_setup_tx_buf(struct i40iw_puda_buf *buf, - struct i40iw_puda_buf *txbuf) -{ - txbuf->maclen = buf->maclen; - txbuf->tcphlen = buf->tcphlen; - txbuf->ipv4 = buf->ipv4; - txbuf->hdrlen = buf->hdrlen; - i40iw_ieq_copy_to_txbuf(buf, txbuf, 0, 0, buf->hdrlen); -} - -/** - * i40iw_ieq_check_first_buf - check if rcv buffer's seq is in range - * @buf: receive exception buffer - * @fps: first partial sequence number - */ -static void i40iw_ieq_check_first_buf(struct i40iw_puda_buf *buf, u32 fps) -{ - u32 offset; - - if (buf->seqnum < fps) { - offset = fps - buf->seqnum; - if (offset > buf->datalen) - return; - buf->data += offset; - buf->datalen -= (u16)offset; - buf->seqnum = fps; - } -} - -/** - * i40iw_ieq_compl_pfpdu - write txbuf with full fpdu - * @ieq: ieq resource - * @rxlist: ieq's received buffer list - * @pbufl: temporary list for buffers for fpddu - * @txbuf: tx buffer for fpdu - * @fpdu_len: total length of fpdu - */ -static void i40iw_ieq_compl_pfpdu(struct i40iw_puda_rsrc *ieq, - struct list_head *rxlist, - struct list_head *pbufl, - struct i40iw_puda_buf *txbuf, - u16 fpdu_len) -{ - struct i40iw_puda_buf *buf; - u32 nextseqnum; - u16 txoffset, bufoffset; - - buf = i40iw_puda_get_listbuf(pbufl); - if (!buf) - return; - nextseqnum = buf->seqnum + fpdu_len; - txbuf->totallen = buf->hdrlen + fpdu_len; - txbuf->data = (u8 *)txbuf->mem.va + buf->hdrlen; - i40iw_ieq_setup_tx_buf(buf, txbuf); - - txoffset = buf->hdrlen; - bufoffset = (u16)(buf->data - (u8 *)buf->mem.va); - - do { - if (buf->datalen >= fpdu_len) { - /* copied full fpdu */ - i40iw_ieq_copy_to_txbuf(buf, txbuf, bufoffset, txoffset, fpdu_len); - buf->datalen -= fpdu_len; - buf->data += fpdu_len; - buf->seqnum = nextseqnum; - break; - } - /* copy partial fpdu */ - i40iw_ieq_copy_to_txbuf(buf, txbuf, bufoffset, txoffset, buf->datalen); - txoffset += buf->datalen; - fpdu_len -= buf->datalen; - i40iw_puda_ret_bufpool(ieq, buf); - buf = i40iw_puda_get_listbuf(pbufl); - if (!buf) - return; - bufoffset = (u16)(buf->data - (u8 *)buf->mem.va); - } while (1); - - /* last buffer on the list*/ - if (buf->datalen) - list_add(&buf->list, rxlist); - else - i40iw_puda_ret_bufpool(ieq, buf); -} - -/** - * i40iw_ieq_create_pbufl - create buffer list for single fpdu - * @pfpdu: partial management per user qp - * @rxlist: resource list for receive ieq buffes - * @pbufl: temp. list for buffers for fpddu - * @buf: first receive buffer - * @fpdu_len: total length of fpdu - */ -static enum i40iw_status_code i40iw_ieq_create_pbufl( - struct i40iw_pfpdu *pfpdu, - struct list_head *rxlist, - struct list_head *pbufl, - struct i40iw_puda_buf *buf, - u16 fpdu_len) -{ - enum i40iw_status_code status = 0; - struct i40iw_puda_buf *nextbuf; - u32 nextseqnum; - u16 plen = fpdu_len - buf->datalen; - bool done = false; - - nextseqnum = buf->seqnum + buf->datalen; - do { - nextbuf = i40iw_puda_get_listbuf(rxlist); - if (!nextbuf) { - status = I40IW_ERR_list_empty; - break; - } - list_add_tail(&nextbuf->list, pbufl); - if (nextbuf->seqnum != nextseqnum) { - pfpdu->bad_seq_num++; - status = I40IW_ERR_SEQ_NUM; - break; - } - if (nextbuf->datalen >= plen) { - done = true; - } else { - plen -= nextbuf->datalen; - nextseqnum = nextbuf->seqnum + nextbuf->datalen; - } - - } while (!done); - - return status; -} - -/** - * i40iw_ieq_handle_partial - process partial fpdu buffer - * @ieq: ieq resource - * @pfpdu: partial management per user qp - * @buf: receive buffer - * @fpdu_len: fpdu len in the buffer - */ -static enum i40iw_status_code i40iw_ieq_handle_partial(struct i40iw_puda_rsrc *ieq, - struct i40iw_pfpdu *pfpdu, - struct i40iw_puda_buf *buf, - u16 fpdu_len) -{ - enum i40iw_status_code status = 0; - u8 *crcptr; - u32 mpacrc; - u32 seqnum = buf->seqnum; - struct list_head pbufl; /* partial buffer list */ - struct i40iw_puda_buf *txbuf = NULL; - struct list_head *rxlist = &pfpdu->rxlist; - - INIT_LIST_HEAD(&pbufl); - list_add(&buf->list, &pbufl); - - status = i40iw_ieq_create_pbufl(pfpdu, rxlist, &pbufl, buf, fpdu_len); - if (status) - goto error; - - txbuf = i40iw_puda_get_bufpool(ieq); - if (!txbuf) { - pfpdu->no_tx_bufs++; - status = I40IW_ERR_NO_TXBUFS; - goto error; - } - - i40iw_ieq_compl_pfpdu(ieq, rxlist, &pbufl, txbuf, fpdu_len); - i40iw_ieq_update_tcpip_info(txbuf, fpdu_len, seqnum); - crcptr = txbuf->data + fpdu_len - 4; - mpacrc = *(u32 *)crcptr; - if (ieq->check_crc) { - status = i40iw_ieq_check_mpacrc(ieq->hash_desc, txbuf->data, - (fpdu_len - 4), mpacrc); - if (status) { - i40iw_debug(ieq->dev, I40IW_DEBUG_IEQ, - "%s: error bad crc\n", __func__); - goto error; - } - } - - i40iw_debug_buf(ieq->dev, I40IW_DEBUG_IEQ, "IEQ TX BUFFER", - txbuf->mem.va, txbuf->totallen); - i40iw_puda_send_buf(ieq, txbuf); - pfpdu->rcv_nxt = seqnum + fpdu_len; - return status; - error: - while (!list_empty(&pbufl)) { - buf = (struct i40iw_puda_buf *)(pbufl.prev); - list_del(&buf->list); - list_add(&buf->list, rxlist); - } - if (txbuf) - i40iw_puda_ret_bufpool(ieq, txbuf); - return status; -} - -/** - * i40iw_ieq_process_buf - process buffer rcvd for ieq - * @ieq: ieq resource - * @pfpdu: partial management per user qp - * @buf: receive buffer - */ -static enum i40iw_status_code i40iw_ieq_process_buf(struct i40iw_puda_rsrc *ieq, - struct i40iw_pfpdu *pfpdu, - struct i40iw_puda_buf *buf) -{ - u16 fpdu_len = 0; - u16 datalen = buf->datalen; - u8 *datap = buf->data; - u8 *crcptr; - u16 ioffset = 0; - u32 mpacrc; - u32 seqnum = buf->seqnum; - u16 length = 0; - u16 full = 0; - bool partial = false; - struct i40iw_puda_buf *txbuf; - struct list_head *rxlist = &pfpdu->rxlist; - enum i40iw_status_code ret = 0; - enum i40iw_status_code status = 0; - - ioffset = (u16)(buf->data - (u8 *)buf->mem.va); - while (datalen) { - fpdu_len = i40iw_ieq_get_fpdu_length(ntohs(*(__be16 *)datap)); - if (fpdu_len > pfpdu->max_fpdu_data) { - i40iw_debug(ieq->dev, I40IW_DEBUG_IEQ, - "%s: error bad fpdu_len\n", __func__); - status = I40IW_ERR_MPA_CRC; - list_add(&buf->list, rxlist); - return status; - } - - if (datalen < fpdu_len) { - partial = true; - break; - } - crcptr = datap + fpdu_len - 4; - mpacrc = *(u32 *)crcptr; - if (ieq->check_crc) - ret = i40iw_ieq_check_mpacrc(ieq->hash_desc, - datap, fpdu_len - 4, mpacrc); - if (ret) { - status = I40IW_ERR_MPA_CRC; - list_add(&buf->list, rxlist); - return status; - } - full++; - pfpdu->fpdu_processed++; - datap += fpdu_len; - length += fpdu_len; - datalen -= fpdu_len; - } - if (full) { - /* copy full pdu's in the txbuf and send them out */ - txbuf = i40iw_puda_get_bufpool(ieq); - if (!txbuf) { - pfpdu->no_tx_bufs++; - status = I40IW_ERR_NO_TXBUFS; - list_add(&buf->list, rxlist); - return status; - } - /* modify txbuf's buffer header */ - i40iw_ieq_setup_tx_buf(buf, txbuf); - /* copy full fpdu's to new buffer */ - i40iw_ieq_copy_to_txbuf(buf, txbuf, ioffset, buf->hdrlen, - length); - txbuf->totallen = buf->hdrlen + length; - - i40iw_ieq_update_tcpip_info(txbuf, length, buf->seqnum); - i40iw_puda_send_buf(ieq, txbuf); - - if (!datalen) { - pfpdu->rcv_nxt = buf->seqnum + length; - i40iw_puda_ret_bufpool(ieq, buf); - return status; - } - buf->data = datap; - buf->seqnum = seqnum + length; - buf->datalen = datalen; - pfpdu->rcv_nxt = buf->seqnum; - } - if (partial) - status = i40iw_ieq_handle_partial(ieq, pfpdu, buf, fpdu_len); - - return status; -} - -/** - * i40iw_ieq_process_fpdus - process fpdu's buffers on its list - * @qp: qp for which partial fpdus - * @ieq: ieq resource - */ -static void i40iw_ieq_process_fpdus(struct i40iw_sc_qp *qp, - struct i40iw_puda_rsrc *ieq) -{ - struct i40iw_pfpdu *pfpdu = &qp->pfpdu; - struct list_head *rxlist = &pfpdu->rxlist; - struct i40iw_puda_buf *buf; - enum i40iw_status_code status; - - do { - if (list_empty(rxlist)) - break; - buf = i40iw_puda_get_listbuf(rxlist); - if (!buf) { - i40iw_debug(ieq->dev, I40IW_DEBUG_IEQ, - "%s: error no buf\n", __func__); - break; - } - if (buf->seqnum != pfpdu->rcv_nxt) { - /* This could be out of order or missing packet */ - pfpdu->out_of_order++; - list_add(&buf->list, rxlist); - break; - } - /* keep processing buffers from the head of the list */ - status = i40iw_ieq_process_buf(ieq, pfpdu, buf); - if (status == I40IW_ERR_MPA_CRC) { - pfpdu->mpa_crc_err = true; - while (!list_empty(rxlist)) { - buf = i40iw_puda_get_listbuf(rxlist); - i40iw_puda_ret_bufpool(ieq, buf); - pfpdu->crc_err++; - } - /* create CQP for AE */ - i40iw_ieq_mpa_crc_ae(ieq->dev, qp); - } - } while (!status); -} - -/** - * i40iw_ieq_handle_exception - handle qp's exception - * @ieq: ieq resource - * @qp: qp receiving excpetion - * @buf: receive buffer - */ -static void i40iw_ieq_handle_exception(struct i40iw_puda_rsrc *ieq, - struct i40iw_sc_qp *qp, - struct i40iw_puda_buf *buf) -{ - struct i40iw_puda_buf *tmpbuf = NULL; - struct i40iw_pfpdu *pfpdu = &qp->pfpdu; - u32 *hw_host_ctx = (u32 *)qp->hw_host_ctx; - u32 rcv_wnd = hw_host_ctx[23]; - /* first partial seq # in q2 */ - u32 fps = *(u32 *)(qp->q2_buf + Q2_FPSN_OFFSET); - struct list_head *rxlist = &pfpdu->rxlist; - struct list_head *plist; - - pfpdu->total_ieq_bufs++; - - if (pfpdu->mpa_crc_err) { - pfpdu->crc_err++; - goto error; - } - if (pfpdu->mode && (fps != pfpdu->fps)) { - /* clean up qp as it is new partial sequence */ - i40iw_ieq_cleanup_qp(ieq, qp); - i40iw_debug(ieq->dev, I40IW_DEBUG_IEQ, - "%s: restarting new partial\n", __func__); - pfpdu->mode = false; - } - - if (!pfpdu->mode) { - i40iw_debug_buf(ieq->dev, I40IW_DEBUG_IEQ, "Q2 BUFFER", (u64 *)qp->q2_buf, 128); - /* First_Partial_Sequence_Number check */ - pfpdu->rcv_nxt = fps; - pfpdu->fps = fps; - pfpdu->mode = true; - pfpdu->max_fpdu_data = (buf->ipv4) ? (ieq->vsi->mtu - I40IW_MTU_TO_MSS_IPV4) : - (ieq->vsi->mtu - I40IW_MTU_TO_MSS_IPV6); - pfpdu->pmode_count++; - INIT_LIST_HEAD(rxlist); - i40iw_ieq_check_first_buf(buf, fps); - } - - if (!(rcv_wnd >= (buf->seqnum - pfpdu->rcv_nxt))) { - pfpdu->bad_seq_num++; - goto error; - } - - if (!list_empty(rxlist)) { - tmpbuf = (struct i40iw_puda_buf *)rxlist->next; - while ((struct list_head *)tmpbuf != rxlist) { - if ((int)(buf->seqnum - tmpbuf->seqnum) < 0) - break; - plist = &tmpbuf->list; - tmpbuf = (struct i40iw_puda_buf *)plist->next; - } - /* Insert buf before tmpbuf */ - list_add_tail(&buf->list, &tmpbuf->list); - } else { - list_add_tail(&buf->list, rxlist); - } - i40iw_ieq_process_fpdus(qp, ieq); - return; - error: - i40iw_puda_ret_bufpool(ieq, buf); -} - -/** - * i40iw_ieq_receive - received exception buffer - * @vsi: pointer to vsi structure - * @buf: exception buffer received - */ -static void i40iw_ieq_receive(struct i40iw_sc_vsi *vsi, - struct i40iw_puda_buf *buf) -{ - struct i40iw_puda_rsrc *ieq = vsi->ieq; - struct i40iw_sc_qp *qp = NULL; - u32 wqe_idx = ieq->compl_rxwqe_idx; - - qp = i40iw_ieq_get_qp(vsi->dev, buf); - if (!qp) { - ieq->stats_bad_qp_id++; - i40iw_puda_ret_bufpool(ieq, buf); - } else { - i40iw_ieq_handle_exception(ieq, qp, buf); - } - /* - * ieq->rx_wqe_idx is used by i40iw_puda_replenish_rq() - * on which wqe_idx to start replenish rq - */ - if (!ieq->rxq_invalid_cnt) - ieq->rx_wqe_idx = wqe_idx; - ieq->rxq_invalid_cnt++; -} - -/** - * i40iw_ieq_tx_compl - put back after sending completed exception buffer - * @vsi: pointer to the vsi structure - * @sqwrid: pointer to puda buffer - */ -static void i40iw_ieq_tx_compl(struct i40iw_sc_vsi *vsi, void *sqwrid) -{ - struct i40iw_puda_rsrc *ieq = vsi->ieq; - struct i40iw_puda_buf *buf = (struct i40iw_puda_buf *)sqwrid; - - i40iw_puda_ret_bufpool(ieq, buf); -} - -/** - * i40iw_ieq_cleanup_qp - qp is being destroyed - * @ieq: ieq resource - * @qp: all pending fpdu buffers - */ -void i40iw_ieq_cleanup_qp(struct i40iw_puda_rsrc *ieq, struct i40iw_sc_qp *qp) -{ - struct i40iw_puda_buf *buf; - struct i40iw_pfpdu *pfpdu = &qp->pfpdu; - struct list_head *rxlist = &pfpdu->rxlist; - - if (!pfpdu->mode) - return; - while (!list_empty(rxlist)) { - buf = i40iw_puda_get_listbuf(rxlist); - i40iw_puda_ret_bufpool(ieq, buf); - } -} diff --git a/drivers/infiniband/hw/i40iw/i40iw_puda.h b/drivers/infiniband/hw/i40iw/i40iw_puda.h deleted file mode 100644 index 53a7d58c84b5..000000000000 --- a/drivers/infiniband/hw/i40iw/i40iw_puda.h +++ /dev/null @@ -1,188 +0,0 @@ -/******************************************************************************* -* -* Copyright (c) 2015-2016 Intel Corporation. All rights reserved. -* -* This software is available to you under a choice of one of two -* licenses. You may choose to be licensed under the terms of the GNU -* General Public License (GPL) Version 2, available from the file -* COPYING in the main directory of this source tree, or the -* OpenFabrics.org BSD license below: -* -* Redistribution and use in source and binary forms, with or -* without modification, are permitted provided that the following -* conditions are met: -* -* - Redistributions of source code must retain the above -* copyright notice, this list of conditions and the following -* disclaimer. -* -* - Redistributions in binary form must reproduce the above -* copyright notice, this list of conditions and the following -* disclaimer in the documentation and/or other materials -* provided with the distribution. -* -* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS -* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN -* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -* SOFTWARE. -* -*******************************************************************************/ - -#ifndef I40IW_PUDA_H -#define I40IW_PUDA_H - -#define I40IW_IEQ_MPA_FRAMING 6 - -struct i40iw_sc_dev; -struct i40iw_sc_qp; -struct i40iw_sc_cq; - -enum puda_resource_type { - I40IW_PUDA_RSRC_TYPE_ILQ = 1, - I40IW_PUDA_RSRC_TYPE_IEQ -}; - -enum puda_rsrc_complete { - PUDA_CQ_CREATED = 1, - PUDA_QP_CREATED, - PUDA_TX_COMPLETE, - PUDA_RX_COMPLETE, - PUDA_HASH_CRC_COMPLETE -}; - -struct i40iw_puda_completion_info { - struct i40iw_qp_uk *qp; - u8 q_type; - u8 vlan_valid; - u8 l3proto; - u8 l4proto; - u16 payload_len; - u32 compl_error; /* No_err=0, else major and minor err code */ - u32 qp_id; - u32 wqe_idx; -}; - -struct i40iw_puda_send_info { - u64 paddr; /* Physical address */ - u32 len; - u8 tcplen; - u8 maclen; - bool ipv4; - bool doloopback; - void *scratch; -}; - -struct i40iw_puda_buf { - struct list_head list; /* MUST be first entry */ - struct i40iw_dma_mem mem; /* DMA memory for the buffer */ - struct i40iw_puda_buf *next; /* for alloclist in rsrc struct */ - struct i40iw_virt_mem buf_mem; /* Buffer memory for this buffer */ - void *scratch; - u8 *iph; - u8 *tcph; - u8 *data; - u16 datalen; - u16 vlan_id; - u8 tcphlen; /* tcp length in bytes */ - u8 maclen; /* mac length in bytes */ - u32 totallen; /* machlen+iphlen+tcphlen+datalen */ - atomic_t refcount; - u8 hdrlen; - bool ipv4; - u32 seqnum; -}; - -struct i40iw_puda_rsrc_info { - enum puda_resource_type type; /* ILQ or IEQ */ - u32 count; - u16 pd_id; - u32 cq_id; - u32 qp_id; - u32 sq_size; - u32 rq_size; - u16 buf_size; - u16 mss; - u32 tx_buf_cnt; /* total bufs allocated will be rq_size + tx_buf_cnt */ - void (*receive)(struct i40iw_sc_vsi *, struct i40iw_puda_buf *); - void (*xmit_complete)(struct i40iw_sc_vsi *, void *); -}; - -struct i40iw_puda_rsrc { - struct i40iw_sc_cq cq; - struct i40iw_sc_qp qp; - struct i40iw_sc_pd sc_pd; - struct i40iw_sc_dev *dev; - struct i40iw_sc_vsi *vsi; - struct i40iw_dma_mem cqmem; - struct i40iw_dma_mem qpmem; - struct i40iw_virt_mem ilq_mem; - enum puda_rsrc_complete completion; - enum puda_resource_type type; - u16 buf_size; /*buffer must be max datalen + tcpip hdr + mac */ - u16 mss; - u32 cq_id; - u32 qp_id; - u32 sq_size; - u32 rq_size; - u32 cq_size; - struct i40iw_sq_uk_wr_trk_info *sq_wrtrk_array; - u64 *rq_wrid_array; - u32 compl_rxwqe_idx; - u32 rx_wqe_idx; - u32 rxq_invalid_cnt; - u32 tx_wqe_avail_cnt; - bool check_crc; - struct shash_desc *hash_desc; - struct list_head txpend; - struct list_head bufpool; /* free buffers pool list for recv and xmit */ - u32 alloc_buf_count; - u32 avail_buf_count; /* snapshot of currently available buffers */ - spinlock_t bufpool_lock; - struct i40iw_puda_buf *alloclist; - void (*receive)(struct i40iw_sc_vsi *, struct i40iw_puda_buf *); - void (*xmit_complete)(struct i40iw_sc_vsi *, void *); - /* puda stats */ - u64 stats_buf_alloc_fail; - u64 stats_pkt_rcvd; - u64 stats_pkt_sent; - u64 stats_rcvd_pkt_err; - u64 stats_sent_pkt_q; - u64 stats_bad_qp_id; -}; - -struct i40iw_puda_buf *i40iw_puda_get_bufpool(struct i40iw_puda_rsrc *rsrc); -void i40iw_puda_ret_bufpool(struct i40iw_puda_rsrc *rsrc, - struct i40iw_puda_buf *buf); -void i40iw_puda_send_buf(struct i40iw_puda_rsrc *rsrc, - struct i40iw_puda_buf *buf); -enum i40iw_status_code i40iw_puda_send(struct i40iw_sc_qp *qp, - struct i40iw_puda_send_info *info); -enum i40iw_status_code i40iw_puda_create_rsrc(struct i40iw_sc_vsi *vsi, - struct i40iw_puda_rsrc_info *info); -void i40iw_puda_dele_resources(struct i40iw_sc_vsi *vsi, - enum puda_resource_type type, - bool reset); -enum i40iw_status_code i40iw_puda_poll_completion(struct i40iw_sc_dev *dev, - struct i40iw_sc_cq *cq, u32 *compl_err); - -struct i40iw_sc_qp *i40iw_ieq_get_qp(struct i40iw_sc_dev *dev, - struct i40iw_puda_buf *buf); -enum i40iw_status_code i40iw_puda_get_tcpip_info(struct i40iw_puda_completion_info *info, - struct i40iw_puda_buf *buf); -enum i40iw_status_code i40iw_ieq_check_mpacrc(struct shash_desc *desc, - void *addr, u32 length, u32 value); -enum i40iw_status_code i40iw_init_hash_desc(struct shash_desc **desc); -void i40iw_ieq_mpa_crc_ae(struct i40iw_sc_dev *dev, struct i40iw_sc_qp *qp); -void i40iw_free_hash_desc(struct shash_desc *desc); -void i40iw_ieq_update_tcpip_info(struct i40iw_puda_buf *buf, u16 length, - u32 seqnum); -enum i40iw_status_code i40iw_cqp_qp_create_cmd(struct i40iw_sc_dev *dev, struct i40iw_sc_qp *qp); -enum i40iw_status_code i40iw_cqp_cq_create_cmd(struct i40iw_sc_dev *dev, struct i40iw_sc_cq *cq); -void i40iw_cqp_qp_destroy_cmd(struct i40iw_sc_dev *dev, struct i40iw_sc_qp *qp); -void i40iw_cqp_cq_destroy_cmd(struct i40iw_sc_dev *dev, struct i40iw_sc_cq *cq); -void i40iw_ieq_cleanup_qp(struct i40iw_puda_rsrc *ieq, struct i40iw_sc_qp *qp); -#endif diff --git a/drivers/infiniband/hw/i40iw/i40iw_register.h b/drivers/infiniband/hw/i40iw/i40iw_register.h deleted file mode 100644 index 57768184e251..000000000000 --- a/drivers/infiniband/hw/i40iw/i40iw_register.h +++ /dev/null @@ -1,1030 +0,0 @@ -/******************************************************************************* -* -* Copyright (c) 2015-2016 Intel Corporation. All rights reserved. -* -* This software is available to you under a choice of one of two -* licenses. You may choose to be licensed under the terms of the GNU -* General Public License (GPL) Version 2, available from the file -* COPYING in the main directory of this source tree, or the -* OpenFabrics.org BSD license below: -* -* Redistribution and use in source and binary forms, with or -* without modification, are permitted provided that the following -* conditions are met: -* -* - Redistributions of source code must retain the above -* copyright notice, this list of conditions and the following -* disclaimer. -* -* - Redistributions in binary form must reproduce the above -* copyright notice, this list of conditions and the following -* disclaimer in the documentation and/or other materials -* provided with the distribution. -* -* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS -* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN -* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -* SOFTWARE. -* -*******************************************************************************/ - -#ifndef I40IW_REGISTER_H -#define I40IW_REGISTER_H - -#define I40E_GLGEN_STAT 0x000B612C /* Reset: POR */ - -#define I40E_PFHMC_PDINV 0x000C0300 /* Reset: PFR */ -#define I40E_PFHMC_PDINV_PMSDIDX_SHIFT 0 -#define I40E_PFHMC_PDINV_PMSDIDX_MASK (0xFFF << I40E_PFHMC_PDINV_PMSDIDX_SHIFT) -#define I40E_PFHMC_PDINV_PMPDIDX_SHIFT 16 -#define I40E_PFHMC_PDINV_PMPDIDX_MASK (0x1FF << I40E_PFHMC_PDINV_PMPDIDX_SHIFT) -#define I40E_PFHMC_SDCMD_PMSDWR_SHIFT 31 -#define I40E_PFHMC_SDCMD_PMSDWR_MASK (0x1 << I40E_PFHMC_SDCMD_PMSDWR_SHIFT) -#define I40E_PFHMC_SDDATALOW_PMSDVALID_SHIFT 0 -#define I40E_PFHMC_SDDATALOW_PMSDVALID_MASK (0x1 << I40E_PFHMC_SDDATALOW_PMSDVALID_SHIFT) -#define I40E_PFHMC_SDDATALOW_PMSDTYPE_SHIFT 1 -#define I40E_PFHMC_SDDATALOW_PMSDTYPE_MASK (0x1 << I40E_PFHMC_SDDATALOW_PMSDTYPE_SHIFT) -#define I40E_PFHMC_SDDATALOW_PMSDBPCOUNT_SHIFT 2 -#define I40E_PFHMC_SDDATALOW_PMSDBPCOUNT_MASK (0x3FF << I40E_PFHMC_SDDATALOW_PMSDBPCOUNT_SHIFT) - -#define I40E_PFINT_DYN_CTLN(_INTPF) (0x00034800 + ((_INTPF) * 4)) /* _i=0...511 */ /* Reset: PFR */ -#define I40E_PFINT_DYN_CTLN_INTENA_SHIFT 0 -#define I40E_PFINT_DYN_CTLN_INTENA_MASK (0x1 << I40E_PFINT_DYN_CTLN_INTENA_SHIFT) -#define I40E_PFINT_DYN_CTLN_CLEARPBA_SHIFT 1 -#define I40E_PFINT_DYN_CTLN_CLEARPBA_MASK (0x1 << I40E_PFINT_DYN_CTLN_CLEARPBA_SHIFT) -#define I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT 3 -#define I40E_PFINT_DYN_CTLN_ITR_INDX_MASK (0x3 << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT) - -#define I40E_VFINT_DYN_CTLN1(_INTVF) (0x00003800 + ((_INTVF) * 4)) /* _i=0...15 */ /* Reset: VFR */ -#define I40E_GLHMC_VFPDINV(_i) (0x000C8300 + ((_i) * 4)) /* _i=0...31 */ /* Reset: CORER */ - -#define I40E_PFHMC_PDINV_PMSDPARTSEL_SHIFT 15 -#define I40E_PFHMC_PDINV_PMSDPARTSEL_MASK (0x1 << I40E_PFHMC_PDINV_PMSDPARTSEL_SHIFT) -#define I40E_GLPCI_LBARCTRL 0x000BE484 /* Reset: POR */ -#define I40E_GLPCI_LBARCTRL_PE_DB_SIZE_SHIFT 4 -#define I40E_GLPCI_LBARCTRL_PE_DB_SIZE_MASK (0x3 << I40E_GLPCI_LBARCTRL_PE_DB_SIZE_SHIFT) -#define I40E_GLPCI_DREVID 0x0009C480 /* Reset: PCIR */ -#define I40E_GLPCI_DREVID_DEFAULT_REVID_SHIFT 0 -#define I40E_GLPCI_DREVID_DEFAULT_REVID_MASK 0xFF - -#define I40E_PFPE_AEQALLOC 0x00131180 /* Reset: PFR */ -#define I40E_PFPE_AEQALLOC_AECOUNT_SHIFT 0 -#define I40E_PFPE_AEQALLOC_AECOUNT_MASK (0xFFFFFFFF << I40E_PFPE_AEQALLOC_AECOUNT_SHIFT) -#define I40E_PFPE_CCQPHIGH 0x00008200 /* Reset: PFR */ -#define I40E_PFPE_CCQPHIGH_PECCQPHIGH_SHIFT 0 -#define I40E_PFPE_CCQPHIGH_PECCQPHIGH_MASK (0xFFFFFFFF << I40E_PFPE_CCQPHIGH_PECCQPHIGH_SHIFT) -#define I40E_PFPE_CCQPLOW 0x00008180 /* Reset: PFR */ -#define I40E_PFPE_CCQPLOW_PECCQPLOW_SHIFT 0 -#define I40E_PFPE_CCQPLOW_PECCQPLOW_MASK (0xFFFFFFFF << I40E_PFPE_CCQPLOW_PECCQPLOW_SHIFT) -#define I40E_PFPE_CCQPSTATUS 0x00008100 /* Reset: PFR */ -#define I40E_PFPE_CCQPSTATUS_CCQP_DONE_SHIFT 0 -#define I40E_PFPE_CCQPSTATUS_CCQP_DONE_MASK (0x1 << I40E_PFPE_CCQPSTATUS_CCQP_DONE_SHIFT) -#define I40E_PFPE_CCQPSTATUS_HMC_PROFILE_SHIFT 4 -#define I40E_PFPE_CCQPSTATUS_HMC_PROFILE_MASK (0x7 << I40E_PFPE_CCQPSTATUS_HMC_PROFILE_SHIFT) -#define I40E_PFPE_CCQPSTATUS_RDMA_EN_VFS_SHIFT 16 -#define I40E_PFPE_CCQPSTATUS_RDMA_EN_VFS_MASK (0x3F << I40E_PFPE_CCQPSTATUS_RDMA_EN_VFS_SHIFT) -#define I40E_PFPE_CCQPSTATUS_CCQP_ERR_SHIFT 31 -#define I40E_PFPE_CCQPSTATUS_CCQP_ERR_MASK (0x1 << I40E_PFPE_CCQPSTATUS_CCQP_ERR_SHIFT) -#define I40E_PFPE_CQACK 0x00131100 /* Reset: PFR */ -#define I40E_PFPE_CQACK_PECQID_SHIFT 0 -#define I40E_PFPE_CQACK_PECQID_MASK (0x1FFFF << I40E_PFPE_CQACK_PECQID_SHIFT) -#define I40E_PFPE_CQARM 0x00131080 /* Reset: PFR */ -#define I40E_PFPE_CQARM_PECQID_SHIFT 0 -#define I40E_PFPE_CQARM_PECQID_MASK (0x1FFFF << I40E_PFPE_CQARM_PECQID_SHIFT) -#define I40E_PFPE_CQPDB 0x00008000 /* Reset: PFR */ -#define I40E_PFPE_CQPDB_WQHEAD_SHIFT 0 -#define I40E_PFPE_CQPDB_WQHEAD_MASK (0x7FF << I40E_PFPE_CQPDB_WQHEAD_SHIFT) -#define I40E_PFPE_CQPERRCODES 0x00008880 /* Reset: PFR */ -#define I40E_PFPE_CQPERRCODES_CQP_MINOR_CODE_SHIFT 0 -#define I40E_PFPE_CQPERRCODES_CQP_MINOR_CODE_MASK (0xFFFF << I40E_PFPE_CQPERRCODES_CQP_MINOR_CODE_SHIFT) -#define I40E_PFPE_CQPERRCODES_CQP_MAJOR_CODE_SHIFT 16 -#define I40E_PFPE_CQPERRCODES_CQP_MAJOR_CODE_MASK (0xFFFF << I40E_PFPE_CQPERRCODES_CQP_MAJOR_CODE_SHIFT) -#define I40E_PFPE_CQPTAIL 0x00008080 /* Reset: PFR */ -#define I40E_PFPE_CQPTAIL_WQTAIL_SHIFT 0 -#define I40E_PFPE_CQPTAIL_WQTAIL_MASK (0x7FF << I40E_PFPE_CQPTAIL_WQTAIL_SHIFT) -#define I40E_PFPE_CQPTAIL_CQP_OP_ERR_SHIFT 31 -#define I40E_PFPE_CQPTAIL_CQP_OP_ERR_MASK (0x1 << I40E_PFPE_CQPTAIL_CQP_OP_ERR_SHIFT) -#define I40E_PFPE_FLMQ1ALLOCERR 0x00008980 /* Reset: PFR */ -#define I40E_PFPE_FLMQ1ALLOCERR_ERROR_COUNT_SHIFT 0 -#define I40E_PFPE_FLMQ1ALLOCERR_ERROR_COUNT_MASK (0xFFFF << I40E_PFPE_FLMQ1ALLOCERR_ERROR_COUNT_SHIFT) -#define I40E_PFPE_FLMXMITALLOCERR 0x00008900 /* Reset: PFR */ -#define I40E_PFPE_FLMXMITALLOCERR_ERROR_COUNT_SHIFT 0 -#define I40E_PFPE_FLMXMITALLOCERR_ERROR_COUNT_MASK (0xFFFF << I40E_PFPE_FLMXMITALLOCERR_ERROR_COUNT_SHIFT) -#define I40E_PFPE_IPCONFIG0 0x00008280 /* Reset: PFR */ -#define I40E_PFPE_IPCONFIG0_PEIPID_SHIFT 0 -#define I40E_PFPE_IPCONFIG0_PEIPID_MASK (0xFFFF << I40E_PFPE_IPCONFIG0_PEIPID_SHIFT) -#define I40E_PFPE_IPCONFIG0_USEENTIREIDRANGE_SHIFT 16 -#define I40E_PFPE_IPCONFIG0_USEENTIREIDRANGE_MASK (0x1 << I40E_PFPE_IPCONFIG0_USEENTIREIDRANGE_SHIFT) -#define I40E_PFPE_MRTEIDXMASK 0x00008600 /* Reset: PFR */ -#define I40E_PFPE_MRTEIDXMASK_MRTEIDXMASKBITS_SHIFT 0 -#define I40E_PFPE_MRTEIDXMASK_MRTEIDXMASKBITS_MASK (0x1F << I40E_PFPE_MRTEIDXMASK_MRTEIDXMASKBITS_SHIFT) -#define I40E_PFPE_RCVUNEXPECTEDERROR 0x00008680 /* Reset: PFR */ -#define I40E_PFPE_RCVUNEXPECTEDERROR_TCP_RX_UNEXP_ERR_SHIFT 0 -#define I40E_PFPE_RCVUNEXPECTEDERROR_TCP_RX_UNEXP_ERR_MASK (0xFFFFFF << I40E_PFPE_RCVUNEXPECTEDERROR_TCP_RX_UNEXP_ERR_SHIFT) -#define I40E_PFPE_TCPNOWTIMER 0x00008580 /* Reset: PFR */ -#define I40E_PFPE_TCPNOWTIMER_TCP_NOW_SHIFT 0 -#define I40E_PFPE_TCPNOWTIMER_TCP_NOW_MASK (0xFFFFFFFF << I40E_PFPE_TCPNOWTIMER_TCP_NOW_SHIFT) - -#define I40E_PFPE_WQEALLOC 0x00138C00 /* Reset: PFR */ -#define I40E_PFPE_WQEALLOC_PEQPID_SHIFT 0 -#define I40E_PFPE_WQEALLOC_PEQPID_MASK (0x3FFFF << I40E_PFPE_WQEALLOC_PEQPID_SHIFT) -#define I40E_PFPE_WQEALLOC_WQE_DESC_INDEX_SHIFT 20 -#define I40E_PFPE_WQEALLOC_WQE_DESC_INDEX_MASK (0xFFF << I40E_PFPE_WQEALLOC_WQE_DESC_INDEX_SHIFT) - -#define I40E_VFPE_AEQALLOC(_VF) (0x00130C00 + ((_VF) * 4)) /* _i=0...127 */ /* Reset: VFR */ -#define I40E_VFPE_AEQALLOC_MAX_INDEX 127 -#define I40E_VFPE_AEQALLOC_AECOUNT_SHIFT 0 -#define I40E_VFPE_AEQALLOC_AECOUNT_MASK (0xFFFFFFFF << I40E_VFPE_AEQALLOC_AECOUNT_SHIFT) -#define I40E_VFPE_CCQPHIGH(_VF) (0x00001000 + ((_VF) * 4)) /* _i=0...127 */ /* Reset: VFR */ -#define I40E_VFPE_CCQPHIGH_MAX_INDEX 127 -#define I40E_VFPE_CCQPHIGH_PECCQPHIGH_SHIFT 0 -#define I40E_VFPE_CCQPHIGH_PECCQPHIGH_MASK (0xFFFFFFFF << I40E_VFPE_CCQPHIGH_PECCQPHIGH_SHIFT) -#define I40E_VFPE_CCQPLOW(_VF) (0x00000C00 + ((_VF) * 4)) /* _i=0...127 */ /* Reset: VFR */ -#define I40E_VFPE_CCQPLOW_MAX_INDEX 127 -#define I40E_VFPE_CCQPLOW_PECCQPLOW_SHIFT 0 -#define I40E_VFPE_CCQPLOW_PECCQPLOW_MASK (0xFFFFFFFF << I40E_VFPE_CCQPLOW_PECCQPLOW_SHIFT) -#define I40E_VFPE_CCQPSTATUS(_VF) (0x00000800 + ((_VF) * 4)) /* _i=0...127 */ /* Reset: VFR */ -#define I40E_VFPE_CCQPSTATUS_MAX_INDEX 127 -#define I40E_VFPE_CCQPSTATUS_CCQP_DONE_SHIFT 0 -#define I40E_VFPE_CCQPSTATUS_CCQP_DONE_MASK (0x1 << I40E_VFPE_CCQPSTATUS_CCQP_DONE_SHIFT) -#define I40E_VFPE_CCQPSTATUS_HMC_PROFILE_SHIFT 4 -#define I40E_VFPE_CCQPSTATUS_HMC_PROFILE_MASK (0x7 << I40E_VFPE_CCQPSTATUS_HMC_PROFILE_SHIFT) -#define I40E_VFPE_CCQPSTATUS_RDMA_EN_VFS_SHIFT 16 -#define I40E_VFPE_CCQPSTATUS_RDMA_EN_VFS_MASK (0x3F << I40E_VFPE_CCQPSTATUS_RDMA_EN_VFS_SHIFT) -#define I40E_VFPE_CCQPSTATUS_CCQP_ERR_SHIFT 31 -#define I40E_VFPE_CCQPSTATUS_CCQP_ERR_MASK (0x1 << I40E_VFPE_CCQPSTATUS_CCQP_ERR_SHIFT) -#define I40E_VFPE_CQACK(_VF) (0x00130800 + ((_VF) * 4)) /* _i=0...127 */ /* Reset: VFR */ -#define I40E_VFPE_CQACK_MAX_INDEX 127 -#define I40E_VFPE_CQACK_PECQID_SHIFT 0 -#define I40E_VFPE_CQACK_PECQID_MASK (0x1FFFF << I40E_VFPE_CQACK_PECQID_SHIFT) -#define I40E_VFPE_CQARM(_VF) (0x00130400 + ((_VF) * 4)) /* _i=0...127 */ /* Reset: VFR */ -#define I40E_VFPE_CQARM_MAX_INDEX 127 -#define I40E_VFPE_CQARM_PECQID_SHIFT 0 -#define I40E_VFPE_CQARM_PECQID_MASK (0x1FFFF << I40E_VFPE_CQARM_PECQID_SHIFT) -#define I40E_VFPE_CQPDB(_VF) (0x00000000 + ((_VF) * 4)) /* _i=0...127 */ /* Reset: VFR */ -#define I40E_VFPE_CQPDB_MAX_INDEX 127 -#define I40E_VFPE_CQPDB_WQHEAD_SHIFT 0 -#define I40E_VFPE_CQPDB_WQHEAD_MASK (0x7FF << I40E_VFPE_CQPDB_WQHEAD_SHIFT) -#define I40E_VFPE_CQPERRCODES(_VF) (0x00001800 + ((_VF) * 4)) /* _i=0...127 */ /* Reset: VFR */ -#define I40E_VFPE_CQPERRCODES_MAX_INDEX 127 -#define I40E_VFPE_CQPERRCODES_CQP_MINOR_CODE_SHIFT 0 -#define I40E_VFPE_CQPERRCODES_CQP_MINOR_CODE_MASK (0xFFFF << I40E_VFPE_CQPERRCODES_CQP_MINOR_CODE_SHIFT) -#define I40E_VFPE_CQPERRCODES_CQP_MAJOR_CODE_SHIFT 16 -#define I40E_VFPE_CQPERRCODES_CQP_MAJOR_CODE_MASK (0xFFFF << I40E_VFPE_CQPERRCODES_CQP_MAJOR_CODE_SHIFT) -#define I40E_VFPE_CQPTAIL(_VF) (0x00000400 + ((_VF) * 4)) /* _i=0...127 */ /* Reset: VFR */ -#define I40E_VFPE_CQPTAIL_MAX_INDEX 127 -#define I40E_VFPE_CQPTAIL_WQTAIL_SHIFT 0 -#define I40E_VFPE_CQPTAIL_WQTAIL_MASK (0x7FF << I40E_VFPE_CQPTAIL_WQTAIL_SHIFT) -#define I40E_VFPE_CQPTAIL_CQP_OP_ERR_SHIFT 31 -#define I40E_VFPE_CQPTAIL_CQP_OP_ERR_MASK (0x1 << I40E_VFPE_CQPTAIL_CQP_OP_ERR_SHIFT) -#define I40E_VFPE_IPCONFIG0(_VF) (0x00001400 + ((_VF) * 4)) /* _i=0...127 */ /* Reset: VFR */ -#define I40E_VFPE_IPCONFIG0_MAX_INDEX 127 -#define I40E_VFPE_IPCONFIG0_PEIPID_SHIFT 0 -#define I40E_VFPE_IPCONFIG0_PEIPID_MASK (0xFFFF << I40E_VFPE_IPCONFIG0_PEIPID_SHIFT) -#define I40E_VFPE_IPCONFIG0_USEENTIREIDRANGE_SHIFT 16 -#define I40E_VFPE_IPCONFIG0_USEENTIREIDRANGE_MASK (0x1 << I40E_VFPE_IPCONFIG0_USEENTIREIDRANGE_SHIFT) -#define I40E_VFPE_MRTEIDXMASK(_VF) (0x00003000 + ((_VF) * 4)) /* _i=0...127 */ /* Reset: VFR */ -#define I40E_VFPE_MRTEIDXMASK_MAX_INDEX 127 -#define I40E_VFPE_MRTEIDXMASK_MRTEIDXMASKBITS_SHIFT 0 -#define I40E_VFPE_MRTEIDXMASK_MRTEIDXMASKBITS_MASK (0x1F << I40E_VFPE_MRTEIDXMASK_MRTEIDXMASKBITS_SHIFT) -#define I40E_VFPE_RCVUNEXPECTEDERROR(_VF) (0x00003400 + ((_VF) * 4)) /* _i=0...127 */ /* Reset: VFR */ -#define I40E_VFPE_RCVUNEXPECTEDERROR_MAX_INDEX 127 -#define I40E_VFPE_RCVUNEXPECTEDERROR_TCP_RX_UNEXP_ERR_SHIFT 0 -#define I40E_VFPE_RCVUNEXPECTEDERROR_TCP_RX_UNEXP_ERR_MASK (0xFFFFFF << I40E_VFPE_RCVUNEXPECTEDERROR_TCP_RX_UNEXP_ERR_SHIFT) -#define I40E_VFPE_TCPNOWTIMER(_VF) (0x00002C00 + ((_VF) * 4)) /* _i=0...127 */ /* Reset: VFR */ -#define I40E_VFPE_TCPNOWTIMER_MAX_INDEX 127 -#define I40E_VFPE_TCPNOWTIMER_TCP_NOW_SHIFT 0 -#define I40E_VFPE_TCPNOWTIMER_TCP_NOW_MASK (0xFFFFFFFF << I40E_VFPE_TCPNOWTIMER_TCP_NOW_SHIFT) -#define I40E_VFPE_WQEALLOC(_VF) (0x00138000 + ((_VF) * 4)) /* _i=0...127 */ /* Reset: VFR */ -#define I40E_VFPE_WQEALLOC_MAX_INDEX 127 -#define I40E_VFPE_WQEALLOC_PEQPID_SHIFT 0 -#define I40E_VFPE_WQEALLOC_PEQPID_MASK (0x3FFFF << I40E_VFPE_WQEALLOC_PEQPID_SHIFT) -#define I40E_VFPE_WQEALLOC_WQE_DESC_INDEX_SHIFT 20 -#define I40E_VFPE_WQEALLOC_WQE_DESC_INDEX_MASK (0xFFF << I40E_VFPE_WQEALLOC_WQE_DESC_INDEX_SHIFT) - -#define I40E_GLPE_CPUSTATUS0 0x0000D040 /* Reset: PE_CORER */ -#define I40E_GLPE_CPUSTATUS0_PECPUSTATUS0_SHIFT 0 -#define I40E_GLPE_CPUSTATUS0_PECPUSTATUS0_MASK (0xFFFFFFFF << I40E_GLPE_CPUSTATUS0_PECPUSTATUS0_SHIFT) -#define I40E_GLPE_CPUSTATUS1 0x0000D044 /* Reset: PE_CORER */ -#define I40E_GLPE_CPUSTATUS1_PECPUSTATUS1_SHIFT 0 -#define I40E_GLPE_CPUSTATUS1_PECPUSTATUS1_MASK (0xFFFFFFFF << I40E_GLPE_CPUSTATUS1_PECPUSTATUS1_SHIFT) -#define I40E_GLPE_CPUSTATUS2 0x0000D048 /* Reset: PE_CORER */ -#define I40E_GLPE_CPUSTATUS2_PECPUSTATUS2_SHIFT 0 -#define I40E_GLPE_CPUSTATUS2_PECPUSTATUS2_MASK (0xFFFFFFFF << I40E_GLPE_CPUSTATUS2_PECPUSTATUS2_SHIFT) -#define I40E_GLPE_CPUTRIG0 0x0000D060 /* Reset: PE_CORER */ -#define I40E_GLPE_CPUTRIG0_PECPUTRIG0_SHIFT 0 -#define I40E_GLPE_CPUTRIG0_PECPUTRIG0_MASK (0xFFFF << I40E_GLPE_CPUTRIG0_PECPUTRIG0_SHIFT) -#define I40E_GLPE_CPUTRIG0_TEPREQUEST0_SHIFT 17 -#define I40E_GLPE_CPUTRIG0_TEPREQUEST0_MASK (0x1 << I40E_GLPE_CPUTRIG0_TEPREQUEST0_SHIFT) -#define I40E_GLPE_CPUTRIG0_OOPREQUEST0_SHIFT 18 -#define I40E_GLPE_CPUTRIG0_OOPREQUEST0_MASK (0x1 << I40E_GLPE_CPUTRIG0_OOPREQUEST0_SHIFT) -#define I40E_GLPE_DUAL40_RUPM 0x0000DA04 /* Reset: PE_CORER */ -#define I40E_GLPE_DUAL40_RUPM_DUAL_40G_MODE_SHIFT 0 -#define I40E_GLPE_DUAL40_RUPM_DUAL_40G_MODE_MASK (0x1 << I40E_GLPE_DUAL40_RUPM_DUAL_40G_MODE_SHIFT) -#define I40E_GLPE_PFAEQEDROPCNT(_i) (0x00131440 + ((_i) * 4)) /* _i=0...15 */ /* Reset: CORER */ -#define I40E_GLPE_PFAEQEDROPCNT_MAX_INDEX 15 -#define I40E_GLPE_PFAEQEDROPCNT_AEQEDROPCNT_SHIFT 0 -#define I40E_GLPE_PFAEQEDROPCNT_AEQEDROPCNT_MASK (0xFFFF << I40E_GLPE_PFAEQEDROPCNT_AEQEDROPCNT_SHIFT) -#define I40E_GLPE_PFCEQEDROPCNT(_i) (0x001313C0 + ((_i) * 4)) /* _i=0...15 */ /* Reset: CORER */ -#define I40E_GLPE_PFCEQEDROPCNT_MAX_INDEX 15 -#define I40E_GLPE_PFCEQEDROPCNT_CEQEDROPCNT_SHIFT 0 -#define I40E_GLPE_PFCEQEDROPCNT_CEQEDROPCNT_MASK (0xFFFF << I40E_GLPE_PFCEQEDROPCNT_CEQEDROPCNT_SHIFT) -#define I40E_GLPE_PFCQEDROPCNT(_i) (0x00131340 + ((_i) * 4)) /* _i=0...15 */ /* Reset: CORER */ -#define I40E_GLPE_PFCQEDROPCNT_MAX_INDEX 15 -#define I40E_GLPE_PFCQEDROPCNT_CQEDROPCNT_SHIFT 0 -#define I40E_GLPE_PFCQEDROPCNT_CQEDROPCNT_MASK (0xFFFF << I40E_GLPE_PFCQEDROPCNT_CQEDROPCNT_SHIFT) -#define I40E_GLPE_RUPM_CQPPOOL 0x0000DACC /* Reset: PE_CORER */ -#define I40E_GLPE_RUPM_CQPPOOL_CQPSPADS_SHIFT 0 -#define I40E_GLPE_RUPM_CQPPOOL_CQPSPADS_MASK (0xFF << I40E_GLPE_RUPM_CQPPOOL_CQPSPADS_SHIFT) -#define I40E_GLPE_RUPM_FLRPOOL 0x0000DAC4 /* Reset: PE_CORER */ -#define I40E_GLPE_RUPM_FLRPOOL_FLRSPADS_SHIFT 0 -#define I40E_GLPE_RUPM_FLRPOOL_FLRSPADS_MASK (0xFF << I40E_GLPE_RUPM_FLRPOOL_FLRSPADS_SHIFT) -#define I40E_GLPE_RUPM_GCTL 0x0000DA00 /* Reset: PE_CORER */ -#define I40E_GLPE_RUPM_GCTL_ALLOFFTH_SHIFT 0 -#define I40E_GLPE_RUPM_GCTL_ALLOFFTH_MASK (0xFF << I40E_GLPE_RUPM_GCTL_ALLOFFTH_SHIFT) -#define I40E_GLPE_RUPM_GCTL_RUPM_P0_DIS_SHIFT 26 -#define I40E_GLPE_RUPM_GCTL_RUPM_P0_DIS_MASK (0x1 << I40E_GLPE_RUPM_GCTL_RUPM_P0_DIS_SHIFT) -#define I40E_GLPE_RUPM_GCTL_RUPM_P1_DIS_SHIFT 27 -#define I40E_GLPE_RUPM_GCTL_RUPM_P1_DIS_MASK (0x1 << I40E_GLPE_RUPM_GCTL_RUPM_P1_DIS_SHIFT) -#define I40E_GLPE_RUPM_GCTL_RUPM_P2_DIS_SHIFT 28 -#define I40E_GLPE_RUPM_GCTL_RUPM_P2_DIS_MASK (0x1 << I40E_GLPE_RUPM_GCTL_RUPM_P2_DIS_SHIFT) -#define I40E_GLPE_RUPM_GCTL_RUPM_P3_DIS_SHIFT 29 -#define I40E_GLPE_RUPM_GCTL_RUPM_P3_DIS_MASK (0x1 << I40E_GLPE_RUPM_GCTL_RUPM_P3_DIS_SHIFT) -#define I40E_GLPE_RUPM_GCTL_RUPM_DIS_SHIFT 30 -#define I40E_GLPE_RUPM_GCTL_RUPM_DIS_MASK (0x1 << I40E_GLPE_RUPM_GCTL_RUPM_DIS_SHIFT) -#define I40E_GLPE_RUPM_GCTL_SWLB_MODE_SHIFT 31 -#define I40E_GLPE_RUPM_GCTL_SWLB_MODE_MASK (0x1 << I40E_GLPE_RUPM_GCTL_SWLB_MODE_SHIFT) -#define I40E_GLPE_RUPM_PTXPOOL 0x0000DAC8 /* Reset: PE_CORER */ -#define I40E_GLPE_RUPM_PTXPOOL_PTXSPADS_SHIFT 0 -#define I40E_GLPE_RUPM_PTXPOOL_PTXSPADS_MASK (0xFF << I40E_GLPE_RUPM_PTXPOOL_PTXSPADS_SHIFT) -#define I40E_GLPE_RUPM_PUSHPOOL 0x0000DAC0 /* Reset: PE_CORER */ -#define I40E_GLPE_RUPM_PUSHPOOL_PUSHSPADS_SHIFT 0 -#define I40E_GLPE_RUPM_PUSHPOOL_PUSHSPADS_MASK (0xFF << I40E_GLPE_RUPM_PUSHPOOL_PUSHSPADS_SHIFT) -#define I40E_GLPE_RUPM_TXHOST_EN 0x0000DA08 /* Reset: PE_CORER */ -#define I40E_GLPE_RUPM_TXHOST_EN_TXHOST_EN_SHIFT 0 -#define I40E_GLPE_RUPM_TXHOST_EN_TXHOST_EN_MASK (0x1 << I40E_GLPE_RUPM_TXHOST_EN_TXHOST_EN_SHIFT) -#define I40E_GLPE_VFAEQEDROPCNT(_i) (0x00132540 + ((_i) * 4)) /* _i=0...31 */ /* Reset: CORER */ -#define I40E_GLPE_VFAEQEDROPCNT_MAX_INDEX 31 -#define I40E_GLPE_VFAEQEDROPCNT_AEQEDROPCNT_SHIFT 0 -#define I40E_GLPE_VFAEQEDROPCNT_AEQEDROPCNT_MASK (0xFFFF << I40E_GLPE_VFAEQEDROPCNT_AEQEDROPCNT_SHIFT) -#define I40E_GLPE_VFCEQEDROPCNT(_i) (0x00132440 + ((_i) * 4)) /* _i=0...31 */ /* Reset: CORER */ -#define I40E_GLPE_VFCEQEDROPCNT_MAX_INDEX 31 -#define I40E_GLPE_VFCEQEDROPCNT_CEQEDROPCNT_SHIFT 0 -#define I40E_GLPE_VFCEQEDROPCNT_CEQEDROPCNT_MASK (0xFFFF << I40E_GLPE_VFCEQEDROPCNT_CEQEDROPCNT_SHIFT) -#define I40E_GLPE_VFCQEDROPCNT(_i) (0x00132340 + ((_i) * 4)) /* _i=0...31 */ /* Reset: CORER */ -#define I40E_GLPE_VFCQEDROPCNT_MAX_INDEX 31 -#define I40E_GLPE_VFCQEDROPCNT_CQEDROPCNT_SHIFT 0 -#define I40E_GLPE_VFCQEDROPCNT_CQEDROPCNT_MASK (0xFFFF << I40E_GLPE_VFCQEDROPCNT_CQEDROPCNT_SHIFT) -#define I40E_GLPE_VFFLMOBJCTRL(_i) (0x0000D400 + ((_i) * 4)) /* _i=0...31 */ /* Reset: PE_CORER */ -#define I40E_GLPE_VFFLMOBJCTRL_MAX_INDEX 31 -#define I40E_GLPE_VFFLMOBJCTRL_XMIT_BLOCKSIZE_SHIFT 0 -#define I40E_GLPE_VFFLMOBJCTRL_XMIT_BLOCKSIZE_MASK (0x7 << I40E_GLPE_VFFLMOBJCTRL_XMIT_BLOCKSIZE_SHIFT) -#define I40E_GLPE_VFFLMOBJCTRL_Q1_BLOCKSIZE_SHIFT 8 -#define I40E_GLPE_VFFLMOBJCTRL_Q1_BLOCKSIZE_MASK (0x7 << I40E_GLPE_VFFLMOBJCTRL_Q1_BLOCKSIZE_SHIFT) -#define I40E_GLPE_VFFLMQ1ALLOCERR(_i) (0x0000C700 + ((_i) * 4)) /* _i=0...31 */ /* Reset: PE_CORER */ -#define I40E_GLPE_VFFLMQ1ALLOCERR_MAX_INDEX 31 -#define I40E_GLPE_VFFLMQ1ALLOCERR_ERROR_COUNT_SHIFT 0 -#define I40E_GLPE_VFFLMQ1ALLOCERR_ERROR_COUNT_MASK (0xFFFF << I40E_GLPE_VFFLMQ1ALLOCERR_ERROR_COUNT_SHIFT) -#define I40E_GLPE_VFFLMXMITALLOCERR(_i) (0x0000C600 + ((_i) * 4)) /* _i=0...31 */ /* Reset: PE_CORER */ -#define I40E_GLPE_VFFLMXMITALLOCERR_MAX_INDEX 31 -#define I40E_GLPE_VFFLMXMITALLOCERR_ERROR_COUNT_SHIFT 0 -#define I40E_GLPE_VFFLMXMITALLOCERR_ERROR_COUNT_MASK (0xFFFF << I40E_GLPE_VFFLMXMITALLOCERR_ERROR_COUNT_SHIFT) -#define I40E_GLPE_VFUDACTRL(_i) (0x0000C000 + ((_i) * 4)) /* _i=0...31 */ /* Reset: PE_CORER */ -#define I40E_GLPE_VFUDACTRL_MAX_INDEX 31 -#define I40E_GLPE_VFUDACTRL_IPV4MCFRAGRESBP_SHIFT 0 -#define I40E_GLPE_VFUDACTRL_IPV4MCFRAGRESBP_MASK (0x1 << I40E_GLPE_VFUDACTRL_IPV4MCFRAGRESBP_SHIFT) -#define I40E_GLPE_VFUDACTRL_IPV4UCFRAGRESBP_SHIFT 1 -#define I40E_GLPE_VFUDACTRL_IPV4UCFRAGRESBP_MASK (0x1 << I40E_GLPE_VFUDACTRL_IPV4UCFRAGRESBP_SHIFT) -#define I40E_GLPE_VFUDACTRL_IPV6MCFRAGRESBP_SHIFT 2 -#define I40E_GLPE_VFUDACTRL_IPV6MCFRAGRESBP_MASK (0x1 << I40E_GLPE_VFUDACTRL_IPV6MCFRAGRESBP_SHIFT) -#define I40E_GLPE_VFUDACTRL_IPV6UCFRAGRESBP_SHIFT 3 -#define I40E_GLPE_VFUDACTRL_IPV6UCFRAGRESBP_MASK (0x1 << I40E_GLPE_VFUDACTRL_IPV6UCFRAGRESBP_SHIFT) -#define I40E_GLPE_VFUDACTRL_UDPMCFRAGRESFAIL_SHIFT 4 -#define I40E_GLPE_VFUDACTRL_UDPMCFRAGRESFAIL_MASK (0x1 << I40E_GLPE_VFUDACTRL_UDPMCFRAGRESFAIL_SHIFT) -#define I40E_GLPE_VFUDAUCFBQPN(_i) (0x0000C100 + ((_i) * 4)) /* _i=0...31 */ /* Reset: PE_CORER */ -#define I40E_GLPE_VFUDAUCFBQPN_MAX_INDEX 31 -#define I40E_GLPE_VFUDAUCFBQPN_QPN_SHIFT 0 -#define I40E_GLPE_VFUDAUCFBQPN_QPN_MASK (0x3FFFF << I40E_GLPE_VFUDAUCFBQPN_QPN_SHIFT) -#define I40E_GLPE_VFUDAUCFBQPN_VALID_SHIFT 31 -#define I40E_GLPE_VFUDAUCFBQPN_VALID_MASK (0x1 << I40E_GLPE_VFUDAUCFBQPN_VALID_SHIFT) - -#define I40E_GLPES_PFIP4RXDISCARD(_i) (0x00010600 + ((_i) * 4)) /* _i=0...15 */ /* Reset: PE_CORER */ -#define I40E_GLPES_PFIP4RXDISCARD_MAX_INDEX 15 -#define I40E_GLPES_PFIP4RXDISCARD_IP4RXDISCARD_SHIFT 0 -#define I40E_GLPES_PFIP4RXDISCARD_IP4RXDISCARD_MASK (0xFFFFFFFF << I40E_GLPES_PFIP4RXDISCARD_IP4RXDISCARD_SHIFT) -#define I40E_GLPES_PFIP4RXFRAGSHI(_i) (0x00010804 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */ -#define I40E_GLPES_PFIP4RXFRAGSHI_MAX_INDEX 15 -#define I40E_GLPES_PFIP4RXFRAGSHI_IP4RXFRAGSHI_SHIFT 0 -#define I40E_GLPES_PFIP4RXFRAGSHI_IP4RXFRAGSHI_MASK (0xFFFF << I40E_GLPES_PFIP4RXFRAGSHI_IP4RXFRAGSHI_SHIFT) -#define I40E_GLPES_PFIP4RXFRAGSLO(_i) (0x00010800 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */ -#define I40E_GLPES_PFIP4RXFRAGSLO_MAX_INDEX 15 -#define I40E_GLPES_PFIP4RXFRAGSLO_IP4RXFRAGSLO_SHIFT 0 -#define I40E_GLPES_PFIP4RXFRAGSLO_IP4RXFRAGSLO_MASK (0xFFFFFFFF << I40E_GLPES_PFIP4RXFRAGSLO_IP4RXFRAGSLO_SHIFT) -#define I40E_GLPES_PFIP4RXMCOCTSHI(_i) (0x00010A04 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */ -#define I40E_GLPES_PFIP4RXMCOCTSHI_MAX_INDEX 15 -#define I40E_GLPES_PFIP4RXMCOCTSHI_IP4RXMCOCTSHI_SHIFT 0 -#define I40E_GLPES_PFIP4RXMCOCTSHI_IP4RXMCOCTSHI_MASK (0xFFFF << I40E_GLPES_PFIP4RXMCOCTSHI_IP4RXMCOCTSHI_SHIFT) -#define I40E_GLPES_PFIP4RXMCOCTSLO(_i) (0x00010A00 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */ -#define I40E_GLPES_PFIP4RXMCOCTSLO_MAX_INDEX 15 -#define I40E_GLPES_PFIP4RXMCOCTSLO_IP4RXMCOCTSLO_SHIFT 0 -#define I40E_GLPES_PFIP4RXMCOCTSLO_IP4RXMCOCTSLO_MASK (0xFFFFFFFF << I40E_GLPES_PFIP4RXMCOCTSLO_IP4RXMCOCTSLO_SHIFT) -#define I40E_GLPES_PFIP4RXMCPKTSHI(_i) (0x00010C04 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */ -#define I40E_GLPES_PFIP4RXMCPKTSHI_MAX_INDEX 15 -#define I40E_GLPES_PFIP4RXMCPKTSHI_IP4RXMCPKTSHI_SHIFT 0 -#define I40E_GLPES_PFIP4RXMCPKTSHI_IP4RXMCPKTSHI_MASK (0xFFFF << I40E_GLPES_PFIP4RXMCPKTSHI_IP4RXMCPKTSHI_SHIFT) -#define I40E_GLPES_PFIP4RXMCPKTSLO(_i) (0x00010C00 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */ -#define I40E_GLPES_PFIP4RXMCPKTSLO_MAX_INDEX 15 -#define I40E_GLPES_PFIP4RXMCPKTSLO_IP4RXMCPKTSLO_SHIFT 0 -#define I40E_GLPES_PFIP4RXMCPKTSLO_IP4RXMCPKTSLO_MASK (0xFFFFFFFF << I40E_GLPES_PFIP4RXMCPKTSLO_IP4RXMCPKTSLO_SHIFT) -#define I40E_GLPES_PFIP4RXOCTSHI(_i) (0x00010204 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */ -#define I40E_GLPES_PFIP4RXOCTSHI_MAX_INDEX 15 -#define I40E_GLPES_PFIP4RXOCTSHI_IP4RXOCTSHI_SHIFT 0 -#define I40E_GLPES_PFIP4RXOCTSHI_IP4RXOCTSHI_MASK (0xFFFF << I40E_GLPES_PFIP4RXOCTSHI_IP4RXOCTSHI_SHIFT) -#define I40E_GLPES_PFIP4RXOCTSLO(_i) (0x00010200 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */ -#define I40E_GLPES_PFIP4RXOCTSLO_MAX_INDEX 15 -#define I40E_GLPES_PFIP4RXOCTSLO_IP4RXOCTSLO_SHIFT 0 -#define I40E_GLPES_PFIP4RXOCTSLO_IP4RXOCTSLO_MASK (0xFFFFFFFF << I40E_GLPES_PFIP4RXOCTSLO_IP4RXOCTSLO_SHIFT) -#define I40E_GLPES_PFIP4RXPKTSHI(_i) (0x00010404 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */ -#define I40E_GLPES_PFIP4RXPKTSHI_MAX_INDEX 15 -#define I40E_GLPES_PFIP4RXPKTSHI_IP4RXPKTSHI_SHIFT 0 -#define I40E_GLPES_PFIP4RXPKTSHI_IP4RXPKTSHI_MASK (0xFFFF << I40E_GLPES_PFIP4RXPKTSHI_IP4RXPKTSHI_SHIFT) -#define I40E_GLPES_PFIP4RXPKTSLO(_i) (0x00010400 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */ -#define I40E_GLPES_PFIP4RXPKTSLO_MAX_INDEX 15 -#define I40E_GLPES_PFIP4RXPKTSLO_IP4RXPKTSLO_SHIFT 0 -#define I40E_GLPES_PFIP4RXPKTSLO_IP4RXPKTSLO_MASK (0xFFFFFFFF << I40E_GLPES_PFIP4RXPKTSLO_IP4RXPKTSLO_SHIFT) -#define I40E_GLPES_PFIP4RXTRUNC(_i) (0x00010700 + ((_i) * 4)) /* _i=0...15 */ /* Reset: PE_CORER */ -#define I40E_GLPES_PFIP4RXTRUNC_MAX_INDEX 15 -#define I40E_GLPES_PFIP4RXTRUNC_IP4RXTRUNC_SHIFT 0 -#define I40E_GLPES_PFIP4RXTRUNC_IP4RXTRUNC_MASK (0xFFFFFFFF << I40E_GLPES_PFIP4RXTRUNC_IP4RXTRUNC_SHIFT) -#define I40E_GLPES_PFIP4TXFRAGSHI(_i) (0x00011E04 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */ -#define I40E_GLPES_PFIP4TXFRAGSHI_MAX_INDEX 15 -#define I40E_GLPES_PFIP4TXFRAGSHI_IP4TXFRAGSHI_SHIFT 0 -#define I40E_GLPES_PFIP4TXFRAGSHI_IP4TXFRAGSHI_MASK (0xFFFF << I40E_GLPES_PFIP4TXFRAGSHI_IP4TXFRAGSHI_SHIFT) -#define I40E_GLPES_PFIP4TXFRAGSLO(_i) (0x00011E00 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */ -#define I40E_GLPES_PFIP4TXFRAGSLO_MAX_INDEX 15 -#define I40E_GLPES_PFIP4TXFRAGSLO_IP4TXFRAGSLO_SHIFT 0 -#define I40E_GLPES_PFIP4TXFRAGSLO_IP4TXFRAGSLO_MASK (0xFFFFFFFF << I40E_GLPES_PFIP4TXFRAGSLO_IP4TXFRAGSLO_SHIFT) -#define I40E_GLPES_PFIP4TXMCOCTSHI(_i) (0x00012004 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */ -#define I40E_GLPES_PFIP4TXMCOCTSHI_MAX_INDEX 15 -#define I40E_GLPES_PFIP4TXMCOCTSHI_IP4TXMCOCTSHI_SHIFT 0 -#define I40E_GLPES_PFIP4TXMCOCTSHI_IP4TXMCOCTSHI_MASK (0xFFFF << I40E_GLPES_PFIP4TXMCOCTSHI_IP4TXMCOCTSHI_SHIFT) -#define I40E_GLPES_PFIP4TXMCOCTSLO(_i) (0x00012000 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */ -#define I40E_GLPES_PFIP4TXMCOCTSLO_MAX_INDEX 15 -#define I40E_GLPES_PFIP4TXMCOCTSLO_IP4TXMCOCTSLO_SHIFT 0 -#define I40E_GLPES_PFIP4TXMCOCTSLO_IP4TXMCOCTSLO_MASK (0xFFFFFFFF << I40E_GLPES_PFIP4TXMCOCTSLO_IP4TXMCOCTSLO_SHIFT) -#define I40E_GLPES_PFIP4TXMCPKTSHI(_i) (0x00012204 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */ -#define I40E_GLPES_PFIP4TXMCPKTSHI_MAX_INDEX 15 -#define I40E_GLPES_PFIP4TXMCPKTSHI_IP4TXMCPKTSHI_SHIFT 0 -#define I40E_GLPES_PFIP4TXMCPKTSHI_IP4TXMCPKTSHI_MASK (0xFFFF << I40E_GLPES_PFIP4TXMCPKTSHI_IP4TXMCPKTSHI_SHIFT) -#define I40E_GLPES_PFIP4TXMCPKTSLO(_i) (0x00012200 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */ -#define I40E_GLPES_PFIP4TXMCPKTSLO_MAX_INDEX 15 -#define I40E_GLPES_PFIP4TXMCPKTSLO_IP4TXMCPKTSLO_SHIFT 0 -#define I40E_GLPES_PFIP4TXMCPKTSLO_IP4TXMCPKTSLO_MASK (0xFFFFFFFF << I40E_GLPES_PFIP4TXMCPKTSLO_IP4TXMCPKTSLO_SHIFT) -#define I40E_GLPES_PFIP4TXNOROUTE(_i) (0x00012E00 + ((_i) * 4)) /* _i=0...15 */ /* Reset: PE_CORER */ -#define I40E_GLPES_PFIP4TXNOROUTE_MAX_INDEX 15 -#define I40E_GLPES_PFIP4TXNOROUTE_IP4TXNOROUTE_SHIFT 0 -#define I40E_GLPES_PFIP4TXNOROUTE_IP4TXNOROUTE_MASK (0xFFFFFF << I40E_GLPES_PFIP4TXNOROUTE_IP4TXNOROUTE_SHIFT) -#define I40E_GLPES_PFIP4TXOCTSHI(_i) (0x00011A04 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */ -#define I40E_GLPES_PFIP4TXOCTSHI_MAX_INDEX 15 -#define I40E_GLPES_PFIP4TXOCTSHI_IP4TXOCTSHI_SHIFT 0 -#define I40E_GLPES_PFIP4TXOCTSHI_IP4TXOCTSHI_MASK (0xFFFF << I40E_GLPES_PFIP4TXOCTSHI_IP4TXOCTSHI_SHIFT) -#define I40E_GLPES_PFIP4TXOCTSLO(_i) (0x00011A00 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */ -#define I40E_GLPES_PFIP4TXOCTSLO_MAX_INDEX 15 -#define I40E_GLPES_PFIP4TXOCTSLO_IP4TXOCTSLO_SHIFT 0 -#define I40E_GLPES_PFIP4TXOCTSLO_IP4TXOCTSLO_MASK (0xFFFFFFFF << I40E_GLPES_PFIP4TXOCTSLO_IP4TXOCTSLO_SHIFT) -#define I40E_GLPES_PFIP4TXPKTSHI(_i) (0x00011C04 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */ -#define I40E_GLPES_PFIP4TXPKTSHI_MAX_INDEX 15 -#define I40E_GLPES_PFIP4TXPKTSHI_IP4TXPKTSHI_SHIFT 0 -#define I40E_GLPES_PFIP4TXPKTSHI_IP4TXPKTSHI_MASK (0xFFFF << I40E_GLPES_PFIP4TXPKTSHI_IP4TXPKTSHI_SHIFT) -#define I40E_GLPES_PFIP4TXPKTSLO(_i) (0x00011C00 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */ -#define I40E_GLPES_PFIP4TXPKTSLO_MAX_INDEX 15 -#define I40E_GLPES_PFIP4TXPKTSLO_IP4TXPKTSLO_SHIFT 0 -#define I40E_GLPES_PFIP4TXPKTSLO_IP4TXPKTSLO_MASK (0xFFFFFFFF << I40E_GLPES_PFIP4TXPKTSLO_IP4TXPKTSLO_SHIFT) -#define I40E_GLPES_PFIP6RXDISCARD(_i) (0x00011200 + ((_i) * 4)) /* _i=0...15 */ /* Reset: PE_CORER */ -#define I40E_GLPES_PFIP6RXDISCARD_MAX_INDEX 15 -#define I40E_GLPES_PFIP6RXDISCARD_IP6RXDISCARD_SHIFT 0 -#define I40E_GLPES_PFIP6RXDISCARD_IP6RXDISCARD_MASK (0xFFFFFFFF << I40E_GLPES_PFIP6RXDISCARD_IP6RXDISCARD_SHIFT) -#define I40E_GLPES_PFIP6RXFRAGSHI(_i) (0x00011404 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */ -#define I40E_GLPES_PFIP6RXFRAGSHI_MAX_INDEX 15 -#define I40E_GLPES_PFIP6RXFRAGSHI_IP6RXFRAGSHI_SHIFT 0 -#define I40E_GLPES_PFIP6RXFRAGSHI_IP6RXFRAGSHI_MASK (0xFFFF << I40E_GLPES_PFIP6RXFRAGSHI_IP6RXFRAGSHI_SHIFT) -#define I40E_GLPES_PFIP6RXFRAGSLO(_i) (0x00011400 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */ -#define I40E_GLPES_PFIP6RXFRAGSLO_MAX_INDEX 15 -#define I40E_GLPES_PFIP6RXFRAGSLO_IP6RXFRAGSLO_SHIFT 0 -#define I40E_GLPES_PFIP6RXFRAGSLO_IP6RXFRAGSLO_MASK (0xFFFFFFFF << I40E_GLPES_PFIP6RXFRAGSLO_IP6RXFRAGSLO_SHIFT) -#define I40E_GLPES_PFIP6RXMCOCTSHI(_i) (0x00011604 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */ -#define I40E_GLPES_PFIP6RXMCOCTSHI_MAX_INDEX 15 -#define I40E_GLPES_PFIP6RXMCOCTSHI_IP6RXMCOCTSHI_SHIFT 0 -#define I40E_GLPES_PFIP6RXMCOCTSHI_IP6RXMCOCTSHI_MASK (0xFFFF << I40E_GLPES_PFIP6RXMCOCTSHI_IP6RXMCOCTSHI_SHIFT) -#define I40E_GLPES_PFIP6RXMCOCTSLO(_i) (0x00011600 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */ -#define I40E_GLPES_PFIP6RXMCOCTSLO_MAX_INDEX 15 -#define I40E_GLPES_PFIP6RXMCOCTSLO_IP6RXMCOCTSLO_SHIFT 0 -#define I40E_GLPES_PFIP6RXMCOCTSLO_IP6RXMCOCTSLO_MASK (0xFFFFFFFF << I40E_GLPES_PFIP6RXMCOCTSLO_IP6RXMCOCTSLO_SHIFT) -#define I40E_GLPES_PFIP6RXMCPKTSHI(_i) (0x00011804 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */ -#define I40E_GLPES_PFIP6RXMCPKTSHI_MAX_INDEX 15 -#define I40E_GLPES_PFIP6RXMCPKTSHI_IP6RXMCPKTSHI_SHIFT 0 -#define I40E_GLPES_PFIP6RXMCPKTSHI_IP6RXMCPKTSHI_MASK (0xFFFF << I40E_GLPES_PFIP6RXMCPKTSHI_IP6RXMCPKTSHI_SHIFT) -#define I40E_GLPES_PFIP6RXMCPKTSLO(_i) (0x00011800 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */ -#define I40E_GLPES_PFIP6RXMCPKTSLO_MAX_INDEX 15 -#define I40E_GLPES_PFIP6RXMCPKTSLO_IP6RXMCPKTSLO_SHIFT 0 -#define I40E_GLPES_PFIP6RXMCPKTSLO_IP6RXMCPKTSLO_MASK (0xFFFFFFFF << I40E_GLPES_PFIP6RXMCPKTSLO_IP6RXMCPKTSLO_SHIFT) -#define I40E_GLPES_PFIP6RXOCTSHI(_i) (0x00010E04 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */ -#define I40E_GLPES_PFIP6RXOCTSHI_MAX_INDEX 15 -#define I40E_GLPES_PFIP6RXOCTSHI_IP6RXOCTSHI_SHIFT 0 -#define I40E_GLPES_PFIP6RXOCTSHI_IP6RXOCTSHI_MASK (0xFFFF << I40E_GLPES_PFIP6RXOCTSHI_IP6RXOCTSHI_SHIFT) -#define I40E_GLPES_PFIP6RXOCTSLO(_i) (0x00010E00 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */ -#define I40E_GLPES_PFIP6RXOCTSLO_MAX_INDEX 15 -#define I40E_GLPES_PFIP6RXOCTSLO_IP6RXOCTSLO_SHIFT 0 -#define I40E_GLPES_PFIP6RXOCTSLO_IP6RXOCTSLO_MASK (0xFFFFFFFF << I40E_GLPES_PFIP6RXOCTSLO_IP6RXOCTSLO_SHIFT) -#define I40E_GLPES_PFIP6RXPKTSHI(_i) (0x00011004 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */ -#define I40E_GLPES_PFIP6RXPKTSHI_MAX_INDEX 15 -#define I40E_GLPES_PFIP6RXPKTSHI_IP6RXPKTSHI_SHIFT 0 -#define I40E_GLPES_PFIP6RXPKTSHI_IP6RXPKTSHI_MASK (0xFFFF << I40E_GLPES_PFIP6RXPKTSHI_IP6RXPKTSHI_SHIFT) -#define I40E_GLPES_PFIP6RXPKTSLO(_i) (0x00011000 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */ -#define I40E_GLPES_PFIP6RXPKTSLO_MAX_INDEX 15 -#define I40E_GLPES_PFIP6RXPKTSLO_IP6RXPKTSLO_SHIFT 0 -#define I40E_GLPES_PFIP6RXPKTSLO_IP6RXPKTSLO_MASK (0xFFFFFFFF << I40E_GLPES_PFIP6RXPKTSLO_IP6RXPKTSLO_SHIFT) -#define I40E_GLPES_PFIP6RXTRUNC(_i) (0x00011300 + ((_i) * 4)) /* _i=0...15 */ /* Reset: PE_CORER */ -#define I40E_GLPES_PFIP6RXTRUNC_MAX_INDEX 15 -#define I40E_GLPES_PFIP6RXTRUNC_IP6RXTRUNC_SHIFT 0 -#define I40E_GLPES_PFIP6RXTRUNC_IP6RXTRUNC_MASK (0xFFFFFFFF << I40E_GLPES_PFIP6RXTRUNC_IP6RXTRUNC_SHIFT) -#define I40E_GLPES_PFIP6TXFRAGSHI(_i) (0x00012804 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */ -#define I40E_GLPES_PFIP6TXFRAGSHI_MAX_INDEX 15 -#define I40E_GLPES_PFIP6TXFRAGSHI_IP6TXFRAGSHI_SHIFT 0 -#define I40E_GLPES_PFIP6TXFRAGSHI_IP6TXFRAGSHI_MASK (0xFFFF << I40E_GLPES_PFIP6TXFRAGSHI_IP6TXFRAGSHI_SHIFT) -#define I40E_GLPES_PFIP6TXFRAGSLO(_i) (0x00012800 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */ -#define I40E_GLPES_PFIP6TXFRAGSLO_MAX_INDEX 15 -#define I40E_GLPES_PFIP6TXFRAGSLO_IP6TXFRAGSLO_SHIFT 0 -#define I40E_GLPES_PFIP6TXFRAGSLO_IP6TXFRAGSLO_MASK (0xFFFFFFFF << I40E_GLPES_PFIP6TXFRAGSLO_IP6TXFRAGSLO_SHIFT) -#define I40E_GLPES_PFIP6TXMCOCTSHI(_i) (0x00012A04 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */ -#define I40E_GLPES_PFIP6TXMCOCTSHI_MAX_INDEX 15 -#define I40E_GLPES_PFIP6TXMCOCTSHI_IP6TXMCOCTSHI_SHIFT 0 -#define I40E_GLPES_PFIP6TXMCOCTSHI_IP6TXMCOCTSHI_MASK (0xFFFF << I40E_GLPES_PFIP6TXMCOCTSHI_IP6TXMCOCTSHI_SHIFT) -#define I40E_GLPES_PFIP6TXMCOCTSLO(_i) (0x00012A00 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */ -#define I40E_GLPES_PFIP6TXMCOCTSLO_MAX_INDEX 15 -#define I40E_GLPES_PFIP6TXMCOCTSLO_IP6TXMCOCTSLO_SHIFT 0 -#define I40E_GLPES_PFIP6TXMCOCTSLO_IP6TXMCOCTSLO_MASK (0xFFFFFFFF << I40E_GLPES_PFIP6TXMCOCTSLO_IP6TXMCOCTSLO_SHIFT) -#define I40E_GLPES_PFIP6TXMCPKTSHI(_i) (0x00012C04 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */ -#define I40E_GLPES_PFIP6TXMCPKTSHI_MAX_INDEX 15 -#define I40E_GLPES_PFIP6TXMCPKTSHI_IP6TXMCPKTSHI_SHIFT 0 -#define I40E_GLPES_PFIP6TXMCPKTSHI_IP6TXMCPKTSHI_MASK (0xFFFF << I40E_GLPES_PFIP6TXMCPKTSHI_IP6TXMCPKTSHI_SHIFT) -#define I40E_GLPES_PFIP6TXMCPKTSLO(_i) (0x00012C00 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */ -#define I40E_GLPES_PFIP6TXMCPKTSLO_MAX_INDEX 15 -#define I40E_GLPES_PFIP6TXMCPKTSLO_IP6TXMCPKTSLO_SHIFT 0 -#define I40E_GLPES_PFIP6TXMCPKTSLO_IP6TXMCPKTSLO_MASK (0xFFFFFFFF << I40E_GLPES_PFIP6TXMCPKTSLO_IP6TXMCPKTSLO_SHIFT) -#define I40E_GLPES_PFIP6TXNOROUTE(_i) (0x00012F00 + ((_i) * 4)) /* _i=0...15 */ /* Reset: PE_CORER */ -#define I40E_GLPES_PFIP6TXNOROUTE_MAX_INDEX 15 -#define I40E_GLPES_PFIP6TXNOROUTE_IP6TXNOROUTE_SHIFT 0 -#define I40E_GLPES_PFIP6TXNOROUTE_IP6TXNOROUTE_MASK (0xFFFFFF << I40E_GLPES_PFIP6TXNOROUTE_IP6TXNOROUTE_SHIFT) -#define I40E_GLPES_PFIP6TXOCTSHI(_i) (0x00012404 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */ -#define I40E_GLPES_PFIP6TXOCTSHI_MAX_INDEX 15 -#define I40E_GLPES_PFIP6TXOCTSHI_IP6TXOCTSHI_SHIFT 0 -#define I40E_GLPES_PFIP6TXOCTSHI_IP6TXOCTSHI_MASK (0xFFFF << I40E_GLPES_PFIP6TXOCTSHI_IP6TXOCTSHI_SHIFT) -#define I40E_GLPES_PFIP6TXOCTSLO(_i) (0x00012400 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */ -#define I40E_GLPES_PFIP6TXOCTSLO_MAX_INDEX 15 -#define I40E_GLPES_PFIP6TXOCTSLO_IP6TXOCTSLO_SHIFT 0 -#define I40E_GLPES_PFIP6TXOCTSLO_IP6TXOCTSLO_MASK (0xFFFFFFFF << I40E_GLPES_PFIP6TXOCTSLO_IP6TXOCTSLO_SHIFT) -#define I40E_GLPES_PFIP6TXPKTSHI(_i) (0x00012604 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */ -#define I40E_GLPES_PFIP6TXPKTSHI_MAX_INDEX 15 -#define I40E_GLPES_PFIP6TXPKTSHI_IP6TXPKTSHI_SHIFT 0 -#define I40E_GLPES_PFIP6TXPKTSHI_IP6TXPKTSHI_MASK (0xFFFF << I40E_GLPES_PFIP6TXPKTSHI_IP6TXPKTSHI_SHIFT) -#define I40E_GLPES_PFIP6TXPKTSLO(_i) (0x00012600 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */ -#define I40E_GLPES_PFIP6TXPKTSLO_MAX_INDEX 15 -#define I40E_GLPES_PFIP6TXPKTSLO_IP6TXPKTSLO_SHIFT 0 -#define I40E_GLPES_PFIP6TXPKTSLO_IP6TXPKTSLO_MASK (0xFFFFFFFF << I40E_GLPES_PFIP6TXPKTSLO_IP6TXPKTSLO_SHIFT) -#define I40E_GLPES_PFRDMARXRDSHI(_i) (0x00013E04 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */ -#define I40E_GLPES_PFRDMARXRDSHI_MAX_INDEX 15 -#define I40E_GLPES_PFRDMARXRDSHI_RDMARXRDSHI_SHIFT 0 -#define I40E_GLPES_PFRDMARXRDSHI_RDMARXRDSHI_MASK (0xFFFF << I40E_GLPES_PFRDMARXRDSHI_RDMARXRDSHI_SHIFT) -#define I40E_GLPES_PFRDMARXRDSLO(_i) (0x00013E00 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */ -#define I40E_GLPES_PFRDMARXRDSLO_MAX_INDEX 15 -#define I40E_GLPES_PFRDMARXRDSLO_RDMARXRDSLO_SHIFT 0 -#define I40E_GLPES_PFRDMARXRDSLO_RDMARXRDSLO_MASK (0xFFFFFFFF << I40E_GLPES_PFRDMARXRDSLO_RDMARXRDSLO_SHIFT) -#define I40E_GLPES_PFRDMARXSNDSHI(_i) (0x00014004 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */ -#define I40E_GLPES_PFRDMARXSNDSHI_MAX_INDEX 15 -#define I40E_GLPES_PFRDMARXSNDSHI_RDMARXSNDSHI_SHIFT 0 -#define I40E_GLPES_PFRDMARXSNDSHI_RDMARXSNDSHI_MASK (0xFFFF << I40E_GLPES_PFRDMARXSNDSHI_RDMARXSNDSHI_SHIFT) -#define I40E_GLPES_PFRDMARXSNDSLO(_i) (0x00014000 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */ -#define I40E_GLPES_PFRDMARXSNDSLO_MAX_INDEX 15 -#define I40E_GLPES_PFRDMARXSNDSLO_RDMARXSNDSLO_SHIFT 0 -#define I40E_GLPES_PFRDMARXSNDSLO_RDMARXSNDSLO_MASK (0xFFFFFFFF << I40E_GLPES_PFRDMARXSNDSLO_RDMARXSNDSLO_SHIFT) -#define I40E_GLPES_PFRDMARXWRSHI(_i) (0x00013C04 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */ -#define I40E_GLPES_PFRDMARXWRSHI_MAX_INDEX 15 -#define I40E_GLPES_PFRDMARXWRSHI_RDMARXWRSHI_SHIFT 0 -#define I40E_GLPES_PFRDMARXWRSHI_RDMARXWRSHI_MASK (0xFFFF << I40E_GLPES_PFRDMARXWRSHI_RDMARXWRSHI_SHIFT) -#define I40E_GLPES_PFRDMARXWRSLO(_i) (0x00013C00 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */ -#define I40E_GLPES_PFRDMARXWRSLO_MAX_INDEX 15 -#define I40E_GLPES_PFRDMARXWRSLO_RDMARXWRSLO_SHIFT 0 -#define I40E_GLPES_PFRDMARXWRSLO_RDMARXWRSLO_MASK (0xFFFFFFFF << I40E_GLPES_PFRDMARXWRSLO_RDMARXWRSLO_SHIFT) -#define I40E_GLPES_PFRDMATXRDSHI(_i) (0x00014404 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */ -#define I40E_GLPES_PFRDMATXRDSHI_MAX_INDEX 15 -#define I40E_GLPES_PFRDMATXRDSHI_RDMARXRDSHI_SHIFT 0 -#define I40E_GLPES_PFRDMATXRDSHI_RDMARXRDSHI_MASK (0xFFFF << I40E_GLPES_PFRDMATXRDSHI_RDMARXRDSHI_SHIFT) -#define I40E_GLPES_PFRDMATXRDSLO(_i) (0x00014400 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */ -#define I40E_GLPES_PFRDMATXRDSLO_MAX_INDEX 15 -#define I40E_GLPES_PFRDMATXRDSLO_RDMARXRDSLO_SHIFT 0 -#define I40E_GLPES_PFRDMATXRDSLO_RDMARXRDSLO_MASK (0xFFFFFFFF << I40E_GLPES_PFRDMATXRDSLO_RDMARXRDSLO_SHIFT) -#define I40E_GLPES_PFRDMATXSNDSHI(_i) (0x00014604 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */ -#define I40E_GLPES_PFRDMATXSNDSHI_MAX_INDEX 15 -#define I40E_GLPES_PFRDMATXSNDSHI_RDMARXSNDSHI_SHIFT 0 -#define I40E_GLPES_PFRDMATXSNDSHI_RDMARXSNDSHI_MASK (0xFFFF << I40E_GLPES_PFRDMATXSNDSHI_RDMARXSNDSHI_SHIFT) -#define I40E_GLPES_PFRDMATXSNDSLO(_i) (0x00014600 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */ -#define I40E_GLPES_PFRDMATXSNDSLO_MAX_INDEX 15 -#define I40E_GLPES_PFRDMATXSNDSLO_RDMARXSNDSLO_SHIFT 0 -#define I40E_GLPES_PFRDMATXSNDSLO_RDMARXSNDSLO_MASK (0xFFFFFFFF << I40E_GLPES_PFRDMATXSNDSLO_RDMARXSNDSLO_SHIFT) -#define I40E_GLPES_PFRDMATXWRSHI(_i) (0x00014204 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */ -#define I40E_GLPES_PFRDMATXWRSHI_MAX_INDEX 15 -#define I40E_GLPES_PFRDMATXWRSHI_RDMARXWRSHI_SHIFT 0 -#define I40E_GLPES_PFRDMATXWRSHI_RDMARXWRSHI_MASK (0xFFFF << I40E_GLPES_PFRDMATXWRSHI_RDMARXWRSHI_SHIFT) -#define I40E_GLPES_PFRDMATXWRSLO(_i) (0x00014200 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */ -#define I40E_GLPES_PFRDMATXWRSLO_MAX_INDEX 15 -#define I40E_GLPES_PFRDMATXWRSLO_RDMARXWRSLO_SHIFT 0 -#define I40E_GLPES_PFRDMATXWRSLO_RDMARXWRSLO_MASK (0xFFFFFFFF << I40E_GLPES_PFRDMATXWRSLO_RDMARXWRSLO_SHIFT) -#define I40E_GLPES_PFRDMAVBNDHI(_i) (0x00014804 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */ -#define I40E_GLPES_PFRDMAVBNDHI_MAX_INDEX 15 -#define I40E_GLPES_PFRDMAVBNDHI_RDMAVBNDHI_SHIFT 0 -#define I40E_GLPES_PFRDMAVBNDHI_RDMAVBNDHI_MASK (0xFFFFFFFF << I40E_GLPES_PFRDMAVBNDHI_RDMAVBNDHI_SHIFT) -#define I40E_GLPES_PFRDMAVBNDLO(_i) (0x00014800 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */ -#define I40E_GLPES_PFRDMAVBNDLO_MAX_INDEX 15 -#define I40E_GLPES_PFRDMAVBNDLO_RDMAVBNDLO_SHIFT 0 -#define I40E_GLPES_PFRDMAVBNDLO_RDMAVBNDLO_MASK (0xFFFFFFFF << I40E_GLPES_PFRDMAVBNDLO_RDMAVBNDLO_SHIFT) -#define I40E_GLPES_PFRDMAVINVHI(_i) (0x00014A04 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */ -#define I40E_GLPES_PFRDMAVINVHI_MAX_INDEX 15 -#define I40E_GLPES_PFRDMAVINVHI_RDMAVINVHI_SHIFT 0 -#define I40E_GLPES_PFRDMAVINVHI_RDMAVINVHI_MASK (0xFFFFFFFF << I40E_GLPES_PFRDMAVINVHI_RDMAVINVHI_SHIFT) -#define I40E_GLPES_PFRDMAVINVLO(_i) (0x00014A00 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */ -#define I40E_GLPES_PFRDMAVINVLO_MAX_INDEX 15 -#define I40E_GLPES_PFRDMAVINVLO_RDMAVINVLO_SHIFT 0 -#define I40E_GLPES_PFRDMAVINVLO_RDMAVINVLO_MASK (0xFFFFFFFF << I40E_GLPES_PFRDMAVINVLO_RDMAVINVLO_SHIFT) -#define I40E_GLPES_PFRXVLANERR(_i) (0x00010000 + ((_i) * 4)) /* _i=0...15 */ /* Reset: PE_CORER */ -#define I40E_GLPES_PFRXVLANERR_MAX_INDEX 15 -#define I40E_GLPES_PFRXVLANERR_RXVLANERR_SHIFT 0 -#define I40E_GLPES_PFRXVLANERR_RXVLANERR_MASK (0xFFFFFF << I40E_GLPES_PFRXVLANERR_RXVLANERR_SHIFT) -#define I40E_GLPES_PFTCPRTXSEG(_i) (0x00013600 + ((_i) * 4)) /* _i=0...15 */ /* Reset: PE_CORER */ -#define I40E_GLPES_PFTCPRTXSEG_MAX_INDEX 15 -#define I40E_GLPES_PFTCPRTXSEG_TCPRTXSEG_SHIFT 0 -#define I40E_GLPES_PFTCPRTXSEG_TCPRTXSEG_MASK (0xFFFFFFFF << I40E_GLPES_PFTCPRTXSEG_TCPRTXSEG_SHIFT) -#define I40E_GLPES_PFTCPRXOPTERR(_i) (0x00013200 + ((_i) * 4)) /* _i=0...15 */ /* Reset: PE_CORER */ -#define I40E_GLPES_PFTCPRXOPTERR_MAX_INDEX 15 -#define I40E_GLPES_PFTCPRXOPTERR_TCPRXOPTERR_SHIFT 0 -#define I40E_GLPES_PFTCPRXOPTERR_TCPRXOPTERR_MASK (0xFFFFFF << I40E_GLPES_PFTCPRXOPTERR_TCPRXOPTERR_SHIFT) -#define I40E_GLPES_PFTCPRXPROTOERR(_i) (0x00013300 + ((_i) * 4)) /* _i=0...15 */ /* Reset: PE_CORER */ -#define I40E_GLPES_PFTCPRXPROTOERR_MAX_INDEX 15 -#define I40E_GLPES_PFTCPRXPROTOERR_TCPRXPROTOERR_SHIFT 0 -#define I40E_GLPES_PFTCPRXPROTOERR_TCPRXPROTOERR_MASK (0xFFFFFF << I40E_GLPES_PFTCPRXPROTOERR_TCPRXPROTOERR_SHIFT) -#define I40E_GLPES_PFTCPRXSEGSHI(_i) (0x00013004 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */ -#define I40E_GLPES_PFTCPRXSEGSHI_MAX_INDEX 15 -#define I40E_GLPES_PFTCPRXSEGSHI_TCPRXSEGSHI_SHIFT 0 -#define I40E_GLPES_PFTCPRXSEGSHI_TCPRXSEGSHI_MASK (0xFFFF << I40E_GLPES_PFTCPRXSEGSHI_TCPRXSEGSHI_SHIFT) -#define I40E_GLPES_PFTCPRXSEGSLO(_i) (0x00013000 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */ -#define I40E_GLPES_PFTCPRXSEGSLO_MAX_INDEX 15 -#define I40E_GLPES_PFTCPRXSEGSLO_TCPRXSEGSLO_SHIFT 0 -#define I40E_GLPES_PFTCPRXSEGSLO_TCPRXSEGSLO_MASK (0xFFFFFFFF << I40E_GLPES_PFTCPRXSEGSLO_TCPRXSEGSLO_SHIFT) -#define I40E_GLPES_PFTCPTXSEGHI(_i) (0x00013404 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */ -#define I40E_GLPES_PFTCPTXSEGHI_MAX_INDEX 15 -#define I40E_GLPES_PFTCPTXSEGHI_TCPTXSEGHI_SHIFT 0 -#define I40E_GLPES_PFTCPTXSEGHI_TCPTXSEGHI_MASK (0xFFFF << I40E_GLPES_PFTCPTXSEGHI_TCPTXSEGHI_SHIFT) -#define I40E_GLPES_PFTCPTXSEGLO(_i) (0x00013400 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */ -#define I40E_GLPES_PFTCPTXSEGLO_MAX_INDEX 15 -#define I40E_GLPES_PFTCPTXSEGLO_TCPTXSEGLO_SHIFT 0 -#define I40E_GLPES_PFTCPTXSEGLO_TCPTXSEGLO_MASK (0xFFFFFFFF << I40E_GLPES_PFTCPTXSEGLO_TCPTXSEGLO_SHIFT) -#define I40E_GLPES_PFUDPRXPKTSHI(_i) (0x00013804 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */ -#define I40E_GLPES_PFUDPRXPKTSHI_MAX_INDEX 15 -#define I40E_GLPES_PFUDPRXPKTSHI_UDPRXPKTSHI_SHIFT 0 -#define I40E_GLPES_PFUDPRXPKTSHI_UDPRXPKTSHI_MASK (0xFFFF << I40E_GLPES_PFUDPRXPKTSHI_UDPRXPKTSHI_SHIFT) -#define I40E_GLPES_PFUDPRXPKTSLO(_i) (0x00013800 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */ -#define I40E_GLPES_PFUDPRXPKTSLO_MAX_INDEX 15 -#define I40E_GLPES_PFUDPRXPKTSLO_UDPRXPKTSLO_SHIFT 0 -#define I40E_GLPES_PFUDPRXPKTSLO_UDPRXPKTSLO_MASK (0xFFFFFFFF << I40E_GLPES_PFUDPRXPKTSLO_UDPRXPKTSLO_SHIFT) -#define I40E_GLPES_PFUDPTXPKTSHI(_i) (0x00013A04 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */ -#define I40E_GLPES_PFUDPTXPKTSHI_MAX_INDEX 15 -#define I40E_GLPES_PFUDPTXPKTSHI_UDPTXPKTSHI_SHIFT 0 -#define I40E_GLPES_PFUDPTXPKTSHI_UDPTXPKTSHI_MASK (0xFFFF << I40E_GLPES_PFUDPTXPKTSHI_UDPTXPKTSHI_SHIFT) -#define I40E_GLPES_PFUDPTXPKTSLO(_i) (0x00013A00 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */ -#define I40E_GLPES_PFUDPTXPKTSLO_MAX_INDEX 15 -#define I40E_GLPES_PFUDPTXPKTSLO_UDPTXPKTSLO_SHIFT 0 -#define I40E_GLPES_PFUDPTXPKTSLO_UDPTXPKTSLO_MASK (0xFFFFFFFF << I40E_GLPES_PFUDPTXPKTSLO_UDPTXPKTSLO_SHIFT) -#define I40E_GLPES_RDMARXMULTFPDUSHI 0x0001E014 /* Reset: PE_CORER */ -#define I40E_GLPES_RDMARXMULTFPDUSHI_RDMARXMULTFPDUSHI_SHIFT 0 -#define I40E_GLPES_RDMARXMULTFPDUSHI_RDMARXMULTFPDUSHI_MASK (0xFFFFFF << I40E_GLPES_RDMARXMULTFPDUSHI_RDMARXMULTFPDUSHI_SHIFT) -#define I40E_GLPES_RDMARXMULTFPDUSLO 0x0001E010 /* Reset: PE_CORER */ -#define I40E_GLPES_RDMARXMULTFPDUSLO_RDMARXMULTFPDUSLO_SHIFT 0 -#define I40E_GLPES_RDMARXMULTFPDUSLO_RDMARXMULTFPDUSLO_MASK (0xFFFFFFFF << I40E_GLPES_RDMARXMULTFPDUSLO_RDMARXMULTFPDUSLO_SHIFT) -#define I40E_GLPES_RDMARXOOODDPHI 0x0001E01C /* Reset: PE_CORER */ -#define I40E_GLPES_RDMARXOOODDPHI_RDMARXOOODDPHI_SHIFT 0 -#define I40E_GLPES_RDMARXOOODDPHI_RDMARXOOODDPHI_MASK (0xFFFFFF << I40E_GLPES_RDMARXOOODDPHI_RDMARXOOODDPHI_SHIFT) -#define I40E_GLPES_RDMARXOOODDPLO 0x0001E018 /* Reset: PE_CORER */ -#define I40E_GLPES_RDMARXOOODDPLO_RDMARXOOODDPLO_SHIFT 0 -#define I40E_GLPES_RDMARXOOODDPLO_RDMARXOOODDPLO_MASK (0xFFFFFFFF << I40E_GLPES_RDMARXOOODDPLO_RDMARXOOODDPLO_SHIFT) -#define I40E_GLPES_RDMARXOOONOMARK 0x0001E004 /* Reset: PE_CORER */ -#define I40E_GLPES_RDMARXOOONOMARK_RDMAOOONOMARK_SHIFT 0 -#define I40E_GLPES_RDMARXOOONOMARK_RDMAOOONOMARK_MASK (0xFFFFFFFF << I40E_GLPES_RDMARXOOONOMARK_RDMAOOONOMARK_SHIFT) -#define I40E_GLPES_RDMARXUNALIGN 0x0001E000 /* Reset: PE_CORER */ -#define I40E_GLPES_RDMARXUNALIGN_RDMRXAUNALIGN_SHIFT 0 -#define I40E_GLPES_RDMARXUNALIGN_RDMRXAUNALIGN_MASK (0xFFFFFFFF << I40E_GLPES_RDMARXUNALIGN_RDMRXAUNALIGN_SHIFT) -#define I40E_GLPES_TCPRXFOURHOLEHI 0x0001E044 /* Reset: PE_CORER */ -#define I40E_GLPES_TCPRXFOURHOLEHI_TCPRXFOURHOLEHI_SHIFT 0 -#define I40E_GLPES_TCPRXFOURHOLEHI_TCPRXFOURHOLEHI_MASK (0xFFFFFF << I40E_GLPES_TCPRXFOURHOLEHI_TCPRXFOURHOLEHI_SHIFT) -#define I40E_GLPES_TCPRXFOURHOLELO 0x0001E040 /* Reset: PE_CORER */ -#define I40E_GLPES_TCPRXFOURHOLELO_TCPRXFOURHOLELO_SHIFT 0 -#define I40E_GLPES_TCPRXFOURHOLELO_TCPRXFOURHOLELO_MASK (0xFFFFFFFF << I40E_GLPES_TCPRXFOURHOLELO_TCPRXFOURHOLELO_SHIFT) -#define I40E_GLPES_TCPRXONEHOLEHI 0x0001E02C /* Reset: PE_CORER */ -#define I40E_GLPES_TCPRXONEHOLEHI_TCPRXONEHOLEHI_SHIFT 0 -#define I40E_GLPES_TCPRXONEHOLEHI_TCPRXONEHOLEHI_MASK (0xFFFFFF << I40E_GLPES_TCPRXONEHOLEHI_TCPRXONEHOLEHI_SHIFT) -#define I40E_GLPES_TCPRXONEHOLELO 0x0001E028 /* Reset: PE_CORER */ -#define I40E_GLPES_TCPRXONEHOLELO_TCPRXONEHOLELO_SHIFT 0 -#define I40E_GLPES_TCPRXONEHOLELO_TCPRXONEHOLELO_MASK (0xFFFFFFFF << I40E_GLPES_TCPRXONEHOLELO_TCPRXONEHOLELO_SHIFT) -#define I40E_GLPES_TCPRXPUREACKHI 0x0001E024 /* Reset: PE_CORER */ -#define I40E_GLPES_TCPRXPUREACKHI_TCPRXPUREACKSHI_SHIFT 0 -#define I40E_GLPES_TCPRXPUREACKHI_TCPRXPUREACKSHI_MASK (0xFFFFFF << I40E_GLPES_TCPRXPUREACKHI_TCPRXPUREACKSHI_SHIFT) -#define I40E_GLPES_TCPRXPUREACKSLO 0x0001E020 /* Reset: PE_CORER */ -#define I40E_GLPES_TCPRXPUREACKSLO_TCPRXPUREACKLO_SHIFT 0 -#define I40E_GLPES_TCPRXPUREACKSLO_TCPRXPUREACKLO_MASK (0xFFFFFFFF << I40E_GLPES_TCPRXPUREACKSLO_TCPRXPUREACKLO_SHIFT) -#define I40E_GLPES_TCPRXTHREEHOLEHI 0x0001E03C /* Reset: PE_CORER */ -#define I40E_GLPES_TCPRXTHREEHOLEHI_TCPRXTHREEHOLEHI_SHIFT 0 -#define I40E_GLPES_TCPRXTHREEHOLEHI_TCPRXTHREEHOLEHI_MASK (0xFFFFFF << I40E_GLPES_TCPRXTHREEHOLEHI_TCPRXTHREEHOLEHI_SHIFT) -#define I40E_GLPES_TCPRXTHREEHOLELO 0x0001E038 /* Reset: PE_CORER */ -#define I40E_GLPES_TCPRXTHREEHOLELO_TCPRXTHREEHOLELO_SHIFT 0 -#define I40E_GLPES_TCPRXTHREEHOLELO_TCPRXTHREEHOLELO_MASK (0xFFFFFFFF << I40E_GLPES_TCPRXTHREEHOLELO_TCPRXTHREEHOLELO_SHIFT) -#define I40E_GLPES_TCPRXTWOHOLEHI 0x0001E034 /* Reset: PE_CORER */ -#define I40E_GLPES_TCPRXTWOHOLEHI_TCPRXTWOHOLEHI_SHIFT 0 -#define I40E_GLPES_TCPRXTWOHOLEHI_TCPRXTWOHOLEHI_MASK (0xFFFFFF << I40E_GLPES_TCPRXTWOHOLEHI_TCPRXTWOHOLEHI_SHIFT) -#define I40E_GLPES_TCPRXTWOHOLELO 0x0001E030 /* Reset: PE_CORER */ -#define I40E_GLPES_TCPRXTWOHOLELO_TCPRXTWOHOLELO_SHIFT 0 -#define I40E_GLPES_TCPRXTWOHOLELO_TCPRXTWOHOLELO_MASK (0xFFFFFFFF << I40E_GLPES_TCPRXTWOHOLELO_TCPRXTWOHOLELO_SHIFT) -#define I40E_GLPES_TCPTXRETRANSFASTHI 0x0001E04C /* Reset: PE_CORER */ -#define I40E_GLPES_TCPTXRETRANSFASTHI_TCPTXRETRANSFASTHI_SHIFT 0 -#define I40E_GLPES_TCPTXRETRANSFASTHI_TCPTXRETRANSFASTHI_MASK (0xFFFFFF << I40E_GLPES_TCPTXRETRANSFASTHI_TCPTXRETRANSFASTHI_SHIFT) -#define I40E_GLPES_TCPTXRETRANSFASTLO 0x0001E048 /* Reset: PE_CORER */ -#define I40E_GLPES_TCPTXRETRANSFASTLO_TCPTXRETRANSFASTLO_SHIFT 0 -#define I40E_GLPES_TCPTXRETRANSFASTLO_TCPTXRETRANSFASTLO_MASK (0xFFFFFFFF << I40E_GLPES_TCPTXRETRANSFASTLO_TCPTXRETRANSFASTLO_SHIFT) -#define I40E_GLPES_TCPTXTOUTSFASTHI 0x0001E054 /* Reset: PE_CORER */ -#define I40E_GLPES_TCPTXTOUTSFASTHI_TCPTXTOUTSFASTHI_SHIFT 0 -#define I40E_GLPES_TCPTXTOUTSFASTHI_TCPTXTOUTSFASTHI_MASK (0xFFFFFF << I40E_GLPES_TCPTXTOUTSFASTHI_TCPTXTOUTSFASTHI_SHIFT) -#define I40E_GLPES_TCPTXTOUTSFASTLO 0x0001E050 /* Reset: PE_CORER */ -#define I40E_GLPES_TCPTXTOUTSFASTLO_TCPTXTOUTSFASTLO_SHIFT 0 -#define I40E_GLPES_TCPTXTOUTSFASTLO_TCPTXTOUTSFASTLO_MASK (0xFFFFFFFF << I40E_GLPES_TCPTXTOUTSFASTLO_TCPTXTOUTSFASTLO_SHIFT) -#define I40E_GLPES_TCPTXTOUTSHI 0x0001E05C /* Reset: PE_CORER */ -#define I40E_GLPES_TCPTXTOUTSHI_TCPTXTOUTSHI_SHIFT 0 -#define I40E_GLPES_TCPTXTOUTSHI_TCPTXTOUTSHI_MASK (0xFFFFFF << I40E_GLPES_TCPTXTOUTSHI_TCPTXTOUTSHI_SHIFT) -#define I40E_GLPES_TCPTXTOUTSLO 0x0001E058 /* Reset: PE_CORER */ -#define I40E_GLPES_TCPTXTOUTSLO_TCPTXTOUTSLO_SHIFT 0 -#define I40E_GLPES_TCPTXTOUTSLO_TCPTXTOUTSLO_MASK (0xFFFFFFFF << I40E_GLPES_TCPTXTOUTSLO_TCPTXTOUTSLO_SHIFT) -#define I40E_GLPES_VFIP4RXDISCARD(_i) (0x00018600 + ((_i) * 4)) /* _i=0...31 */ /* Reset: PE_CORER */ -#define I40E_GLPES_VFIP4RXDISCARD_MAX_INDEX 31 -#define I40E_GLPES_VFIP4RXDISCARD_IP4RXDISCARD_SHIFT 0 -#define I40E_GLPES_VFIP4RXDISCARD_IP4RXDISCARD_MASK (0xFFFFFFFF << I40E_GLPES_VFIP4RXDISCARD_IP4RXDISCARD_SHIFT) -#define I40E_GLPES_VFIP4RXFRAGSHI(_i) (0x00018804 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */ -#define I40E_GLPES_VFIP4RXFRAGSHI_MAX_INDEX 31 -#define I40E_GLPES_VFIP4RXFRAGSHI_IP4RXFRAGSHI_SHIFT 0 -#define I40E_GLPES_VFIP4RXFRAGSHI_IP4RXFRAGSHI_MASK (0xFFFF << I40E_GLPES_VFIP4RXFRAGSHI_IP4RXFRAGSHI_SHIFT) -#define I40E_GLPES_VFIP4RXFRAGSLO(_i) (0x00018800 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */ -#define I40E_GLPES_VFIP4RXFRAGSLO_MAX_INDEX 31 -#define I40E_GLPES_VFIP4RXFRAGSLO_IP4RXFRAGSLO_SHIFT 0 -#define I40E_GLPES_VFIP4RXFRAGSLO_IP4RXFRAGSLO_MASK (0xFFFFFFFF << I40E_GLPES_VFIP4RXFRAGSLO_IP4RXFRAGSLO_SHIFT) -#define I40E_GLPES_VFIP4RXMCOCTSHI(_i) (0x00018A04 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */ -#define I40E_GLPES_VFIP4RXMCOCTSHI_MAX_INDEX 31 -#define I40E_GLPES_VFIP4RXMCOCTSHI_IP4RXMCOCTSHI_SHIFT 0 -#define I40E_GLPES_VFIP4RXMCOCTSHI_IP4RXMCOCTSHI_MASK (0xFFFF << I40E_GLPES_VFIP4RXMCOCTSHI_IP4RXMCOCTSHI_SHIFT) -#define I40E_GLPES_VFIP4RXMCOCTSLO(_i) (0x00018A00 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */ -#define I40E_GLPES_VFIP4RXMCOCTSLO_MAX_INDEX 31 -#define I40E_GLPES_VFIP4RXMCOCTSLO_IP4RXMCOCTSLO_SHIFT 0 -#define I40E_GLPES_VFIP4RXMCOCTSLO_IP4RXMCOCTSLO_MASK (0xFFFFFFFF << I40E_GLPES_VFIP4RXMCOCTSLO_IP4RXMCOCTSLO_SHIFT) -#define I40E_GLPES_VFIP4RXMCPKTSHI(_i) (0x00018C04 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */ -#define I40E_GLPES_VFIP4RXMCPKTSHI_MAX_INDEX 31 -#define I40E_GLPES_VFIP4RXMCPKTSHI_IP4RXMCPKTSHI_SHIFT 0 -#define I40E_GLPES_VFIP4RXMCPKTSHI_IP4RXMCPKTSHI_MASK (0xFFFF << I40E_GLPES_VFIP4RXMCPKTSHI_IP4RXMCPKTSHI_SHIFT) -#define I40E_GLPES_VFIP4RXMCPKTSLO(_i) (0x00018C00 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */ -#define I40E_GLPES_VFIP4RXMCPKTSLO_MAX_INDEX 31 -#define I40E_GLPES_VFIP4RXMCPKTSLO_IP4RXMCPKTSLO_SHIFT 0 -#define I40E_GLPES_VFIP4RXMCPKTSLO_IP4RXMCPKTSLO_MASK (0xFFFFFFFF << I40E_GLPES_VFIP4RXMCPKTSLO_IP4RXMCPKTSLO_SHIFT) -#define I40E_GLPES_VFIP4RXOCTSHI(_i) (0x00018204 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */ -#define I40E_GLPES_VFIP4RXOCTSHI_MAX_INDEX 31 -#define I40E_GLPES_VFIP4RXOCTSHI_IP4RXOCTSHI_SHIFT 0 -#define I40E_GLPES_VFIP4RXOCTSHI_IP4RXOCTSHI_MASK (0xFFFF << I40E_GLPES_VFIP4RXOCTSHI_IP4RXOCTSHI_SHIFT) -#define I40E_GLPES_VFIP4RXOCTSLO(_i) (0x00018200 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */ -#define I40E_GLPES_VFIP4RXOCTSLO_MAX_INDEX 31 -#define I40E_GLPES_VFIP4RXOCTSLO_IP4RXOCTSLO_SHIFT 0 -#define I40E_GLPES_VFIP4RXOCTSLO_IP4RXOCTSLO_MASK (0xFFFFFFFF << I40E_GLPES_VFIP4RXOCTSLO_IP4RXOCTSLO_SHIFT) -#define I40E_GLPES_VFIP4RXPKTSHI(_i) (0x00018404 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */ -#define I40E_GLPES_VFIP4RXPKTSHI_MAX_INDEX 31 -#define I40E_GLPES_VFIP4RXPKTSHI_IP4RXPKTSHI_SHIFT 0 -#define I40E_GLPES_VFIP4RXPKTSHI_IP4RXPKTSHI_MASK (0xFFFF << I40E_GLPES_VFIP4RXPKTSHI_IP4RXPKTSHI_SHIFT) -#define I40E_GLPES_VFIP4RXPKTSLO(_i) (0x00018400 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */ -#define I40E_GLPES_VFIP4RXPKTSLO_MAX_INDEX 31 -#define I40E_GLPES_VFIP4RXPKTSLO_IP4RXPKTSLO_SHIFT 0 -#define I40E_GLPES_VFIP4RXPKTSLO_IP4RXPKTSLO_MASK (0xFFFFFFFF << I40E_GLPES_VFIP4RXPKTSLO_IP4RXPKTSLO_SHIFT) -#define I40E_GLPES_VFIP4RXTRUNC(_i) (0x00018700 + ((_i) * 4)) /* _i=0...31 */ /* Reset: PE_CORER */ -#define I40E_GLPES_VFIP4RXTRUNC_MAX_INDEX 31 -#define I40E_GLPES_VFIP4RXTRUNC_IP4RXTRUNC_SHIFT 0 -#define I40E_GLPES_VFIP4RXTRUNC_IP4RXTRUNC_MASK (0xFFFFFFFF << I40E_GLPES_VFIP4RXTRUNC_IP4RXTRUNC_SHIFT) -#define I40E_GLPES_VFIP4TXFRAGSHI(_i) (0x00019E04 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */ -#define I40E_GLPES_VFIP4TXFRAGSHI_MAX_INDEX 31 -#define I40E_GLPES_VFIP4TXFRAGSHI_IP4TXFRAGSHI_SHIFT 0 -#define I40E_GLPES_VFIP4TXFRAGSHI_IP4TXFRAGSHI_MASK (0xFFFF << I40E_GLPES_VFIP4TXFRAGSHI_IP4TXFRAGSHI_SHIFT) -#define I40E_GLPES_VFIP4TXFRAGSLO(_i) (0x00019E00 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */ -#define I40E_GLPES_VFIP4TXFRAGSLO_MAX_INDEX 31 -#define I40E_GLPES_VFIP4TXFRAGSLO_IP4TXFRAGSLO_SHIFT 0 -#define I40E_GLPES_VFIP4TXFRAGSLO_IP4TXFRAGSLO_MASK (0xFFFFFFFF << I40E_GLPES_VFIP4TXFRAGSLO_IP4TXFRAGSLO_SHIFT) -#define I40E_GLPES_VFIP4TXMCOCTSHI(_i) (0x0001A004 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */ -#define I40E_GLPES_VFIP4TXMCOCTSHI_MAX_INDEX 31 -#define I40E_GLPES_VFIP4TXMCOCTSHI_IP4TXMCOCTSHI_SHIFT 0 -#define I40E_GLPES_VFIP4TXMCOCTSHI_IP4TXMCOCTSHI_MASK (0xFFFF << I40E_GLPES_VFIP4TXMCOCTSHI_IP4TXMCOCTSHI_SHIFT) -#define I40E_GLPES_VFIP4TXMCOCTSLO(_i) (0x0001A000 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */ -#define I40E_GLPES_VFIP4TXMCOCTSLO_MAX_INDEX 31 -#define I40E_GLPES_VFIP4TXMCOCTSLO_IP4TXMCOCTSLO_SHIFT 0 -#define I40E_GLPES_VFIP4TXMCOCTSLO_IP4TXMCOCTSLO_MASK (0xFFFFFFFF << I40E_GLPES_VFIP4TXMCOCTSLO_IP4TXMCOCTSLO_SHIFT) -#define I40E_GLPES_VFIP4TXMCPKTSHI(_i) (0x0001A204 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */ -#define I40E_GLPES_VFIP4TXMCPKTSHI_MAX_INDEX 31 -#define I40E_GLPES_VFIP4TXMCPKTSHI_IP4TXMCPKTSHI_SHIFT 0 -#define I40E_GLPES_VFIP4TXMCPKTSHI_IP4TXMCPKTSHI_MASK (0xFFFF << I40E_GLPES_VFIP4TXMCPKTSHI_IP4TXMCPKTSHI_SHIFT) -#define I40E_GLPES_VFIP4TXMCPKTSLO(_i) (0x0001A200 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */ -#define I40E_GLPES_VFIP4TXMCPKTSLO_MAX_INDEX 31 -#define I40E_GLPES_VFIP4TXMCPKTSLO_IP4TXMCPKTSLO_SHIFT 0 -#define I40E_GLPES_VFIP4TXMCPKTSLO_IP4TXMCPKTSLO_MASK (0xFFFFFFFF << I40E_GLPES_VFIP4TXMCPKTSLO_IP4TXMCPKTSLO_SHIFT) -#define I40E_GLPES_VFIP4TXNOROUTE(_i) (0x0001AE00 + ((_i) * 4)) /* _i=0...31 */ /* Reset: PE_CORER */ -#define I40E_GLPES_VFIP4TXNOROUTE_MAX_INDEX 31 -#define I40E_GLPES_VFIP4TXNOROUTE_IP4TXNOROUTE_SHIFT 0 -#define I40E_GLPES_VFIP4TXNOROUTE_IP4TXNOROUTE_MASK (0xFFFFFF << I40E_GLPES_VFIP4TXNOROUTE_IP4TXNOROUTE_SHIFT) -#define I40E_GLPES_VFIP4TXOCTSHI(_i) (0x00019A04 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */ -#define I40E_GLPES_VFIP4TXOCTSHI_MAX_INDEX 31 -#define I40E_GLPES_VFIP4TXOCTSHI_IP4TXOCTSHI_SHIFT 0 -#define I40E_GLPES_VFIP4TXOCTSHI_IP4TXOCTSHI_MASK (0xFFFF << I40E_GLPES_VFIP4TXOCTSHI_IP4TXOCTSHI_SHIFT) -#define I40E_GLPES_VFIP4TXOCTSLO(_i) (0x00019A00 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */ -#define I40E_GLPES_VFIP4TXOCTSLO_MAX_INDEX 31 -#define I40E_GLPES_VFIP4TXOCTSLO_IP4TXOCTSLO_SHIFT 0 -#define I40E_GLPES_VFIP4TXOCTSLO_IP4TXOCTSLO_MASK (0xFFFFFFFF << I40E_GLPES_VFIP4TXOCTSLO_IP4TXOCTSLO_SHIFT) -#define I40E_GLPES_VFIP4TXPKTSHI(_i) (0x00019C04 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */ -#define I40E_GLPES_VFIP4TXPKTSHI_MAX_INDEX 31 -#define I40E_GLPES_VFIP4TXPKTSHI_IP4TXPKTSHI_SHIFT 0 -#define I40E_GLPES_VFIP4TXPKTSHI_IP4TXPKTSHI_MASK (0xFFFF << I40E_GLPES_VFIP4TXPKTSHI_IP4TXPKTSHI_SHIFT) -#define I40E_GLPES_VFIP4TXPKTSLO(_i) (0x00019C00 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */ -#define I40E_GLPES_VFIP4TXPKTSLO_MAX_INDEX 31 -#define I40E_GLPES_VFIP4TXPKTSLO_IP4TXPKTSLO_SHIFT 0 -#define I40E_GLPES_VFIP4TXPKTSLO_IP4TXPKTSLO_MASK (0xFFFFFFFF << I40E_GLPES_VFIP4TXPKTSLO_IP4TXPKTSLO_SHIFT) -#define I40E_GLPES_VFIP6RXDISCARD(_i) (0x00019200 + ((_i) * 4)) /* _i=0...31 */ /* Reset: PE_CORER */ -#define I40E_GLPES_VFIP6RXDISCARD_MAX_INDEX 31 -#define I40E_GLPES_VFIP6RXDISCARD_IP6RXDISCARD_SHIFT 0 -#define I40E_GLPES_VFIP6RXDISCARD_IP6RXDISCARD_MASK (0xFFFFFFFF << I40E_GLPES_VFIP6RXDISCARD_IP6RXDISCARD_SHIFT) -#define I40E_GLPES_VFIP6RXFRAGSHI(_i) (0x00019404 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */ -#define I40E_GLPES_VFIP6RXFRAGSHI_MAX_INDEX 31 -#define I40E_GLPES_VFIP6RXFRAGSHI_IP6RXFRAGSHI_SHIFT 0 -#define I40E_GLPES_VFIP6RXFRAGSHI_IP6RXFRAGSHI_MASK (0xFFFF << I40E_GLPES_VFIP6RXFRAGSHI_IP6RXFRAGSHI_SHIFT) -#define I40E_GLPES_VFIP6RXFRAGSLO(_i) (0x00019400 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */ -#define I40E_GLPES_VFIP6RXFRAGSLO_MAX_INDEX 31 -#define I40E_GLPES_VFIP6RXFRAGSLO_IP6RXFRAGSLO_SHIFT 0 -#define I40E_GLPES_VFIP6RXFRAGSLO_IP6RXFRAGSLO_MASK (0xFFFFFFFF << I40E_GLPES_VFIP6RXFRAGSLO_IP6RXFRAGSLO_SHIFT) -#define I40E_GLPES_VFIP6RXMCOCTSHI(_i) (0x00019604 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */ -#define I40E_GLPES_VFIP6RXMCOCTSHI_MAX_INDEX 31 -#define I40E_GLPES_VFIP6RXMCOCTSHI_IP6RXMCOCTSHI_SHIFT 0 -#define I40E_GLPES_VFIP6RXMCOCTSHI_IP6RXMCOCTSHI_MASK (0xFFFF << I40E_GLPES_VFIP6RXMCOCTSHI_IP6RXMCOCTSHI_SHIFT) -#define I40E_GLPES_VFIP6RXMCOCTSLO(_i) (0x00019600 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */ -#define I40E_GLPES_VFIP6RXMCOCTSLO_MAX_INDEX 31 -#define I40E_GLPES_VFIP6RXMCOCTSLO_IP6RXMCOCTSLO_SHIFT 0 -#define I40E_GLPES_VFIP6RXMCOCTSLO_IP6RXMCOCTSLO_MASK (0xFFFFFFFF << I40E_GLPES_VFIP6RXMCOCTSLO_IP6RXMCOCTSLO_SHIFT) -#define I40E_GLPES_VFIP6RXMCPKTSHI(_i) (0x00019804 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */ -#define I40E_GLPES_VFIP6RXMCPKTSHI_MAX_INDEX 31 -#define I40E_GLPES_VFIP6RXMCPKTSHI_IP6RXMCPKTSHI_SHIFT 0 -#define I40E_GLPES_VFIP6RXMCPKTSHI_IP6RXMCPKTSHI_MASK (0xFFFF << I40E_GLPES_VFIP6RXMCPKTSHI_IP6RXMCPKTSHI_SHIFT) -#define I40E_GLPES_VFIP6RXMCPKTSLO(_i) (0x00019800 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */ -#define I40E_GLPES_VFIP6RXMCPKTSLO_MAX_INDEX 31 -#define I40E_GLPES_VFIP6RXMCPKTSLO_IP6RXMCPKTSLO_SHIFT 0 -#define I40E_GLPES_VFIP6RXMCPKTSLO_IP6RXMCPKTSLO_MASK (0xFFFFFFFF << I40E_GLPES_VFIP6RXMCPKTSLO_IP6RXMCPKTSLO_SHIFT) -#define I40E_GLPES_VFIP6RXOCTSHI(_i) (0x00018E04 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */ -#define I40E_GLPES_VFIP6RXOCTSHI_MAX_INDEX 31 -#define I40E_GLPES_VFIP6RXOCTSHI_IP6RXOCTSHI_SHIFT 0 -#define I40E_GLPES_VFIP6RXOCTSHI_IP6RXOCTSHI_MASK (0xFFFF << I40E_GLPES_VFIP6RXOCTSHI_IP6RXOCTSHI_SHIFT) -#define I40E_GLPES_VFIP6RXOCTSLO(_i) (0x00018E00 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */ -#define I40E_GLPES_VFIP6RXOCTSLO_MAX_INDEX 31 -#define I40E_GLPES_VFIP6RXOCTSLO_IP6RXOCTSLO_SHIFT 0 -#define I40E_GLPES_VFIP6RXOCTSLO_IP6RXOCTSLO_MASK (0xFFFFFFFF << I40E_GLPES_VFIP6RXOCTSLO_IP6RXOCTSLO_SHIFT) -#define I40E_GLPES_VFIP6RXPKTSHI(_i) (0x00019004 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */ -#define I40E_GLPES_VFIP6RXPKTSHI_MAX_INDEX 31 -#define I40E_GLPES_VFIP6RXPKTSHI_IP6RXPKTSHI_SHIFT 0 -#define I40E_GLPES_VFIP6RXPKTSHI_IP6RXPKTSHI_MASK (0xFFFF << I40E_GLPES_VFIP6RXPKTSHI_IP6RXPKTSHI_SHIFT) -#define I40E_GLPES_VFIP6RXPKTSLO(_i) (0x00019000 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */ -#define I40E_GLPES_VFIP6RXPKTSLO_MAX_INDEX 31 -#define I40E_GLPES_VFIP6RXPKTSLO_IP6RXPKTSLO_SHIFT 0 -#define I40E_GLPES_VFIP6RXPKTSLO_IP6RXPKTSLO_MASK (0xFFFFFFFF << I40E_GLPES_VFIP6RXPKTSLO_IP6RXPKTSLO_SHIFT) -#define I40E_GLPES_VFIP6RXTRUNC(_i) (0x00019300 + ((_i) * 4)) /* _i=0...31 */ /* Reset: PE_CORER */ -#define I40E_GLPES_VFIP6RXTRUNC_MAX_INDEX 31 -#define I40E_GLPES_VFIP6RXTRUNC_IP6RXTRUNC_SHIFT 0 -#define I40E_GLPES_VFIP6RXTRUNC_IP6RXTRUNC_MASK (0xFFFFFFFF << I40E_GLPES_VFIP6RXTRUNC_IP6RXTRUNC_SHIFT) -#define I40E_GLPES_VFIP6TXFRAGSHI(_i) (0x0001A804 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */ -#define I40E_GLPES_VFIP6TXFRAGSHI_MAX_INDEX 31 -#define I40E_GLPES_VFIP6TXFRAGSHI_IP6TXFRAGSHI_SHIFT 0 -#define I40E_GLPES_VFIP6TXFRAGSHI_IP6TXFRAGSHI_MASK (0xFFFF << I40E_GLPES_VFIP6TXFRAGSHI_IP6TXFRAGSHI_SHIFT) -#define I40E_GLPES_VFIP6TXFRAGSLO(_i) (0x0001A800 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */ -#define I40E_GLPES_VFIP6TXFRAGSLO_MAX_INDEX 31 -#define I40E_GLPES_VFIP6TXFRAGSLO_IP6TXFRAGSLO_SHIFT 0 -#define I40E_GLPES_VFIP6TXFRAGSLO_IP6TXFRAGSLO_MASK (0xFFFFFFFF << I40E_GLPES_VFIP6TXFRAGSLO_IP6TXFRAGSLO_SHIFT) -#define I40E_GLPES_VFIP6TXMCOCTSHI(_i) (0x0001AA04 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */ -#define I40E_GLPES_VFIP6TXMCOCTSHI_MAX_INDEX 31 -#define I40E_GLPES_VFIP6TXMCOCTSHI_IP6TXMCOCTSHI_SHIFT 0 -#define I40E_GLPES_VFIP6TXMCOCTSHI_IP6TXMCOCTSHI_MASK (0xFFFF << I40E_GLPES_VFIP6TXMCOCTSHI_IP6TXMCOCTSHI_SHIFT) -#define I40E_GLPES_VFIP6TXMCOCTSLO(_i) (0x0001AA00 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */ -#define I40E_GLPES_VFIP6TXMCOCTSLO_MAX_INDEX 31 -#define I40E_GLPES_VFIP6TXMCOCTSLO_IP6TXMCOCTSLO_SHIFT 0 -#define I40E_GLPES_VFIP6TXMCOCTSLO_IP6TXMCOCTSLO_MASK (0xFFFFFFFF << I40E_GLPES_VFIP6TXMCOCTSLO_IP6TXMCOCTSLO_SHIFT) -#define I40E_GLPES_VFIP6TXMCPKTSHI(_i) (0x0001AC04 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */ -#define I40E_GLPES_VFIP6TXMCPKTSHI_MAX_INDEX 31 -#define I40E_GLPES_VFIP6TXMCPKTSHI_IP6TXMCPKTSHI_SHIFT 0 -#define I40E_GLPES_VFIP6TXMCPKTSHI_IP6TXMCPKTSHI_MASK (0xFFFF << I40E_GLPES_VFIP6TXMCPKTSHI_IP6TXMCPKTSHI_SHIFT) -#define I40E_GLPES_VFIP6TXMCPKTSLO(_i) (0x0001AC00 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */ -#define I40E_GLPES_VFIP6TXMCPKTSLO_MAX_INDEX 31 -#define I40E_GLPES_VFIP6TXMCPKTSLO_IP6TXMCPKTSLO_SHIFT 0 -#define I40E_GLPES_VFIP6TXMCPKTSLO_IP6TXMCPKTSLO_MASK (0xFFFFFFFF << I40E_GLPES_VFIP6TXMCPKTSLO_IP6TXMCPKTSLO_SHIFT) -#define I40E_GLPES_VFIP6TXNOROUTE(_i) (0x0001AF00 + ((_i) * 4)) /* _i=0...31 */ /* Reset: PE_CORER */ -#define I40E_GLPES_VFIP6TXNOROUTE_MAX_INDEX 31 -#define I40E_GLPES_VFIP6TXNOROUTE_IP6TXNOROUTE_SHIFT 0 -#define I40E_GLPES_VFIP6TXNOROUTE_IP6TXNOROUTE_MASK (0xFFFFFF << I40E_GLPES_VFIP6TXNOROUTE_IP6TXNOROUTE_SHIFT) -#define I40E_GLPES_VFIP6TXOCTSHI(_i) (0x0001A404 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */ -#define I40E_GLPES_VFIP6TXOCTSHI_MAX_INDEX 31 -#define I40E_GLPES_VFIP6TXOCTSHI_IP6TXOCTSHI_SHIFT 0 -#define I40E_GLPES_VFIP6TXOCTSHI_IP6TXOCTSHI_MASK (0xFFFF << I40E_GLPES_VFIP6TXOCTSHI_IP6TXOCTSHI_SHIFT) -#define I40E_GLPES_VFIP6TXOCTSLO(_i) (0x0001A400 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */ -#define I40E_GLPES_VFIP6TXOCTSLO_MAX_INDEX 31 -#define I40E_GLPES_VFIP6TXOCTSLO_IP6TXOCTSLO_SHIFT 0 -#define I40E_GLPES_VFIP6TXOCTSLO_IP6TXOCTSLO_MASK (0xFFFFFFFF << I40E_GLPES_VFIP6TXOCTSLO_IP6TXOCTSLO_SHIFT) -#define I40E_GLPES_VFIP6TXPKTSHI(_i) (0x0001A604 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */ -#define I40E_GLPES_VFIP6TXPKTSHI_MAX_INDEX 31 -#define I40E_GLPES_VFIP6TXPKTSHI_IP6TXPKTSHI_SHIFT 0 -#define I40E_GLPES_VFIP6TXPKTSHI_IP6TXPKTSHI_MASK (0xFFFF << I40E_GLPES_VFIP6TXPKTSHI_IP6TXPKTSHI_SHIFT) -#define I40E_GLPES_VFIP6TXPKTSLO(_i) (0x0001A600 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */ -#define I40E_GLPES_VFIP6TXPKTSLO_MAX_INDEX 31 -#define I40E_GLPES_VFIP6TXPKTSLO_IP6TXPKTSLO_SHIFT 0 -#define I40E_GLPES_VFIP6TXPKTSLO_IP6TXPKTSLO_MASK (0xFFFFFFFF << I40E_GLPES_VFIP6TXPKTSLO_IP6TXPKTSLO_SHIFT) -#define I40E_GLPES_VFRDMARXRDSHI(_i) (0x0001BE04 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */ -#define I40E_GLPES_VFRDMARXRDSHI_MAX_INDEX 31 -#define I40E_GLPES_VFRDMARXRDSHI_RDMARXRDSHI_SHIFT 0 -#define I40E_GLPES_VFRDMARXRDSHI_RDMARXRDSHI_MASK (0xFFFF << I40E_GLPES_VFRDMARXRDSHI_RDMARXRDSHI_SHIFT) -#define I40E_GLPES_VFRDMARXRDSLO(_i) (0x0001BE00 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */ -#define I40E_GLPES_VFRDMARXRDSLO_MAX_INDEX 31 -#define I40E_GLPES_VFRDMARXRDSLO_RDMARXRDSLO_SHIFT 0 -#define I40E_GLPES_VFRDMARXRDSLO_RDMARXRDSLO_MASK (0xFFFFFFFF << I40E_GLPES_VFRDMARXRDSLO_RDMARXRDSLO_SHIFT) -#define I40E_GLPES_VFRDMARXSNDSHI(_i) (0x0001C004 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */ -#define I40E_GLPES_VFRDMARXSNDSHI_MAX_INDEX 31 -#define I40E_GLPES_VFRDMARXSNDSHI_RDMARXSNDSHI_SHIFT 0 -#define I40E_GLPES_VFRDMARXSNDSHI_RDMARXSNDSHI_MASK (0xFFFF << I40E_GLPES_VFRDMARXSNDSHI_RDMARXSNDSHI_SHIFT) -#define I40E_GLPES_VFRDMARXSNDSLO(_i) (0x0001C000 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */ -#define I40E_GLPES_VFRDMARXSNDSLO_MAX_INDEX 31 -#define I40E_GLPES_VFRDMARXSNDSLO_RDMARXSNDSLO_SHIFT 0 -#define I40E_GLPES_VFRDMARXSNDSLO_RDMARXSNDSLO_MASK (0xFFFFFFFF << I40E_GLPES_VFRDMARXSNDSLO_RDMARXSNDSLO_SHIFT) -#define I40E_GLPES_VFRDMARXWRSHI(_i) (0x0001BC04 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */ -#define I40E_GLPES_VFRDMARXWRSHI_MAX_INDEX 31 -#define I40E_GLPES_VFRDMARXWRSHI_RDMARXWRSHI_SHIFT 0 -#define I40E_GLPES_VFRDMARXWRSHI_RDMARXWRSHI_MASK (0xFFFF << I40E_GLPES_VFRDMARXWRSHI_RDMARXWRSHI_SHIFT) -#define I40E_GLPES_VFRDMARXWRSLO(_i) (0x0001BC00 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */ -#define I40E_GLPES_VFRDMARXWRSLO_MAX_INDEX 31 -#define I40E_GLPES_VFRDMARXWRSLO_RDMARXWRSLO_SHIFT 0 -#define I40E_GLPES_VFRDMARXWRSLO_RDMARXWRSLO_MASK (0xFFFFFFFF << I40E_GLPES_VFRDMARXWRSLO_RDMARXWRSLO_SHIFT) -#define I40E_GLPES_VFRDMATXRDSHI(_i) (0x0001C404 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */ -#define I40E_GLPES_VFRDMATXRDSHI_MAX_INDEX 31 -#define I40E_GLPES_VFRDMATXRDSHI_RDMARXRDSHI_SHIFT 0 -#define I40E_GLPES_VFRDMATXRDSHI_RDMARXRDSHI_MASK (0xFFFF << I40E_GLPES_VFRDMATXRDSHI_RDMARXRDSHI_SHIFT) -#define I40E_GLPES_VFRDMATXRDSLO(_i) (0x0001C400 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */ -#define I40E_GLPES_VFRDMATXRDSLO_MAX_INDEX 31 -#define I40E_GLPES_VFRDMATXRDSLO_RDMARXRDSLO_SHIFT 0 -#define I40E_GLPES_VFRDMATXRDSLO_RDMARXRDSLO_MASK (0xFFFFFFFF << I40E_GLPES_VFRDMATXRDSLO_RDMARXRDSLO_SHIFT) -#define I40E_GLPES_VFRDMATXSNDSHI(_i) (0x0001C604 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */ -#define I40E_GLPES_VFRDMATXSNDSHI_MAX_INDEX 31 -#define I40E_GLPES_VFRDMATXSNDSHI_RDMARXSNDSHI_SHIFT 0 -#define I40E_GLPES_VFRDMATXSNDSHI_RDMARXSNDSHI_MASK (0xFFFF << I40E_GLPES_VFRDMATXSNDSHI_RDMARXSNDSHI_SHIFT) -#define I40E_GLPES_VFRDMATXSNDSLO(_i) (0x0001C600 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */ -#define I40E_GLPES_VFRDMATXSNDSLO_MAX_INDEX 31 -#define I40E_GLPES_VFRDMATXSNDSLO_RDMARXSNDSLO_SHIFT 0 -#define I40E_GLPES_VFRDMATXSNDSLO_RDMARXSNDSLO_MASK (0xFFFFFFFF << I40E_GLPES_VFRDMATXSNDSLO_RDMARXSNDSLO_SHIFT) -#define I40E_GLPES_VFRDMATXWRSHI(_i) (0x0001C204 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */ -#define I40E_GLPES_VFRDMATXWRSHI_MAX_INDEX 31 -#define I40E_GLPES_VFRDMATXWRSHI_RDMARXWRSHI_SHIFT 0 -#define I40E_GLPES_VFRDMATXWRSHI_RDMARXWRSHI_MASK (0xFFFF << I40E_GLPES_VFRDMATXWRSHI_RDMARXWRSHI_SHIFT) -#define I40E_GLPES_VFRDMATXWRSLO(_i) (0x0001C200 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */ -#define I40E_GLPES_VFRDMATXWRSLO_MAX_INDEX 31 -#define I40E_GLPES_VFRDMATXWRSLO_RDMARXWRSLO_SHIFT 0 -#define I40E_GLPES_VFRDMATXWRSLO_RDMARXWRSLO_MASK (0xFFFFFFFF << I40E_GLPES_VFRDMATXWRSLO_RDMARXWRSLO_SHIFT) -#define I40E_GLPES_VFRDMAVBNDHI(_i) (0x0001C804 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */ -#define I40E_GLPES_VFRDMAVBNDHI_MAX_INDEX 31 -#define I40E_GLPES_VFRDMAVBNDHI_RDMAVBNDHI_SHIFT 0 -#define I40E_GLPES_VFRDMAVBNDHI_RDMAVBNDHI_MASK (0xFFFFFFFF << I40E_GLPES_VFRDMAVBNDHI_RDMAVBNDHI_SHIFT) -#define I40E_GLPES_VFRDMAVBNDLO(_i) (0x0001C800 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */ -#define I40E_GLPES_VFRDMAVBNDLO_MAX_INDEX 31 -#define I40E_GLPES_VFRDMAVBNDLO_RDMAVBNDLO_SHIFT 0 -#define I40E_GLPES_VFRDMAVBNDLO_RDMAVBNDLO_MASK (0xFFFFFFFF << I40E_GLPES_VFRDMAVBNDLO_RDMAVBNDLO_SHIFT) -#define I40E_GLPES_VFRDMAVINVHI(_i) (0x0001CA04 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */ -#define I40E_GLPES_VFRDMAVINVHI_MAX_INDEX 31 -#define I40E_GLPES_VFRDMAVINVHI_RDMAVINVHI_SHIFT 0 -#define I40E_GLPES_VFRDMAVINVHI_RDMAVINVHI_MASK (0xFFFFFFFF << I40E_GLPES_VFRDMAVINVHI_RDMAVINVHI_SHIFT) -#define I40E_GLPES_VFRDMAVINVLO(_i) (0x0001CA00 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */ -#define I40E_GLPES_VFRDMAVINVLO_MAX_INDEX 31 -#define I40E_GLPES_VFRDMAVINVLO_RDMAVINVLO_SHIFT 0 -#define I40E_GLPES_VFRDMAVINVLO_RDMAVINVLO_MASK (0xFFFFFFFF << I40E_GLPES_VFRDMAVINVLO_RDMAVINVLO_SHIFT) -#define I40E_GLPES_VFRXVLANERR(_i) (0x00018000 + ((_i) * 4)) /* _i=0...31 */ /* Reset: PE_CORER */ -#define I40E_GLPES_VFRXVLANERR_MAX_INDEX 31 -#define I40E_GLPES_VFRXVLANERR_RXVLANERR_SHIFT 0 -#define I40E_GLPES_VFRXVLANERR_RXVLANERR_MASK (0xFFFFFF << I40E_GLPES_VFRXVLANERR_RXVLANERR_SHIFT) -#define I40E_GLPES_VFTCPRTXSEG(_i) (0x0001B600 + ((_i) * 4)) /* _i=0...31 */ /* Reset: PE_CORER */ -#define I40E_GLPES_VFTCPRTXSEG_MAX_INDEX 31 -#define I40E_GLPES_VFTCPRTXSEG_TCPRTXSEG_SHIFT 0 -#define I40E_GLPES_VFTCPRTXSEG_TCPRTXSEG_MASK (0xFFFFFFFF << I40E_GLPES_VFTCPRTXSEG_TCPRTXSEG_SHIFT) -#define I40E_GLPES_VFTCPRXOPTERR(_i) (0x0001B200 + ((_i) * 4)) /* _i=0...31 */ /* Reset: PE_CORER */ -#define I40E_GLPES_VFTCPRXOPTERR_MAX_INDEX 31 -#define I40E_GLPES_VFTCPRXOPTERR_TCPRXOPTERR_SHIFT 0 -#define I40E_GLPES_VFTCPRXOPTERR_TCPRXOPTERR_MASK (0xFFFFFF << I40E_GLPES_VFTCPRXOPTERR_TCPRXOPTERR_SHIFT) -#define I40E_GLPES_VFTCPRXPROTOERR(_i) (0x0001B300 + ((_i) * 4)) /* _i=0...31 */ /* Reset: PE_CORER */ -#define I40E_GLPES_VFTCPRXPROTOERR_MAX_INDEX 31 -#define I40E_GLPES_VFTCPRXPROTOERR_TCPRXPROTOERR_SHIFT 0 -#define I40E_GLPES_VFTCPRXPROTOERR_TCPRXPROTOERR_MASK (0xFFFFFF << I40E_GLPES_VFTCPRXPROTOERR_TCPRXPROTOERR_SHIFT) -#define I40E_GLPES_VFTCPRXSEGSHI(_i) (0x0001B004 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */ -#define I40E_GLPES_VFTCPRXSEGSHI_MAX_INDEX 31 -#define I40E_GLPES_VFTCPRXSEGSHI_TCPRXSEGSHI_SHIFT 0 -#define I40E_GLPES_VFTCPRXSEGSHI_TCPRXSEGSHI_MASK (0xFFFF << I40E_GLPES_VFTCPRXSEGSHI_TCPRXSEGSHI_SHIFT) -#define I40E_GLPES_VFTCPRXSEGSLO(_i) (0x0001B000 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */ -#define I40E_GLPES_VFTCPRXSEGSLO_MAX_INDEX 31 -#define I40E_GLPES_VFTCPRXSEGSLO_TCPRXSEGSLO_SHIFT 0 -#define I40E_GLPES_VFTCPRXSEGSLO_TCPRXSEGSLO_MASK (0xFFFFFFFF << I40E_GLPES_VFTCPRXSEGSLO_TCPRXSEGSLO_SHIFT) -#define I40E_GLPES_VFTCPTXSEGHI(_i) (0x0001B404 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */ -#define I40E_GLPES_VFTCPTXSEGHI_MAX_INDEX 31 -#define I40E_GLPES_VFTCPTXSEGHI_TCPTXSEGHI_SHIFT 0 -#define I40E_GLPES_VFTCPTXSEGHI_TCPTXSEGHI_MASK (0xFFFF << I40E_GLPES_VFTCPTXSEGHI_TCPTXSEGHI_SHIFT) -#define I40E_GLPES_VFTCPTXSEGLO(_i) (0x0001B400 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */ -#define I40E_GLPES_VFTCPTXSEGLO_MAX_INDEX 31 -#define I40E_GLPES_VFTCPTXSEGLO_TCPTXSEGLO_SHIFT 0 -#define I40E_GLPES_VFTCPTXSEGLO_TCPTXSEGLO_MASK (0xFFFFFFFF << I40E_GLPES_VFTCPTXSEGLO_TCPTXSEGLO_SHIFT) -#define I40E_GLPES_VFUDPRXPKTSHI(_i) (0x0001B804 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */ -#define I40E_GLPES_VFUDPRXPKTSHI_MAX_INDEX 31 -#define I40E_GLPES_VFUDPRXPKTSHI_UDPRXPKTSHI_SHIFT 0 -#define I40E_GLPES_VFUDPRXPKTSHI_UDPRXPKTSHI_MASK (0xFFFF << I40E_GLPES_VFUDPRXPKTSHI_UDPRXPKTSHI_SHIFT) -#define I40E_GLPES_VFUDPRXPKTSLO(_i) (0x0001B800 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */ -#define I40E_GLPES_VFUDPRXPKTSLO_MAX_INDEX 31 -#define I40E_GLPES_VFUDPRXPKTSLO_UDPRXPKTSLO_SHIFT 0 -#define I40E_GLPES_VFUDPRXPKTSLO_UDPRXPKTSLO_MASK (0xFFFFFFFF << I40E_GLPES_VFUDPRXPKTSLO_UDPRXPKTSLO_SHIFT) -#define I40E_GLPES_VFUDPTXPKTSHI(_i) (0x0001BA04 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */ -#define I40E_GLPES_VFUDPTXPKTSHI_MAX_INDEX 31 -#define I40E_GLPES_VFUDPTXPKTSHI_UDPTXPKTSHI_SHIFT 0 -#define I40E_GLPES_VFUDPTXPKTSHI_UDPTXPKTSHI_MASK (0xFFFF << I40E_GLPES_VFUDPTXPKTSHI_UDPTXPKTSHI_SHIFT) -#define I40E_GLPES_VFUDPTXPKTSLO(_i) (0x0001BA00 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */ -#define I40E_GLPES_VFUDPTXPKTSLO_MAX_INDEX 31 -#define I40E_GLPES_VFUDPTXPKTSLO_UDPTXPKTSLO_SHIFT 0 -#define I40E_GLPES_VFUDPTXPKTSLO_UDPTXPKTSLO_MASK (0xFFFFFFFF << I40E_GLPES_VFUDPTXPKTSLO_UDPTXPKTSLO_SHIFT) - -#define I40E_VFPE_AEQALLOC1 0x0000A400 /* Reset: VFR */ -#define I40E_VFPE_AEQALLOC1_AECOUNT_SHIFT 0 -#define I40E_VFPE_AEQALLOC1_AECOUNT_MASK (0xFFFFFFFF << I40E_VFPE_AEQALLOC1_AECOUNT_SHIFT) -#define I40E_VFPE_CCQPHIGH1 0x00009800 /* Reset: VFR */ -#define I40E_VFPE_CCQPHIGH1_PECCQPHIGH_SHIFT 0 -#define I40E_VFPE_CCQPHIGH1_PECCQPHIGH_MASK (0xFFFFFFFF << I40E_VFPE_CCQPHIGH1_PECCQPHIGH_SHIFT) -#define I40E_VFPE_CCQPLOW1 0x0000AC00 /* Reset: VFR */ -#define I40E_VFPE_CCQPLOW1_PECCQPLOW_SHIFT 0 -#define I40E_VFPE_CCQPLOW1_PECCQPLOW_MASK (0xFFFFFFFF << I40E_VFPE_CCQPLOW1_PECCQPLOW_SHIFT) -#define I40E_VFPE_CCQPSTATUS1 0x0000B800 /* Reset: VFR */ -#define I40E_VFPE_CCQPSTATUS1_CCQP_DONE_SHIFT 0 -#define I40E_VFPE_CCQPSTATUS1_CCQP_DONE_MASK (0x1 << I40E_VFPE_CCQPSTATUS1_CCQP_DONE_SHIFT) -#define I40E_VFPE_CCQPSTATUS1_HMC_PROFILE_SHIFT 4 -#define I40E_VFPE_CCQPSTATUS1_HMC_PROFILE_MASK (0x7 << I40E_VFPE_CCQPSTATUS1_HMC_PROFILE_SHIFT) -#define I40E_VFPE_CCQPSTATUS1_RDMA_EN_VFS_SHIFT 16 -#define I40E_VFPE_CCQPSTATUS1_RDMA_EN_VFS_MASK (0x3F << I40E_VFPE_CCQPSTATUS1_RDMA_EN_VFS_SHIFT) -#define I40E_VFPE_CCQPSTATUS1_CCQP_ERR_SHIFT 31 -#define I40E_VFPE_CCQPSTATUS1_CCQP_ERR_MASK (0x1 << I40E_VFPE_CCQPSTATUS1_CCQP_ERR_SHIFT) -#define I40E_VFPE_CQACK1 0x0000B000 /* Reset: VFR */ -#define I40E_VFPE_CQACK1_PECQID_SHIFT 0 -#define I40E_VFPE_CQACK1_PECQID_MASK (0x1FFFF << I40E_VFPE_CQACK1_PECQID_SHIFT) -#define I40E_VFPE_CQARM1 0x0000B400 /* Reset: VFR */ -#define I40E_VFPE_CQARM1_PECQID_SHIFT 0 -#define I40E_VFPE_CQARM1_PECQID_MASK (0x1FFFF << I40E_VFPE_CQARM1_PECQID_SHIFT) -#define I40E_VFPE_CQPDB1 0x0000BC00 /* Reset: VFR */ -#define I40E_VFPE_CQPDB1_WQHEAD_SHIFT 0 -#define I40E_VFPE_CQPDB1_WQHEAD_MASK (0x7FF << I40E_VFPE_CQPDB1_WQHEAD_SHIFT) -#define I40E_VFPE_CQPERRCODES1 0x00009C00 /* Reset: VFR */ -#define I40E_VFPE_CQPERRCODES1_CQP_MINOR_CODE_SHIFT 0 -#define I40E_VFPE_CQPERRCODES1_CQP_MINOR_CODE_MASK (0xFFFF << I40E_VFPE_CQPERRCODES1_CQP_MINOR_CODE_SHIFT) -#define I40E_VFPE_CQPERRCODES1_CQP_MAJOR_CODE_SHIFT 16 -#define I40E_VFPE_CQPERRCODES1_CQP_MAJOR_CODE_MASK (0xFFFF << I40E_VFPE_CQPERRCODES1_CQP_MAJOR_CODE_SHIFT) -#define I40E_VFPE_CQPTAIL1 0x0000A000 /* Reset: VFR */ -#define I40E_VFPE_CQPTAIL1_WQTAIL_SHIFT 0 -#define I40E_VFPE_CQPTAIL1_WQTAIL_MASK (0x7FF << I40E_VFPE_CQPTAIL1_WQTAIL_SHIFT) -#define I40E_VFPE_CQPTAIL1_CQP_OP_ERR_SHIFT 31 -#define I40E_VFPE_CQPTAIL1_CQP_OP_ERR_MASK (0x1 << I40E_VFPE_CQPTAIL1_CQP_OP_ERR_SHIFT) -#define I40E_VFPE_IPCONFIG01 0x00008C00 /* Reset: VFR */ -#define I40E_VFPE_IPCONFIG01_PEIPID_SHIFT 0 -#define I40E_VFPE_IPCONFIG01_PEIPID_MASK (0xFFFF << I40E_VFPE_IPCONFIG01_PEIPID_SHIFT) -#define I40E_VFPE_IPCONFIG01_USEENTIREIDRANGE_SHIFT 16 -#define I40E_VFPE_IPCONFIG01_USEENTIREIDRANGE_MASK (0x1 << I40E_VFPE_IPCONFIG01_USEENTIREIDRANGE_SHIFT) -#define I40E_VFPE_MRTEIDXMASK1 0x00009000 /* Reset: VFR */ -#define I40E_VFPE_MRTEIDXMASK1_MRTEIDXMASKBITS_SHIFT 0 -#define I40E_VFPE_MRTEIDXMASK1_MRTEIDXMASKBITS_MASK (0x1F << I40E_VFPE_MRTEIDXMASK1_MRTEIDXMASKBITS_SHIFT) -#define I40E_VFPE_RCVUNEXPECTEDERROR1 0x00009400 /* Reset: VFR */ -#define I40E_VFPE_RCVUNEXPECTEDERROR1_TCP_RX_UNEXP_ERR_SHIFT 0 -#define I40E_VFPE_RCVUNEXPECTEDERROR1_TCP_RX_UNEXP_ERR_MASK (0xFFFFFF << I40E_VFPE_RCVUNEXPECTEDERROR1_TCP_RX_UNEXP_ERR_SHIFT) -#define I40E_VFPE_TCPNOWTIMER1 0x0000A800 /* Reset: VFR */ -#define I40E_VFPE_TCPNOWTIMER1_TCP_NOW_SHIFT 0 -#define I40E_VFPE_TCPNOWTIMER1_TCP_NOW_MASK (0xFFFFFFFF << I40E_VFPE_TCPNOWTIMER1_TCP_NOW_SHIFT) -#define I40E_VFPE_WQEALLOC1 0x0000C000 /* Reset: VFR */ -#define I40E_VFPE_WQEALLOC1_PEQPID_SHIFT 0 -#define I40E_VFPE_WQEALLOC1_PEQPID_MASK (0x3FFFF << I40E_VFPE_WQEALLOC1_PEQPID_SHIFT) -#define I40E_VFPE_WQEALLOC1_WQE_DESC_INDEX_SHIFT 20 -#define I40E_VFPE_WQEALLOC1_WQE_DESC_INDEX_MASK (0xFFF << I40E_VFPE_WQEALLOC1_WQE_DESC_INDEX_SHIFT) -#endif /* I40IW_REGISTER_H */ diff --git a/drivers/infiniband/hw/i40iw/i40iw_status.h b/drivers/infiniband/hw/i40iw/i40iw_status.h deleted file mode 100644 index 36a19c4e5bba..000000000000 --- a/drivers/infiniband/hw/i40iw/i40iw_status.h +++ /dev/null @@ -1,101 +0,0 @@ -/******************************************************************************* -* -* Copyright (c) 2015-2016 Intel Corporation. All rights reserved. -* -* This software is available to you under a choice of one of two -* licenses. You may choose to be licensed under the terms of the GNU -* General Public License (GPL) Version 2, available from the file -* COPYING in the main directory of this source tree, or the -* OpenFabrics.org BSD license below: -* -* Redistribution and use in source and binary forms, with or -* without modification, are permitted provided that the following -* conditions are met: -* -* - Redistributions of source code must retain the above -* copyright notice, this list of conditions and the following -* disclaimer. -* -* - Redistributions in binary form must reproduce the above -* copyright notice, this list of conditions and the following -* disclaimer in the documentation and/or other materials -* provided with the distribution. -* -* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS -* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN -* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -* SOFTWARE. -* -*******************************************************************************/ - -#ifndef I40IW_STATUS_H -#define I40IW_STATUS_H - -/* Error Codes */ -enum i40iw_status_code { - I40IW_SUCCESS = 0, - I40IW_ERR_NVM = -1, - I40IW_ERR_NVM_CHECKSUM = -2, - I40IW_ERR_CONFIG = -4, - I40IW_ERR_PARAM = -5, - I40IW_ERR_DEVICE_NOT_SUPPORTED = -6, - I40IW_ERR_RESET_FAILED = -7, - I40IW_ERR_SWFW_SYNC = -8, - I40IW_ERR_NO_MEMORY = -9, - I40IW_ERR_BAD_PTR = -10, - I40IW_ERR_INVALID_PD_ID = -11, - I40IW_ERR_INVALID_QP_ID = -12, - I40IW_ERR_INVALID_CQ_ID = -13, - I40IW_ERR_INVALID_CEQ_ID = -14, - I40IW_ERR_INVALID_AEQ_ID = -15, - I40IW_ERR_INVALID_SIZE = -16, - I40IW_ERR_INVALID_ARP_INDEX = -17, - I40IW_ERR_INVALID_FPM_FUNC_ID = -18, - I40IW_ERR_QP_INVALID_MSG_SIZE = -19, - I40IW_ERR_QP_TOOMANY_WRS_POSTED = -20, - I40IW_ERR_INVALID_FRAG_COUNT = -21, - I40IW_ERR_QUEUE_EMPTY = -22, - I40IW_ERR_INVALID_ALIGNMENT = -23, - I40IW_ERR_FLUSHED_QUEUE = -24, - I40IW_ERR_INVALID_INLINE_DATA_SIZE = -26, - I40IW_ERR_TIMEOUT = -27, - I40IW_ERR_OPCODE_MISMATCH = -28, - I40IW_ERR_CQP_COMPL_ERROR = -29, - I40IW_ERR_INVALID_VF_ID = -30, - I40IW_ERR_INVALID_HMCFN_ID = -31, - I40IW_ERR_BACKING_PAGE_ERROR = -32, - I40IW_ERR_NO_PBLCHUNKS_AVAILABLE = -33, - I40IW_ERR_INVALID_PBLE_INDEX = -34, - I40IW_ERR_INVALID_SD_INDEX = -35, - I40IW_ERR_INVALID_PAGE_DESC_INDEX = -36, - I40IW_ERR_INVALID_SD_TYPE = -37, - I40IW_ERR_MEMCPY_FAILED = -38, - I40IW_ERR_INVALID_HMC_OBJ_INDEX = -39, - I40IW_ERR_INVALID_HMC_OBJ_COUNT = -40, - I40IW_ERR_INVALID_SRQ_ARM_LIMIT = -41, - I40IW_ERR_SRQ_ENABLED = -42, - I40IW_ERR_BUF_TOO_SHORT = -43, - I40IW_ERR_BAD_IWARP_CQE = -44, - I40IW_ERR_NVM_BLANK_MODE = -45, - I40IW_ERR_NOT_IMPLEMENTED = -46, - I40IW_ERR_PE_DOORBELL_NOT_ENABLED = -47, - I40IW_ERR_NOT_READY = -48, - I40IW_NOT_SUPPORTED = -49, - I40IW_ERR_FIRMWARE_API_VERSION = -50, - I40IW_ERR_RING_FULL = -51, - I40IW_ERR_MPA_CRC = -61, - I40IW_ERR_NO_TXBUFS = -62, - I40IW_ERR_SEQ_NUM = -63, - I40IW_ERR_list_empty = -64, - I40IW_ERR_INVALID_MAC_ADDR = -65, - I40IW_ERR_BAD_STAG = -66, - I40IW_ERR_CQ_COMPL_ERROR = -67, - I40IW_ERR_QUEUE_DESTROYED = -68, - I40IW_ERR_INVALID_FEAT_CNT = -69 - -}; -#endif diff --git a/drivers/infiniband/hw/i40iw/i40iw_type.h b/drivers/infiniband/hw/i40iw/i40iw_type.h deleted file mode 100644 index 394e182686cf..000000000000 --- a/drivers/infiniband/hw/i40iw/i40iw_type.h +++ /dev/null @@ -1,1358 +0,0 @@ -/******************************************************************************* -* -* Copyright (c) 2015-2016 Intel Corporation. All rights reserved. -* -* This software is available to you under a choice of one of two -* licenses. You may choose to be licensed under the terms of the GNU -* General Public License (GPL) Version 2, available from the file -* COPYING in the main directory of this source tree, or the -* OpenFabrics.org BSD license below: -* -* Redistribution and use in source and binary forms, with or -* without modification, are permitted provided that the following -* conditions are met: -* -* - Redistributions of source code must retain the above -* copyright notice, this list of conditions and the following -* disclaimer. -* -* - Redistributions in binary form must reproduce the above -* copyright notice, this list of conditions and the following -* disclaimer in the documentation and/or other materials -* provided with the distribution. -* -* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS -* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN -* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -* SOFTWARE. -* -*******************************************************************************/ - -#ifndef I40IW_TYPE_H -#define I40IW_TYPE_H -#include "i40iw_user.h" -#include "i40iw_hmc.h" -#include "i40iw_vf.h" -#include "i40iw_virtchnl.h" - -struct i40iw_cqp_sq_wqe { - u64 buf[I40IW_CQP_WQE_SIZE]; -}; - -struct i40iw_sc_aeqe { - u64 buf[I40IW_AEQE_SIZE]; -}; - -struct i40iw_ceqe { - u64 buf[I40IW_CEQE_SIZE]; -}; - -struct i40iw_cqp_ctx { - u64 buf[I40IW_CQP_CTX_SIZE]; -}; - -struct i40iw_cq_shadow_area { - u64 buf[I40IW_SHADOW_AREA_SIZE]; -}; - -struct i40iw_sc_dev; -struct i40iw_hmc_info; -struct i40iw_vsi_pestat; - -struct i40iw_cqp_ops; -struct i40iw_ccq_ops; -struct i40iw_ceq_ops; -struct i40iw_aeq_ops; -struct i40iw_mr_ops; -struct i40iw_cqp_misc_ops; -struct i40iw_pd_ops; -struct i40iw_priv_qp_ops; -struct i40iw_priv_cq_ops; -struct i40iw_hmc_ops; -struct pci_dev; - -enum i40iw_page_size { - I40IW_PAGE_SIZE_4K, - I40IW_PAGE_SIZE_2M -}; - -enum i40iw_resource_indicator_type { - I40IW_RSRC_INDICATOR_TYPE_ADAPTER = 0, - I40IW_RSRC_INDICATOR_TYPE_CQ, - I40IW_RSRC_INDICATOR_TYPE_QP, - I40IW_RSRC_INDICATOR_TYPE_SRQ -}; - -enum i40iw_hdrct_flags { - DDP_LEN_FLAG = 0x80, - DDP_HDR_FLAG = 0x40, - RDMA_HDR_FLAG = 0x20 -}; - -enum i40iw_term_layers { - LAYER_RDMA = 0, - LAYER_DDP = 1, - LAYER_MPA = 2 -}; - -enum i40iw_term_error_types { - RDMAP_REMOTE_PROT = 1, - RDMAP_REMOTE_OP = 2, - DDP_CATASTROPHIC = 0, - DDP_TAGGED_BUFFER = 1, - DDP_UNTAGGED_BUFFER = 2, - DDP_LLP = 3 -}; - -enum i40iw_term_rdma_errors { - RDMAP_INV_STAG = 0x00, - RDMAP_INV_BOUNDS = 0x01, - RDMAP_ACCESS = 0x02, - RDMAP_UNASSOC_STAG = 0x03, - RDMAP_TO_WRAP = 0x04, - RDMAP_INV_RDMAP_VER = 0x05, - RDMAP_UNEXPECTED_OP = 0x06, - RDMAP_CATASTROPHIC_LOCAL = 0x07, - RDMAP_CATASTROPHIC_GLOBAL = 0x08, - RDMAP_CANT_INV_STAG = 0x09, - RDMAP_UNSPECIFIED = 0xff -}; - -enum i40iw_term_ddp_errors { - DDP_CATASTROPHIC_LOCAL = 0x00, - DDP_TAGGED_INV_STAG = 0x00, - DDP_TAGGED_BOUNDS = 0x01, - DDP_TAGGED_UNASSOC_STAG = 0x02, - DDP_TAGGED_TO_WRAP = 0x03, - DDP_TAGGED_INV_DDP_VER = 0x04, - DDP_UNTAGGED_INV_QN = 0x01, - DDP_UNTAGGED_INV_MSN_NO_BUF = 0x02, - DDP_UNTAGGED_INV_MSN_RANGE = 0x03, - DDP_UNTAGGED_INV_MO = 0x04, - DDP_UNTAGGED_INV_TOO_LONG = 0x05, - DDP_UNTAGGED_INV_DDP_VER = 0x06 -}; - -enum i40iw_term_mpa_errors { - MPA_CLOSED = 0x01, - MPA_CRC = 0x02, - MPA_MARKER = 0x03, - MPA_REQ_RSP = 0x04, -}; - -enum i40iw_flush_opcode { - FLUSH_INVALID = 0, - FLUSH_PROT_ERR, - FLUSH_REM_ACCESS_ERR, - FLUSH_LOC_QP_OP_ERR, - FLUSH_REM_OP_ERR, - FLUSH_LOC_LEN_ERR, - FLUSH_GENERAL_ERR, - FLUSH_FATAL_ERR -}; - -enum i40iw_term_eventtypes { - TERM_EVENT_QP_FATAL, - TERM_EVENT_QP_ACCESS_ERR -}; - -struct i40iw_terminate_hdr { - u8 layer_etype; - u8 error_code; - u8 hdrct; - u8 rsvd; -}; - -enum i40iw_debug_flag { - I40IW_DEBUG_NONE = 0x00000000, - I40IW_DEBUG_ERR = 0x00000001, - I40IW_DEBUG_INIT = 0x00000002, - I40IW_DEBUG_DEV = 0x00000004, - I40IW_DEBUG_CM = 0x00000008, - I40IW_DEBUG_VERBS = 0x00000010, - I40IW_DEBUG_PUDA = 0x00000020, - I40IW_DEBUG_ILQ = 0x00000040, - I40IW_DEBUG_IEQ = 0x00000080, - I40IW_DEBUG_QP = 0x00000100, - I40IW_DEBUG_CQ = 0x00000200, - I40IW_DEBUG_MR = 0x00000400, - I40IW_DEBUG_PBLE = 0x00000800, - I40IW_DEBUG_WQE = 0x00001000, - I40IW_DEBUG_AEQ = 0x00002000, - I40IW_DEBUG_CQP = 0x00004000, - I40IW_DEBUG_HMC = 0x00008000, - I40IW_DEBUG_USER = 0x00010000, - I40IW_DEBUG_VIRT = 0x00020000, - I40IW_DEBUG_DCB = 0x00040000, - I40IW_DEBUG_CQE = 0x00800000, - I40IW_DEBUG_ALL = 0xFFFFFFFF -}; - -enum i40iw_hw_stats_index_32b { - I40IW_HW_STAT_INDEX_IP4RXDISCARD = 0, - I40IW_HW_STAT_INDEX_IP4RXTRUNC, - I40IW_HW_STAT_INDEX_IP4TXNOROUTE, - I40IW_HW_STAT_INDEX_IP6RXDISCARD, - I40IW_HW_STAT_INDEX_IP6RXTRUNC, - I40IW_HW_STAT_INDEX_IP6TXNOROUTE, - I40IW_HW_STAT_INDEX_TCPRTXSEG, - I40IW_HW_STAT_INDEX_TCPRXOPTERR, - I40IW_HW_STAT_INDEX_TCPRXPROTOERR, - I40IW_HW_STAT_INDEX_MAX_32 -}; - -enum i40iw_hw_stats_index_64b { - I40IW_HW_STAT_INDEX_IP4RXOCTS = 0, - I40IW_HW_STAT_INDEX_IP4RXPKTS, - I40IW_HW_STAT_INDEX_IP4RXFRAGS, - I40IW_HW_STAT_INDEX_IP4RXMCPKTS, - I40IW_HW_STAT_INDEX_IP4TXOCTS, - I40IW_HW_STAT_INDEX_IP4TXPKTS, - I40IW_HW_STAT_INDEX_IP4TXFRAGS, - I40IW_HW_STAT_INDEX_IP4TXMCPKTS, - I40IW_HW_STAT_INDEX_IP6RXOCTS, - I40IW_HW_STAT_INDEX_IP6RXPKTS, - I40IW_HW_STAT_INDEX_IP6RXFRAGS, - I40IW_HW_STAT_INDEX_IP6RXMCPKTS, - I40IW_HW_STAT_INDEX_IP6TXOCTS, - I40IW_HW_STAT_INDEX_IP6TXPKTS, - I40IW_HW_STAT_INDEX_IP6TXFRAGS, - I40IW_HW_STAT_INDEX_IP6TXMCPKTS, - I40IW_HW_STAT_INDEX_TCPRXSEGS, - I40IW_HW_STAT_INDEX_TCPTXSEG, - I40IW_HW_STAT_INDEX_RDMARXRDS, - I40IW_HW_STAT_INDEX_RDMARXSNDS, - I40IW_HW_STAT_INDEX_RDMARXWRS, - I40IW_HW_STAT_INDEX_RDMATXRDS, - I40IW_HW_STAT_INDEX_RDMATXSNDS, - I40IW_HW_STAT_INDEX_RDMATXWRS, - I40IW_HW_STAT_INDEX_RDMAVBND, - I40IW_HW_STAT_INDEX_RDMAVINV, - I40IW_HW_STAT_INDEX_MAX_64 -}; - -enum i40iw_feature_type { - I40IW_FEATURE_FW_INFO = 0, - I40IW_MAX_FEATURES -}; - -struct i40iw_dev_hw_stats_offsets { - u32 stats_offset_32[I40IW_HW_STAT_INDEX_MAX_32]; - u32 stats_offset_64[I40IW_HW_STAT_INDEX_MAX_64]; -}; - -struct i40iw_dev_hw_stats { - u64 stats_value_32[I40IW_HW_STAT_INDEX_MAX_32]; - u64 stats_value_64[I40IW_HW_STAT_INDEX_MAX_64]; -}; - -struct i40iw_vsi_pestat { - struct i40iw_hw *hw; - struct i40iw_dev_hw_stats hw_stats; - struct i40iw_dev_hw_stats last_read_hw_stats; - struct i40iw_dev_hw_stats_offsets hw_stats_offsets; - struct timer_list stats_timer; - struct i40iw_sc_vsi *vsi; - spinlock_t lock; /* rdma stats lock */ -}; - -struct i40iw_hw { - u8 __iomem *hw_addr; - struct pci_dev *pcidev; - struct i40iw_hmc_info hmc; -}; - -struct i40iw_pfpdu { - struct list_head rxlist; - u32 rcv_nxt; - u32 fps; - u32 max_fpdu_data; - bool mode; - bool mpa_crc_err; - u64 total_ieq_bufs; - u64 fpdu_processed; - u64 bad_seq_num; - u64 crc_err; - u64 no_tx_bufs; - u64 tx_err; - u64 out_of_order; - u64 pmode_count; -}; - -struct i40iw_sc_pd { - u32 size; - struct i40iw_sc_dev *dev; - u16 pd_id; - int abi_ver; -}; - -struct i40iw_cqp_quanta { - u64 elem[I40IW_CQP_WQE_SIZE]; -}; - -struct i40iw_sc_cqp { - u32 size; - u64 sq_pa; - u64 host_ctx_pa; - void *back_cqp; - struct i40iw_sc_dev *dev; - enum i40iw_status_code (*process_cqp_sds)(struct i40iw_sc_dev *, - struct i40iw_update_sds_info *); - struct i40iw_dma_mem sdbuf; - struct i40iw_ring sq_ring; - struct i40iw_cqp_quanta *sq_base; - u64 *host_ctx; - u64 *scratch_array; - u32 cqp_id; - u32 sq_size; - u32 hw_sq_size; - u8 struct_ver; - u8 polarity; - bool en_datacenter_tcp; - u8 hmc_profile; - u8 enabled_vf_count; - u8 timeout_count; -}; - -struct i40iw_sc_aeq { - u32 size; - u64 aeq_elem_pa; - struct i40iw_sc_dev *dev; - struct i40iw_sc_aeqe *aeqe_base; - void *pbl_list; - u32 elem_cnt; - struct i40iw_ring aeq_ring; - bool virtual_map; - u8 pbl_chunk_size; - u32 first_pm_pbl_idx; - u8 polarity; -}; - -struct i40iw_sc_ceq { - u32 size; - u64 ceq_elem_pa; - struct i40iw_sc_dev *dev; - struct i40iw_ceqe *ceqe_base; - void *pbl_list; - u32 ceq_id; - u32 elem_cnt; - struct i40iw_ring ceq_ring; - bool virtual_map; - u8 pbl_chunk_size; - bool tph_en; - u8 tph_val; - u32 first_pm_pbl_idx; - u8 polarity; -}; - -struct i40iw_sc_cq { - struct i40iw_cq_uk cq_uk; - u64 cq_pa; - u64 shadow_area_pa; - struct i40iw_sc_dev *dev; - struct i40iw_sc_vsi *vsi; - void *pbl_list; - void *back_cq; - u32 ceq_id; - u32 shadow_read_threshold; - bool ceqe_mask; - bool virtual_map; - u8 pbl_chunk_size; - u8 cq_type; - bool ceq_id_valid; - bool tph_en; - u8 tph_val; - u32 first_pm_pbl_idx; - bool check_overflow; -}; - -struct i40iw_sc_qp { - struct i40iw_qp_uk qp_uk; - u64 sq_pa; - u64 rq_pa; - u64 hw_host_ctx_pa; - u64 shadow_area_pa; - u64 q2_pa; - struct i40iw_sc_dev *dev; - struct i40iw_sc_vsi *vsi; - struct i40iw_sc_pd *pd; - u64 *hw_host_ctx; - void *llp_stream_handle; - void *back_qp; - struct i40iw_pfpdu pfpdu; - u8 *q2_buf; - u64 qp_compl_ctx; - u16 qs_handle; - u8 sq_tph_val; - u8 rq_tph_val; - u8 qp_state; - u8 qp_type; - u8 hw_sq_size; - u8 hw_rq_size; - u8 src_mac_addr_idx; - bool sq_tph_en; - bool rq_tph_en; - bool rcv_tph_en; - bool xmit_tph_en; - bool virtual_map; - bool flush_sq; - bool flush_rq; - u8 user_pri; - struct list_head list; - bool on_qoslist; - bool sq_flush; - enum i40iw_flush_opcode flush_code; - enum i40iw_term_eventtypes eventtype; - u8 term_flags; -}; - -struct i40iw_hmc_fpm_misc { - u32 max_ceqs; - u32 max_sds; - u32 xf_block_size; - u32 q1_block_size; - u32 ht_multiplier; - u32 timer_bucket; -}; - -struct i40iw_vchnl_if { - enum i40iw_status_code (*vchnl_recv)(struct i40iw_sc_dev *, u32, u8 *, u16); - enum i40iw_status_code (*vchnl_send)(struct i40iw_sc_dev *dev, u32, u8 *, u16); -}; - -#define I40IW_VCHNL_MAX_VF_MSG_SIZE 512 - -struct i40iw_vchnl_vf_msg_buffer { - struct i40iw_virtchnl_op_buf vchnl_msg; - char parm_buffer[I40IW_VCHNL_MAX_VF_MSG_SIZE - 1]; -}; - -struct i40iw_qos { - struct list_head qplist; - spinlock_t lock; /* qos list */ - u16 qs_handle; -}; - -struct i40iw_vfdev { - struct i40iw_sc_dev *pf_dev; - u8 *hmc_info_mem; - struct i40iw_vsi_pestat pestat; - struct i40iw_hmc_pble_info *pble_info; - struct i40iw_hmc_info hmc_info; - struct i40iw_vchnl_vf_msg_buffer vf_msg_buffer; - u64 fpm_query_buf_pa; - u64 *fpm_query_buf; - u32 vf_id; - u32 msg_count; - bool pf_hmc_initialized; - u16 pmf_index; - u16 iw_vf_idx; /* VF Device table index */ - bool stats_initialized; -}; - -#define I40IW_INVALID_FCN_ID 0xff -struct i40iw_sc_vsi { - struct i40iw_sc_dev *dev; - void *back_vsi; /* Owned by OS */ - u32 ilq_count; - struct i40iw_virt_mem ilq_mem; - struct i40iw_puda_rsrc *ilq; - u32 ieq_count; - struct i40iw_virt_mem ieq_mem; - struct i40iw_puda_rsrc *ieq; - u16 exception_lan_queue; - u16 mtu; - u8 fcn_id; - bool stats_fcn_id_alloc; - struct i40iw_qos qos[I40IW_MAX_USER_PRIORITY]; - struct i40iw_vsi_pestat *pestat; -}; - -struct i40iw_sc_dev { - struct list_head cqp_cmd_head; /* head of the CQP command list */ - spinlock_t cqp_lock; /* cqp list sync */ - struct i40iw_dev_uk dev_uk; - bool fcn_id_array[I40IW_MAX_STATS_COUNT]; - struct i40iw_dma_mem vf_fpm_query_buf[I40IW_MAX_PE_ENABLED_VF_COUNT]; - u64 fpm_query_buf_pa; - u64 fpm_commit_buf_pa; - u64 *fpm_query_buf; - u64 *fpm_commit_buf; - void *back_dev; - struct i40iw_hw *hw; - u8 __iomem *db_addr; - struct i40iw_hmc_info *hmc_info; - struct i40iw_hmc_pble_info *pble_info; - struct i40iw_vfdev *vf_dev[I40IW_MAX_PE_ENABLED_VF_COUNT]; - struct i40iw_sc_cqp *cqp; - struct i40iw_sc_aeq *aeq; - struct i40iw_sc_ceq *ceq[I40IW_CEQ_MAX_COUNT]; - struct i40iw_sc_cq *ccq; - const struct i40iw_cqp_ops *cqp_ops; - const struct i40iw_ccq_ops *ccq_ops; - const struct i40iw_ceq_ops *ceq_ops; - const struct i40iw_aeq_ops *aeq_ops; - const struct i40iw_pd_ops *iw_pd_ops; - const struct i40iw_priv_qp_ops *iw_priv_qp_ops; - const struct i40iw_priv_cq_ops *iw_priv_cq_ops; - const struct i40iw_mr_ops *mr_ops; - const struct i40iw_cqp_misc_ops *cqp_misc_ops; - const struct i40iw_hmc_ops *hmc_ops; - struct i40iw_vchnl_if vchnl_if; - const struct i40iw_vf_cqp_ops *iw_vf_cqp_ops; - - struct i40iw_hmc_fpm_misc hmc_fpm_misc; - u64 feature_info[I40IW_MAX_FEATURES]; - u32 debug_mask; - u8 hmc_fn_id; - bool is_pf; - bool vchnl_up; - bool ceq_valid; - u8 vf_id; - wait_queue_head_t vf_reqs; - u64 cqp_cmd_stats[OP_SIZE_CQP_STAT_ARRAY]; - struct i40iw_vchnl_vf_msg_buffer vchnl_vf_msg_buf; - u8 hw_rev; -}; - -struct i40iw_modify_cq_info { - u64 cq_pa; - struct i40iw_cqe *cq_base; - void *pbl_list; - u32 ceq_id; - u32 cq_size; - u32 shadow_read_threshold; - bool virtual_map; - u8 pbl_chunk_size; - bool check_overflow; - bool cq_resize; - bool ceq_change; - bool check_overflow_change; - u32 first_pm_pbl_idx; - bool ceq_valid; -}; - -struct i40iw_create_qp_info { - u8 next_iwarp_state; - bool ord_valid; - bool tcp_ctx_valid; - bool cq_num_valid; - bool arp_cache_idx_valid; -}; - -struct i40iw_modify_qp_info { - u64 rx_win0; - u64 rx_win1; - u8 next_iwarp_state; - u8 termlen; - bool ord_valid; - bool tcp_ctx_valid; - bool cq_num_valid; - bool arp_cache_idx_valid; - bool reset_tcp_conn; - bool remove_hash_idx; - bool dont_send_term; - bool dont_send_fin; - bool cached_var_valid; - bool force_loopback; -}; - -struct i40iw_ccq_cqe_info { - struct i40iw_sc_cqp *cqp; - u64 scratch; - u32 op_ret_val; - u16 maj_err_code; - u16 min_err_code; - u8 op_code; - bool error; -}; - -struct i40iw_l2params { - u16 qs_handle_list[I40IW_MAX_USER_PRIORITY]; - u16 mtu; -}; - -struct i40iw_vsi_init_info { - struct i40iw_sc_dev *dev; - void *back_vsi; - struct i40iw_l2params *params; - u16 exception_lan_queue; -}; - -struct i40iw_vsi_stats_info { - struct i40iw_vsi_pestat *pestat; - u8 fcn_id; - bool alloc_fcn_id; - bool stats_initialize; -}; - -struct i40iw_device_init_info { - u64 fpm_query_buf_pa; - u64 fpm_commit_buf_pa; - u64 *fpm_query_buf; - u64 *fpm_commit_buf; - struct i40iw_hw *hw; - void __iomem *bar0; - enum i40iw_status_code (*vchnl_send)(struct i40iw_sc_dev *, u32, u8 *, u16); - u8 hmc_fn_id; - bool is_pf; - u32 debug_mask; -}; - -enum i40iw_cqp_hmc_profile { - I40IW_HMC_PROFILE_DEFAULT = 1, - I40IW_HMC_PROFILE_FAVOR_VF = 2, - I40IW_HMC_PROFILE_EQUAL = 3, -}; - -struct i40iw_cqp_init_info { - u64 cqp_compl_ctx; - u64 host_ctx_pa; - u64 sq_pa; - struct i40iw_sc_dev *dev; - struct i40iw_cqp_quanta *sq; - u64 *host_ctx; - u64 *scratch_array; - u32 sq_size; - u8 struct_ver; - bool en_datacenter_tcp; - u8 hmc_profile; - u8 enabled_vf_count; -}; - -struct i40iw_ceq_init_info { - u64 ceqe_pa; - struct i40iw_sc_dev *dev; - u64 *ceqe_base; - void *pbl_list; - u32 elem_cnt; - u32 ceq_id; - bool virtual_map; - u8 pbl_chunk_size; - bool tph_en; - u8 tph_val; - u32 first_pm_pbl_idx; -}; - -struct i40iw_aeq_init_info { - u64 aeq_elem_pa; - struct i40iw_sc_dev *dev; - u32 *aeqe_base; - void *pbl_list; - u32 elem_cnt; - bool virtual_map; - u8 pbl_chunk_size; - u32 first_pm_pbl_idx; -}; - -struct i40iw_ccq_init_info { - u64 cq_pa; - u64 shadow_area_pa; - struct i40iw_sc_dev *dev; - struct i40iw_cqe *cq_base; - u64 *shadow_area; - void *pbl_list; - u32 num_elem; - u32 ceq_id; - u32 shadow_read_threshold; - bool ceqe_mask; - bool ceq_id_valid; - bool tph_en; - u8 tph_val; - bool avoid_mem_cflct; - bool virtual_map; - u8 pbl_chunk_size; - u32 first_pm_pbl_idx; -}; - -struct i40iwarp_offload_info { - u16 rcv_mark_offset; - u16 snd_mark_offset; - u16 pd_id; - u8 ddp_ver; - u8 rdmap_ver; - u8 ord_size; - u8 ird_size; - bool wr_rdresp_en; - bool rd_enable; - bool snd_mark_en; - bool rcv_mark_en; - bool bind_en; - bool fast_reg_en; - bool priv_mode_en; - bool lsmm_present; - u8 iwarp_mode; - bool align_hdrs; - bool rcv_no_mpa_crc; - - u8 last_byte_sent; -}; - -struct i40iw_tcp_offload_info { - bool ipv4; - bool no_nagle; - bool insert_vlan_tag; - bool time_stamp; - u8 cwnd_inc_limit; - bool drop_ooo_seg; - u8 dup_ack_thresh; - u8 ttl; - u8 src_mac_addr_idx; - bool avoid_stretch_ack; - u8 tos; - u16 src_port; - u16 dst_port; - u32 dest_ip_addr0; - u32 dest_ip_addr1; - u32 dest_ip_addr2; - u32 dest_ip_addr3; - u32 snd_mss; - u16 vlan_tag; - u16 arp_idx; - u32 flow_label; - bool wscale; - u8 tcp_state; - u8 snd_wscale; - u8 rcv_wscale; - u32 time_stamp_recent; - u32 time_stamp_age; - u32 snd_nxt; - u32 snd_wnd; - u32 rcv_nxt; - u32 rcv_wnd; - u32 snd_max; - u32 snd_una; - u32 srtt; - u32 rtt_var; - u32 ss_thresh; - u32 cwnd; - u32 snd_wl1; - u32 snd_wl2; - u32 max_snd_window; - u8 rexmit_thresh; - u32 local_ipaddr0; - u32 local_ipaddr1; - u32 local_ipaddr2; - u32 local_ipaddr3; - bool ignore_tcp_opt; - bool ignore_tcp_uns_opt; -}; - -struct i40iw_qp_host_ctx_info { - u64 qp_compl_ctx; - struct i40iw_tcp_offload_info *tcp_info; - struct i40iwarp_offload_info *iwarp_info; - u32 send_cq_num; - u32 rcv_cq_num; - bool tcp_info_valid; - bool iwarp_info_valid; - bool err_rq_idx_valid; - u16 err_rq_idx; - bool add_to_qoslist; - u8 user_pri; -}; - -struct i40iw_aeqe_info { - u64 compl_ctx; - u32 qp_cq_id; - u16 ae_id; - u16 wqe_idx; - u8 tcp_state; - u8 iwarp_state; - bool qp; - bool cq; - bool sq; - bool in_rdrsp_wr; - bool out_rdrsp; - u8 q2_data_written; - bool aeqe_overflow; -}; - -struct i40iw_allocate_stag_info { - u64 total_len; - u32 chunk_size; - u32 stag_idx; - u32 page_size; - u16 pd_id; - u16 access_rights; - bool remote_access; - bool use_hmc_fcn_index; - u8 hmc_fcn_index; - bool use_pf_rid; -}; - -struct i40iw_reg_ns_stag_info { - u64 reg_addr_pa; - u64 fbo; - void *va; - u64 total_len; - u32 page_size; - u32 chunk_size; - u32 first_pm_pbl_index; - enum i40iw_addressing_type addr_type; - i40iw_stag_index stag_idx; - u16 access_rights; - u16 pd_id; - i40iw_stag_key stag_key; - bool use_hmc_fcn_index; - u8 hmc_fcn_index; - bool use_pf_rid; -}; - -struct i40iw_fast_reg_stag_info { - u64 wr_id; - u64 reg_addr_pa; - u64 fbo; - void *va; - u64 total_len; - u32 page_size; - u32 chunk_size; - u32 first_pm_pbl_index; - enum i40iw_addressing_type addr_type; - i40iw_stag_index stag_idx; - u16 access_rights; - u16 pd_id; - i40iw_stag_key stag_key; - bool local_fence; - bool read_fence; - bool signaled; - bool use_hmc_fcn_index; - u8 hmc_fcn_index; - bool use_pf_rid; - bool defer_flag; -}; - -struct i40iw_dealloc_stag_info { - u32 stag_idx; - u16 pd_id; - bool mr; - bool dealloc_pbl; -}; - -struct i40iw_register_shared_stag { - void *va; - enum i40iw_addressing_type addr_type; - i40iw_stag_index new_stag_idx; - i40iw_stag_index parent_stag_idx; - u32 access_rights; - u16 pd_id; - i40iw_stag_key new_stag_key; -}; - -struct i40iw_qp_init_info { - struct i40iw_qp_uk_init_info qp_uk_init_info; - struct i40iw_sc_pd *pd; - struct i40iw_sc_vsi *vsi; - u64 *host_ctx; - u8 *q2; - u64 sq_pa; - u64 rq_pa; - u64 host_ctx_pa; - u64 q2_pa; - u64 shadow_area_pa; - int abi_ver; - u8 sq_tph_val; - u8 rq_tph_val; - u8 type; - bool sq_tph_en; - bool rq_tph_en; - bool rcv_tph_en; - bool xmit_tph_en; - bool virtual_map; -}; - -struct i40iw_cq_init_info { - struct i40iw_sc_dev *dev; - u64 cq_base_pa; - u64 shadow_area_pa; - u32 ceq_id; - u32 shadow_read_threshold; - bool virtual_map; - bool ceqe_mask; - u8 pbl_chunk_size; - u32 first_pm_pbl_idx; - bool ceq_id_valid; - bool tph_en; - u8 tph_val; - u8 type; - struct i40iw_cq_uk_init_info cq_uk_init_info; -}; - -struct i40iw_upload_context_info { - u64 buf_pa; - bool freeze_qp; - bool raw_format; - u32 qp_id; - u8 qp_type; -}; - -struct i40iw_add_arp_cache_entry_info { - u8 mac_addr[6]; - u32 reach_max; - u16 arp_index; - bool permanent; -}; - -struct i40iw_apbvt_info { - u16 port; - bool add; -}; - -enum i40iw_quad_entry_type { - I40IW_QHASH_TYPE_TCP_ESTABLISHED = 1, - I40IW_QHASH_TYPE_TCP_SYN, -}; - -enum i40iw_quad_hash_manage_type { - I40IW_QHASH_MANAGE_TYPE_DELETE = 0, - I40IW_QHASH_MANAGE_TYPE_ADD, - I40IW_QHASH_MANAGE_TYPE_MODIFY -}; - -struct i40iw_qhash_table_info { - struct i40iw_sc_vsi *vsi; - enum i40iw_quad_hash_manage_type manage; - enum i40iw_quad_entry_type entry_type; - bool vlan_valid; - bool ipv4_valid; - u8 mac_addr[6]; - u16 vlan_id; - u8 user_pri; - u32 qp_num; - u32 dest_ip[4]; - u32 src_ip[4]; - u16 dest_port; - u16 src_port; -}; - -struct i40iw_local_mac_ipaddr_entry_info { - u8 mac_addr[6]; - u8 entry_idx; -}; - -struct i40iw_qp_flush_info { - u16 sq_minor_code; - u16 sq_major_code; - u16 rq_minor_code; - u16 rq_major_code; - u16 ae_code; - u8 ae_source; - bool sq; - bool rq; - bool userflushcode; - bool generate_ae; -}; - -struct i40iw_cqp_commit_fpm_values { - u64 qp_base; - u64 cq_base; - u32 hte_base; - u32 arp_base; - u32 apbvt_inuse_base; - u32 mr_base; - u32 xf_base; - u32 xffl_base; - u32 q1_base; - u32 q1fl_base; - u32 fsimc_base; - u32 fsiav_base; - u32 pbl_base; - - u32 qp_cnt; - u32 cq_cnt; - u32 hte_cnt; - u32 arp_cnt; - u32 mr_cnt; - u32 xf_cnt; - u32 xffl_cnt; - u32 q1_cnt; - u32 q1fl_cnt; - u32 fsimc_cnt; - u32 fsiav_cnt; - u32 pbl_cnt; -}; - -struct i40iw_cqp_query_fpm_values { - u16 first_pe_sd_index; - u32 qp_objsize; - u32 cq_objsize; - u32 hte_objsize; - u32 arp_objsize; - u32 mr_objsize; - u32 xf_objsize; - u32 q1_objsize; - u32 fsimc_objsize; - u32 fsiav_objsize; - - u32 qp_max; - u32 cq_max; - u32 hte_max; - u32 arp_max; - u32 mr_max; - u32 xf_max; - u32 xffl_max; - u32 q1_max; - u32 q1fl_max; - u32 fsimc_max; - u32 fsiav_max; - u32 pbl_max; -}; - -struct i40iw_gen_ae_info { - u16 ae_code; - u8 ae_source; -}; - -struct i40iw_cqp_ops { - enum i40iw_status_code (*cqp_init)(struct i40iw_sc_cqp *, - struct i40iw_cqp_init_info *); - enum i40iw_status_code (*cqp_create)(struct i40iw_sc_cqp *, u16 *, u16 *); - void (*cqp_post_sq)(struct i40iw_sc_cqp *); - u64 *(*cqp_get_next_send_wqe)(struct i40iw_sc_cqp *, u64 scratch); - enum i40iw_status_code (*cqp_destroy)(struct i40iw_sc_cqp *); - enum i40iw_status_code (*poll_for_cqp_op_done)(struct i40iw_sc_cqp *, u8, - struct i40iw_ccq_cqe_info *); -}; - -struct i40iw_ccq_ops { - enum i40iw_status_code (*ccq_init)(struct i40iw_sc_cq *, - struct i40iw_ccq_init_info *); - enum i40iw_status_code (*ccq_create)(struct i40iw_sc_cq *, u64, bool, bool); - enum i40iw_status_code (*ccq_destroy)(struct i40iw_sc_cq *, u64, bool); - enum i40iw_status_code (*ccq_create_done)(struct i40iw_sc_cq *); - enum i40iw_status_code (*ccq_get_cqe_info)(struct i40iw_sc_cq *, - struct i40iw_ccq_cqe_info *); - void (*ccq_arm)(struct i40iw_sc_cq *); -}; - -struct i40iw_ceq_ops { - enum i40iw_status_code (*ceq_init)(struct i40iw_sc_ceq *, - struct i40iw_ceq_init_info *); - enum i40iw_status_code (*ceq_create)(struct i40iw_sc_ceq *, u64, bool); - enum i40iw_status_code (*cceq_create_done)(struct i40iw_sc_ceq *); - enum i40iw_status_code (*cceq_destroy_done)(struct i40iw_sc_ceq *); - enum i40iw_status_code (*cceq_create)(struct i40iw_sc_ceq *, u64); - enum i40iw_status_code (*ceq_destroy)(struct i40iw_sc_ceq *, u64, bool); - void *(*process_ceq)(struct i40iw_sc_dev *, struct i40iw_sc_ceq *); -}; - -struct i40iw_aeq_ops { - enum i40iw_status_code (*aeq_init)(struct i40iw_sc_aeq *, - struct i40iw_aeq_init_info *); - enum i40iw_status_code (*aeq_create)(struct i40iw_sc_aeq *, u64, bool); - enum i40iw_status_code (*aeq_destroy)(struct i40iw_sc_aeq *, u64, bool); - enum i40iw_status_code (*get_next_aeqe)(struct i40iw_sc_aeq *, - struct i40iw_aeqe_info *); - enum i40iw_status_code (*repost_aeq_entries)(struct i40iw_sc_dev *, u32); - enum i40iw_status_code (*aeq_create_done)(struct i40iw_sc_aeq *); - enum i40iw_status_code (*aeq_destroy_done)(struct i40iw_sc_aeq *); -}; - -struct i40iw_pd_ops { - void (*pd_init)(struct i40iw_sc_dev *, struct i40iw_sc_pd *, u16, int); -}; - -struct i40iw_priv_qp_ops { - enum i40iw_status_code (*qp_init)(struct i40iw_sc_qp *, struct i40iw_qp_init_info *); - enum i40iw_status_code (*qp_create)(struct i40iw_sc_qp *, - struct i40iw_create_qp_info *, u64, bool); - enum i40iw_status_code (*qp_modify)(struct i40iw_sc_qp *, - struct i40iw_modify_qp_info *, u64, bool); - enum i40iw_status_code (*qp_destroy)(struct i40iw_sc_qp *, u64, bool, bool, bool); - enum i40iw_status_code (*qp_flush_wqes)(struct i40iw_sc_qp *, - struct i40iw_qp_flush_info *, u64, bool); - enum i40iw_status_code (*qp_upload_context)(struct i40iw_sc_dev *, - struct i40iw_upload_context_info *, - u64, bool); - enum i40iw_status_code (*qp_setctx)(struct i40iw_sc_qp *, u64 *, - struct i40iw_qp_host_ctx_info *); - - void (*qp_send_lsmm)(struct i40iw_sc_qp *, void *, u32, i40iw_stag); - void (*qp_send_lsmm_nostag)(struct i40iw_sc_qp *, void *, u32); - void (*qp_send_rtt)(struct i40iw_sc_qp *, bool); - enum i40iw_status_code (*qp_post_wqe0)(struct i40iw_sc_qp *, u8); - enum i40iw_status_code (*iw_mr_fast_register)(struct i40iw_sc_qp *, - struct i40iw_fast_reg_stag_info *, - bool); -}; - -struct i40iw_priv_cq_ops { - enum i40iw_status_code (*cq_init)(struct i40iw_sc_cq *, struct i40iw_cq_init_info *); - enum i40iw_status_code (*cq_create)(struct i40iw_sc_cq *, u64, bool, bool); - enum i40iw_status_code (*cq_destroy)(struct i40iw_sc_cq *, u64, bool); - enum i40iw_status_code (*cq_modify)(struct i40iw_sc_cq *, - struct i40iw_modify_cq_info *, u64, bool); -}; - -struct i40iw_mr_ops { - enum i40iw_status_code (*alloc_stag)(struct i40iw_sc_dev *, - struct i40iw_allocate_stag_info *, u64, bool); - enum i40iw_status_code (*mr_reg_non_shared)(struct i40iw_sc_dev *, - struct i40iw_reg_ns_stag_info *, - u64, bool); - enum i40iw_status_code (*mr_reg_shared)(struct i40iw_sc_dev *, - struct i40iw_register_shared_stag *, - u64, bool); - enum i40iw_status_code (*dealloc_stag)(struct i40iw_sc_dev *, - struct i40iw_dealloc_stag_info *, - u64, bool); - enum i40iw_status_code (*query_stag)(struct i40iw_sc_dev *, u64, u32, bool); - enum i40iw_status_code (*mw_alloc)(struct i40iw_sc_dev *, u64, u32, u16, bool); -}; - -struct i40iw_cqp_misc_ops { - enum i40iw_status_code (*manage_hmc_pm_func_table)(struct i40iw_sc_cqp *, - u64, u8, bool, bool); - enum i40iw_status_code (*set_hmc_resource_profile)(struct i40iw_sc_cqp *, - u64, u8, u8, bool, bool); - enum i40iw_status_code (*commit_fpm_values)(struct i40iw_sc_cqp *, u64, u8, - struct i40iw_dma_mem *, bool, u8); - enum i40iw_status_code (*query_fpm_values)(struct i40iw_sc_cqp *, u64, u8, - struct i40iw_dma_mem *, bool, u8); - enum i40iw_status_code (*static_hmc_pages_allocated)(struct i40iw_sc_cqp *, - u64, u8, bool, bool); - enum i40iw_status_code (*add_arp_cache_entry)(struct i40iw_sc_cqp *, - struct i40iw_add_arp_cache_entry_info *, - u64, bool); - enum i40iw_status_code (*del_arp_cache_entry)(struct i40iw_sc_cqp *, u64, u16, bool); - enum i40iw_status_code (*query_arp_cache_entry)(struct i40iw_sc_cqp *, u64, u16, bool); - enum i40iw_status_code (*manage_apbvt_entry)(struct i40iw_sc_cqp *, - struct i40iw_apbvt_info *, u64, bool); - enum i40iw_status_code (*manage_qhash_table_entry)(struct i40iw_sc_cqp *, - struct i40iw_qhash_table_info *, u64, bool); - enum i40iw_status_code (*alloc_local_mac_ipaddr_table_entry)(struct i40iw_sc_cqp *, u64, bool); - enum i40iw_status_code (*add_local_mac_ipaddr_entry)(struct i40iw_sc_cqp *, - struct i40iw_local_mac_ipaddr_entry_info *, - u64, bool); - enum i40iw_status_code (*del_local_mac_ipaddr_entry)(struct i40iw_sc_cqp *, u64, u8, u8, bool); - enum i40iw_status_code (*cqp_nop)(struct i40iw_sc_cqp *, u64, bool); - enum i40iw_status_code (*commit_fpm_values_done)(struct i40iw_sc_cqp - *); - enum i40iw_status_code (*query_fpm_values_done)(struct i40iw_sc_cqp *); - enum i40iw_status_code (*manage_hmc_pm_func_table_done)(struct i40iw_sc_cqp *); - enum i40iw_status_code (*update_suspend_qp)(struct i40iw_sc_cqp *, struct i40iw_sc_qp *, u64); - enum i40iw_status_code (*update_resume_qp)(struct i40iw_sc_cqp *, struct i40iw_sc_qp *, u64); -}; - -struct i40iw_hmc_ops { - enum i40iw_status_code (*init_iw_hmc)(struct i40iw_sc_dev *, u8); - enum i40iw_status_code (*parse_fpm_query_buf)(u64 *, struct i40iw_hmc_info *, - struct i40iw_hmc_fpm_misc *); - enum i40iw_status_code (*configure_iw_fpm)(struct i40iw_sc_dev *, u8); - enum i40iw_status_code (*parse_fpm_commit_buf)(u64 *, struct i40iw_hmc_obj_info *, u32 *sd); - enum i40iw_status_code (*create_hmc_object)(struct i40iw_sc_dev *dev, - struct i40iw_hmc_create_obj_info *); - enum i40iw_status_code (*del_hmc_object)(struct i40iw_sc_dev *dev, - struct i40iw_hmc_del_obj_info *, - bool reset); - enum i40iw_status_code (*pf_init_vfhmc)(struct i40iw_sc_dev *, u8, u32 *); - enum i40iw_status_code (*vf_configure_vffpm)(struct i40iw_sc_dev *, u32 *); -}; - -struct cqp_info { - union { - struct { - struct i40iw_sc_qp *qp; - struct i40iw_create_qp_info info; - u64 scratch; - } qp_create; - - struct { - struct i40iw_sc_qp *qp; - struct i40iw_modify_qp_info info; - u64 scratch; - } qp_modify; - - struct { - struct i40iw_sc_qp *qp; - u64 scratch; - bool remove_hash_idx; - bool ignore_mw_bnd; - } qp_destroy; - - struct { - struct i40iw_sc_cq *cq; - u64 scratch; - bool check_overflow; - } cq_create; - - struct { - struct i40iw_sc_cq *cq; - u64 scratch; - } cq_destroy; - - struct { - struct i40iw_sc_dev *dev; - struct i40iw_allocate_stag_info info; - u64 scratch; - } alloc_stag; - - struct { - struct i40iw_sc_dev *dev; - u64 scratch; - u32 mw_stag_index; - u16 pd_id; - } mw_alloc; - - struct { - struct i40iw_sc_dev *dev; - struct i40iw_reg_ns_stag_info info; - u64 scratch; - } mr_reg_non_shared; - - struct { - struct i40iw_sc_dev *dev; - struct i40iw_dealloc_stag_info info; - u64 scratch; - } dealloc_stag; - - struct { - struct i40iw_sc_cqp *cqp; - struct i40iw_local_mac_ipaddr_entry_info info; - u64 scratch; - } add_local_mac_ipaddr_entry; - - struct { - struct i40iw_sc_cqp *cqp; - struct i40iw_add_arp_cache_entry_info info; - u64 scratch; - } add_arp_cache_entry; - - struct { - struct i40iw_sc_cqp *cqp; - u64 scratch; - u8 entry_idx; - u8 ignore_ref_count; - } del_local_mac_ipaddr_entry; - - struct { - struct i40iw_sc_cqp *cqp; - u64 scratch; - u16 arp_index; - } del_arp_cache_entry; - - struct { - struct i40iw_sc_cqp *cqp; - struct i40iw_manage_vf_pble_info info; - u64 scratch; - } manage_vf_pble_bp; - - struct { - struct i40iw_sc_dev *dev; - struct i40iw_upload_context_info info; - u64 scratch; - } qp_upload_context; - - struct { - struct i40iw_sc_cqp *cqp; - u64 scratch; - } alloc_local_mac_ipaddr_entry; - - struct { - struct i40iw_sc_dev *dev; - struct i40iw_hmc_fcn_info info; - u64 scratch; - } manage_hmc_pm; - - struct { - struct i40iw_sc_ceq *ceq; - u64 scratch; - } ceq_create; - - struct { - struct i40iw_sc_ceq *ceq; - u64 scratch; - } ceq_destroy; - - struct { - struct i40iw_sc_aeq *aeq; - u64 scratch; - } aeq_create; - - struct { - struct i40iw_sc_aeq *aeq; - u64 scratch; - } aeq_destroy; - - struct { - struct i40iw_sc_qp *qp; - struct i40iw_qp_flush_info info; - u64 scratch; - } qp_flush_wqes; - - struct { - struct i40iw_sc_qp *qp; - struct i40iw_gen_ae_info info; - u64 scratch; - } gen_ae; - - struct { - struct i40iw_sc_cqp *cqp; - void *fpm_values_va; - u64 fpm_values_pa; - u8 hmc_fn_id; - u64 scratch; - } query_fpm_values; - - struct { - struct i40iw_sc_cqp *cqp; - void *fpm_values_va; - u64 fpm_values_pa; - u8 hmc_fn_id; - u64 scratch; - } commit_fpm_values; - - struct { - struct i40iw_sc_cqp *cqp; - struct i40iw_apbvt_info info; - u64 scratch; - } manage_apbvt_entry; - - struct { - struct i40iw_sc_cqp *cqp; - struct i40iw_qhash_table_info info; - u64 scratch; - } manage_qhash_table_entry; - - struct { - struct i40iw_sc_dev *dev; - struct i40iw_update_sds_info info; - u64 scratch; - } update_pe_sds; - - struct { - struct i40iw_sc_cqp *cqp; - struct i40iw_sc_qp *qp; - u64 scratch; - } suspend_resume; - struct { - struct i40iw_sc_cqp *cqp; - void *cap_va; - u64 cap_pa; - u64 scratch; - } query_rdma_features; - } u; -}; - -struct cqp_commands_info { - struct list_head cqp_cmd_entry; - u8 cqp_cmd; - u8 post_sq; - struct cqp_info in; -}; - -struct i40iw_virtchnl_work_info { - void (*callback_fcn)(void *vf_dev); - void *worker_vf_dev; -}; - -struct i40iw_cqp_timeout { - u64 compl_cqp_cmds; - u8 count; -}; - -#endif diff --git a/drivers/infiniband/hw/i40iw/i40iw_uk.c b/drivers/infiniband/hw/i40iw/i40iw_uk.c deleted file mode 100644 index f521be16bf31..000000000000 --- a/drivers/infiniband/hw/i40iw/i40iw_uk.c +++ /dev/null @@ -1,1200 +0,0 @@ -/******************************************************************************* -* -* Copyright (c) 2015-2016 Intel Corporation. All rights reserved. -* -* This software is available to you under a choice of one of two -* licenses. You may choose to be licensed under the terms of the GNU -* General Public License (GPL) Version 2, available from the file -* COPYING in the main directory of this source tree, or the -* OpenFabrics.org BSD license below: -* -* Redistribution and use in source and binary forms, with or -* without modification, are permitted provided that the following -* conditions are met: -* -* - Redistributions of source code must retain the above -* copyright notice, this list of conditions and the following -* disclaimer. -* -* - Redistributions in binary form must reproduce the above -* copyright notice, this list of conditions and the following -* disclaimer in the documentation and/or other materials -* provided with the distribution. -* -* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS -* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN -* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -* SOFTWARE. -* -*******************************************************************************/ - -#include "i40iw_osdep.h" -#include "i40iw_status.h" -#include "i40iw_d.h" -#include "i40iw_user.h" -#include "i40iw_register.h" - -static u32 nop_signature = 0x55550000; - -/** - * i40iw_nop_1 - insert a nop wqe and move head. no post work - * @qp: hw qp ptr - */ -static enum i40iw_status_code i40iw_nop_1(struct i40iw_qp_uk *qp) -{ - u64 header, *wqe; - u64 *wqe_0 = NULL; - u32 wqe_idx, peek_head; - bool signaled = false; - - if (!qp->sq_ring.head) - return I40IW_ERR_PARAM; - - wqe_idx = I40IW_RING_GETCURRENT_HEAD(qp->sq_ring); - wqe = qp->sq_base[wqe_idx].elem; - - qp->sq_wrtrk_array[wqe_idx].wqe_size = I40IW_QP_WQE_MIN_SIZE; - - peek_head = (qp->sq_ring.head + 1) % qp->sq_ring.size; - wqe_0 = qp->sq_base[peek_head].elem; - if (peek_head) - wqe_0[3] = LS_64(!qp->swqe_polarity, I40IWQPSQ_VALID); - else - wqe_0[3] = LS_64(qp->swqe_polarity, I40IWQPSQ_VALID); - - set_64bit_val(wqe, 0, 0); - set_64bit_val(wqe, 8, 0); - set_64bit_val(wqe, 16, 0); - - header = LS_64(I40IWQP_OP_NOP, I40IWQPSQ_OPCODE) | - LS_64(signaled, I40IWQPSQ_SIGCOMPL) | - LS_64(qp->swqe_polarity, I40IWQPSQ_VALID) | nop_signature++; - - wmb(); /* Memory barrier to ensure data is written before valid bit is set */ - - set_64bit_val(wqe, 24, header); - return 0; -} - -/** - * i40iw_qp_post_wr - post wr to hrdware - * @qp: hw qp ptr - */ -void i40iw_qp_post_wr(struct i40iw_qp_uk *qp) -{ - u64 temp; - u32 hw_sq_tail; - u32 sw_sq_head; - - mb(); /* valid bit is written and loads completed before reading shadow */ - - /* read the doorbell shadow area */ - get_64bit_val(qp->shadow_area, 0, &temp); - - hw_sq_tail = (u32)RS_64(temp, I40IW_QP_DBSA_HW_SQ_TAIL); - sw_sq_head = I40IW_RING_GETCURRENT_HEAD(qp->sq_ring); - if (sw_sq_head != hw_sq_tail) { - if (sw_sq_head > qp->initial_ring.head) { - if ((hw_sq_tail >= qp->initial_ring.head) && - (hw_sq_tail < sw_sq_head)) { - writel(qp->qp_id, qp->wqe_alloc_reg); - } - } else if (sw_sq_head != qp->initial_ring.head) { - if ((hw_sq_tail >= qp->initial_ring.head) || - (hw_sq_tail < sw_sq_head)) { - writel(qp->qp_id, qp->wqe_alloc_reg); - } - } - } - - qp->initial_ring.head = qp->sq_ring.head; -} - -/** - * i40iw_qp_get_next_send_wqe - return next wqe ptr - * @qp: hw qp ptr - * @wqe_idx: return wqe index - * @wqe_size: size of sq wqe - * @total_size: work request length - * @wr_id: work request id - */ -u64 *i40iw_qp_get_next_send_wqe(struct i40iw_qp_uk *qp, - u32 *wqe_idx, - u8 wqe_size, - u32 total_size, - u64 wr_id - ) -{ - u64 *wqe = NULL; - u64 wqe_ptr; - u32 peek_head = 0; - u16 offset; - enum i40iw_status_code ret_code = 0; - u8 nop_wqe_cnt = 0, i; - u64 *wqe_0 = NULL; - - *wqe_idx = I40IW_RING_GETCURRENT_HEAD(qp->sq_ring); - - if (!*wqe_idx) - qp->swqe_polarity = !qp->swqe_polarity; - wqe_ptr = (uintptr_t)qp->sq_base[*wqe_idx].elem; - offset = (u16)(wqe_ptr) & 0x7F; - if ((offset + wqe_size) > I40IW_QP_WQE_MAX_SIZE) { - nop_wqe_cnt = (u8)(I40IW_QP_WQE_MAX_SIZE - offset) / I40IW_QP_WQE_MIN_SIZE; - for (i = 0; i < nop_wqe_cnt; i++) { - i40iw_nop_1(qp); - I40IW_RING_MOVE_HEAD(qp->sq_ring, ret_code); - if (ret_code) - return NULL; - } - - *wqe_idx = I40IW_RING_GETCURRENT_HEAD(qp->sq_ring); - if (!*wqe_idx) - qp->swqe_polarity = !qp->swqe_polarity; - } - - if (((*wqe_idx & 3) == 1) && (wqe_size == I40IW_WQE_SIZE_64)) { - i40iw_nop_1(qp); - I40IW_RING_MOVE_HEAD(qp->sq_ring, ret_code); - if (ret_code) - return NULL; - *wqe_idx = I40IW_RING_GETCURRENT_HEAD(qp->sq_ring); - if (!*wqe_idx) - qp->swqe_polarity = !qp->swqe_polarity; - } - I40IW_RING_MOVE_HEAD_BY_COUNT(qp->sq_ring, - wqe_size / I40IW_QP_WQE_MIN_SIZE, ret_code); - if (ret_code) - return NULL; - - wqe = qp->sq_base[*wqe_idx].elem; - - peek_head = I40IW_RING_GETCURRENT_HEAD(qp->sq_ring); - wqe_0 = qp->sq_base[peek_head].elem; - - if (((peek_head & 3) == 1) || ((peek_head & 3) == 3)) { - if (RS_64(wqe_0[3], I40IWQPSQ_VALID) != !qp->swqe_polarity) - wqe_0[3] = LS_64(!qp->swqe_polarity, I40IWQPSQ_VALID); - } - - qp->sq_wrtrk_array[*wqe_idx].wrid = wr_id; - qp->sq_wrtrk_array[*wqe_idx].wr_len = total_size; - qp->sq_wrtrk_array[*wqe_idx].wqe_size = wqe_size; - return wqe; -} - -/** - * i40iw_set_fragment - set fragment in wqe - * @wqe: wqe for setting fragment - * @offset: offset value - * @sge: sge length and stag - */ -static void i40iw_set_fragment(u64 *wqe, u32 offset, struct i40iw_sge *sge) -{ - if (sge) { - set_64bit_val(wqe, offset, LS_64(sge->tag_off, I40IWQPSQ_FRAG_TO)); - set_64bit_val(wqe, (offset + 8), - (LS_64(sge->len, I40IWQPSQ_FRAG_LEN) | - LS_64(sge->stag, I40IWQPSQ_FRAG_STAG))); - } -} - -/** - * i40iw_qp_get_next_recv_wqe - get next qp's rcv wqe - * @qp: hw qp ptr - * @wqe_idx: return wqe index - */ -u64 *i40iw_qp_get_next_recv_wqe(struct i40iw_qp_uk *qp, u32 *wqe_idx) -{ - u64 *wqe = NULL; - enum i40iw_status_code ret_code; - - if (I40IW_RING_FULL_ERR(qp->rq_ring)) - return NULL; - - I40IW_ATOMIC_RING_MOVE_HEAD(qp->rq_ring, *wqe_idx, ret_code); - if (ret_code) - return NULL; - if (!*wqe_idx) - qp->rwqe_polarity = !qp->rwqe_polarity; - /* rq_wqe_size_multiplier is no of qwords in one rq wqe */ - wqe = qp->rq_base[*wqe_idx * (qp->rq_wqe_size_multiplier >> 2)].elem; - - return wqe; -} - -/** - * i40iw_rdma_write - rdma write operation - * @qp: hw qp ptr - * @info: post sq information - * @post_sq: flag to post sq - */ -static enum i40iw_status_code i40iw_rdma_write(struct i40iw_qp_uk *qp, - struct i40iw_post_sq_info *info, - bool post_sq) -{ - u64 header; - u64 *wqe; - struct i40iw_rdma_write *op_info; - u32 i, wqe_idx; - u32 total_size = 0, byte_off; - enum i40iw_status_code ret_code; - bool read_fence = false; - u8 wqe_size; - - op_info = &info->op.rdma_write; - if (op_info->num_lo_sges > qp->max_sq_frag_cnt) - return I40IW_ERR_INVALID_FRAG_COUNT; - - for (i = 0; i < op_info->num_lo_sges; i++) - total_size += op_info->lo_sg_list[i].len; - - if (total_size > I40IW_MAX_OUTBOUND_MESSAGE_SIZE) - return I40IW_ERR_QP_INVALID_MSG_SIZE; - - read_fence |= info->read_fence; - - ret_code = i40iw_fragcnt_to_wqesize_sq(op_info->num_lo_sges, &wqe_size); - if (ret_code) - return ret_code; - - wqe = i40iw_qp_get_next_send_wqe(qp, &wqe_idx, wqe_size, total_size, info->wr_id); - if (!wqe) - return I40IW_ERR_QP_TOOMANY_WRS_POSTED; - set_64bit_val(wqe, 16, - LS_64(op_info->rem_addr.tag_off, I40IWQPSQ_FRAG_TO)); - if (!op_info->rem_addr.stag) - return I40IW_ERR_BAD_STAG; - - header = LS_64(op_info->rem_addr.stag, I40IWQPSQ_REMSTAG) | - LS_64(I40IWQP_OP_RDMA_WRITE, I40IWQPSQ_OPCODE) | - LS_64((op_info->num_lo_sges > 1 ? (op_info->num_lo_sges - 1) : 0), I40IWQPSQ_ADDFRAGCNT) | - LS_64(read_fence, I40IWQPSQ_READFENCE) | - LS_64(info->local_fence, I40IWQPSQ_LOCALFENCE) | - LS_64(info->signaled, I40IWQPSQ_SIGCOMPL) | - LS_64(qp->swqe_polarity, I40IWQPSQ_VALID); - - i40iw_set_fragment(wqe, 0, op_info->lo_sg_list); - - for (i = 1, byte_off = 32; i < op_info->num_lo_sges; i++) { - i40iw_set_fragment(wqe, byte_off, &op_info->lo_sg_list[i]); - byte_off += 16; - } - - wmb(); /* make sure WQE is populated before valid bit is set */ - - set_64bit_val(wqe, 24, header); - - if (post_sq) - i40iw_qp_post_wr(qp); - - return 0; -} - -/** - * i40iw_rdma_read - rdma read command - * @qp: hw qp ptr - * @info: post sq information - * @inv_stag: flag for inv_stag - * @post_sq: flag to post sq - */ -static enum i40iw_status_code i40iw_rdma_read(struct i40iw_qp_uk *qp, - struct i40iw_post_sq_info *info, - bool inv_stag, - bool post_sq) -{ - u64 *wqe; - struct i40iw_rdma_read *op_info; - u64 header; - u32 wqe_idx; - enum i40iw_status_code ret_code; - u8 wqe_size; - bool local_fence = false; - - op_info = &info->op.rdma_read; - ret_code = i40iw_fragcnt_to_wqesize_sq(1, &wqe_size); - if (ret_code) - return ret_code; - wqe = i40iw_qp_get_next_send_wqe(qp, &wqe_idx, wqe_size, op_info->lo_addr.len, info->wr_id); - if (!wqe) - return I40IW_ERR_QP_TOOMANY_WRS_POSTED; - local_fence |= info->local_fence; - - set_64bit_val(wqe, 16, LS_64(op_info->rem_addr.tag_off, I40IWQPSQ_FRAG_TO)); - header = LS_64(op_info->rem_addr.stag, I40IWQPSQ_REMSTAG) | - LS_64((inv_stag ? I40IWQP_OP_RDMA_READ_LOC_INV : I40IWQP_OP_RDMA_READ), I40IWQPSQ_OPCODE) | - LS_64(info->read_fence, I40IWQPSQ_READFENCE) | - LS_64(local_fence, I40IWQPSQ_LOCALFENCE) | - LS_64(info->signaled, I40IWQPSQ_SIGCOMPL) | - LS_64(qp->swqe_polarity, I40IWQPSQ_VALID); - - i40iw_set_fragment(wqe, 0, &op_info->lo_addr); - - wmb(); /* make sure WQE is populated before valid bit is set */ - - set_64bit_val(wqe, 24, header); - if (post_sq) - i40iw_qp_post_wr(qp); - - return 0; -} - -/** - * i40iw_send - rdma send command - * @qp: hw qp ptr - * @info: post sq information - * @stag_to_inv: stag_to_inv value - * @post_sq: flag to post sq - */ -static enum i40iw_status_code i40iw_send(struct i40iw_qp_uk *qp, - struct i40iw_post_sq_info *info, - u32 stag_to_inv, - bool post_sq) -{ - u64 *wqe; - struct i40iw_post_send *op_info; - u64 header; - u32 i, wqe_idx, total_size = 0, byte_off; - enum i40iw_status_code ret_code; - bool read_fence = false; - u8 wqe_size; - - op_info = &info->op.send; - if (qp->max_sq_frag_cnt < op_info->num_sges) - return I40IW_ERR_INVALID_FRAG_COUNT; - - for (i = 0; i < op_info->num_sges; i++) - total_size += op_info->sg_list[i].len; - ret_code = i40iw_fragcnt_to_wqesize_sq(op_info->num_sges, &wqe_size); - if (ret_code) - return ret_code; - - wqe = i40iw_qp_get_next_send_wqe(qp, &wqe_idx, wqe_size, total_size, info->wr_id); - if (!wqe) - return I40IW_ERR_QP_TOOMANY_WRS_POSTED; - - read_fence |= info->read_fence; - set_64bit_val(wqe, 16, 0); - header = LS_64(stag_to_inv, I40IWQPSQ_REMSTAG) | - LS_64(info->op_type, I40IWQPSQ_OPCODE) | - LS_64((op_info->num_sges > 1 ? (op_info->num_sges - 1) : 0), - I40IWQPSQ_ADDFRAGCNT) | - LS_64(read_fence, I40IWQPSQ_READFENCE) | - LS_64(info->local_fence, I40IWQPSQ_LOCALFENCE) | - LS_64(info->signaled, I40IWQPSQ_SIGCOMPL) | - LS_64(qp->swqe_polarity, I40IWQPSQ_VALID); - - i40iw_set_fragment(wqe, 0, op_info->sg_list); - - for (i = 1, byte_off = 32; i < op_info->num_sges; i++) { - i40iw_set_fragment(wqe, byte_off, &op_info->sg_list[i]); - byte_off += 16; - } - - wmb(); /* make sure WQE is populated before valid bit is set */ - - set_64bit_val(wqe, 24, header); - if (post_sq) - i40iw_qp_post_wr(qp); - - return 0; -} - -/** - * i40iw_inline_rdma_write - inline rdma write operation - * @qp: hw qp ptr - * @info: post sq information - * @post_sq: flag to post sq - */ -static enum i40iw_status_code i40iw_inline_rdma_write(struct i40iw_qp_uk *qp, - struct i40iw_post_sq_info *info, - bool post_sq) -{ - u64 *wqe; - u8 *dest, *src; - struct i40iw_inline_rdma_write *op_info; - u64 header = 0; - u32 wqe_idx; - enum i40iw_status_code ret_code; - bool read_fence = false; - u8 wqe_size; - - op_info = &info->op.inline_rdma_write; - if (op_info->len > I40IW_MAX_INLINE_DATA_SIZE) - return I40IW_ERR_INVALID_INLINE_DATA_SIZE; - - ret_code = i40iw_inline_data_size_to_wqesize(op_info->len, &wqe_size); - if (ret_code) - return ret_code; - - wqe = i40iw_qp_get_next_send_wqe(qp, &wqe_idx, wqe_size, op_info->len, info->wr_id); - if (!wqe) - return I40IW_ERR_QP_TOOMANY_WRS_POSTED; - - read_fence |= info->read_fence; - set_64bit_val(wqe, 16, - LS_64(op_info->rem_addr.tag_off, I40IWQPSQ_FRAG_TO)); - - header = LS_64(op_info->rem_addr.stag, I40IWQPSQ_REMSTAG) | - LS_64(I40IWQP_OP_RDMA_WRITE, I40IWQPSQ_OPCODE) | - LS_64(op_info->len, I40IWQPSQ_INLINEDATALEN) | - LS_64(1, I40IWQPSQ_INLINEDATAFLAG) | - LS_64(read_fence, I40IWQPSQ_READFENCE) | - LS_64(info->local_fence, I40IWQPSQ_LOCALFENCE) | - LS_64(info->signaled, I40IWQPSQ_SIGCOMPL) | - LS_64(qp->swqe_polarity, I40IWQPSQ_VALID); - - dest = (u8 *)wqe; - src = (u8 *)(op_info->data); - - if (op_info->len <= 16) { - memcpy(dest, src, op_info->len); - } else { - memcpy(dest, src, 16); - src += 16; - dest = (u8 *)wqe + 32; - memcpy(dest, src, op_info->len - 16); - } - - wmb(); /* make sure WQE is populated before valid bit is set */ - - set_64bit_val(wqe, 24, header); - - if (post_sq) - i40iw_qp_post_wr(qp); - - return 0; -} - -/** - * i40iw_inline_send - inline send operation - * @qp: hw qp ptr - * @info: post sq information - * @stag_to_inv: remote stag - * @post_sq: flag to post sq - */ -static enum i40iw_status_code i40iw_inline_send(struct i40iw_qp_uk *qp, - struct i40iw_post_sq_info *info, - u32 stag_to_inv, - bool post_sq) -{ - u64 *wqe; - u8 *dest, *src; - struct i40iw_post_inline_send *op_info; - u64 header; - u32 wqe_idx; - enum i40iw_status_code ret_code; - bool read_fence = false; - u8 wqe_size; - - op_info = &info->op.inline_send; - if (op_info->len > I40IW_MAX_INLINE_DATA_SIZE) - return I40IW_ERR_INVALID_INLINE_DATA_SIZE; - - ret_code = i40iw_inline_data_size_to_wqesize(op_info->len, &wqe_size); - if (ret_code) - return ret_code; - - wqe = i40iw_qp_get_next_send_wqe(qp, &wqe_idx, wqe_size, op_info->len, info->wr_id); - if (!wqe) - return I40IW_ERR_QP_TOOMANY_WRS_POSTED; - - read_fence |= info->read_fence; - header = LS_64(stag_to_inv, I40IWQPSQ_REMSTAG) | - LS_64(info->op_type, I40IWQPSQ_OPCODE) | - LS_64(op_info->len, I40IWQPSQ_INLINEDATALEN) | - LS_64(1, I40IWQPSQ_INLINEDATAFLAG) | - LS_64(read_fence, I40IWQPSQ_READFENCE) | - LS_64(info->local_fence, I40IWQPSQ_LOCALFENCE) | - LS_64(info->signaled, I40IWQPSQ_SIGCOMPL) | - LS_64(qp->swqe_polarity, I40IWQPSQ_VALID); - - dest = (u8 *)wqe; - src = (u8 *)(op_info->data); - - if (op_info->len <= 16) { - memcpy(dest, src, op_info->len); - } else { - memcpy(dest, src, 16); - src += 16; - dest = (u8 *)wqe + 32; - memcpy(dest, src, op_info->len - 16); - } - - wmb(); /* make sure WQE is populated before valid bit is set */ - - set_64bit_val(wqe, 24, header); - - if (post_sq) - i40iw_qp_post_wr(qp); - - return 0; -} - -/** - * i40iw_stag_local_invalidate - stag invalidate operation - * @qp: hw qp ptr - * @info: post sq information - * @post_sq: flag to post sq - */ -static enum i40iw_status_code i40iw_stag_local_invalidate(struct i40iw_qp_uk *qp, - struct i40iw_post_sq_info *info, - bool post_sq) -{ - u64 *wqe; - struct i40iw_inv_local_stag *op_info; - u64 header; - u32 wqe_idx; - bool local_fence = false; - - op_info = &info->op.inv_local_stag; - local_fence = info->local_fence; - - wqe = i40iw_qp_get_next_send_wqe(qp, &wqe_idx, I40IW_QP_WQE_MIN_SIZE, 0, info->wr_id); - if (!wqe) - return I40IW_ERR_QP_TOOMANY_WRS_POSTED; - set_64bit_val(wqe, 0, 0); - set_64bit_val(wqe, 8, - LS_64(op_info->target_stag, I40IWQPSQ_LOCSTAG)); - set_64bit_val(wqe, 16, 0); - header = LS_64(I40IW_OP_TYPE_INV_STAG, I40IWQPSQ_OPCODE) | - LS_64(info->read_fence, I40IWQPSQ_READFENCE) | - LS_64(local_fence, I40IWQPSQ_LOCALFENCE) | - LS_64(info->signaled, I40IWQPSQ_SIGCOMPL) | - LS_64(qp->swqe_polarity, I40IWQPSQ_VALID); - - wmb(); /* make sure WQE is populated before valid bit is set */ - - set_64bit_val(wqe, 24, header); - - if (post_sq) - i40iw_qp_post_wr(qp); - - return 0; -} - -/** - * i40iw_mw_bind - Memory Window bind operation - * @qp: hw qp ptr - * @info: post sq information - * @post_sq: flag to post sq - */ -static enum i40iw_status_code i40iw_mw_bind(struct i40iw_qp_uk *qp, - struct i40iw_post_sq_info *info, - bool post_sq) -{ - u64 *wqe; - struct i40iw_bind_window *op_info; - u64 header; - u32 wqe_idx; - bool local_fence = false; - - op_info = &info->op.bind_window; - - local_fence |= info->local_fence; - wqe = i40iw_qp_get_next_send_wqe(qp, &wqe_idx, I40IW_QP_WQE_MIN_SIZE, 0, info->wr_id); - if (!wqe) - return I40IW_ERR_QP_TOOMANY_WRS_POSTED; - set_64bit_val(wqe, 0, (uintptr_t)op_info->va); - set_64bit_val(wqe, 8, - LS_64(op_info->mr_stag, I40IWQPSQ_PARENTMRSTAG) | - LS_64(op_info->mw_stag, I40IWQPSQ_MWSTAG)); - set_64bit_val(wqe, 16, op_info->bind_length); - header = LS_64(I40IW_OP_TYPE_BIND_MW, I40IWQPSQ_OPCODE) | - LS_64(((op_info->enable_reads << 2) | - (op_info->enable_writes << 3)), - I40IWQPSQ_STAGRIGHTS) | - LS_64((op_info->addressing_type == I40IW_ADDR_TYPE_VA_BASED ? 1 : 0), - I40IWQPSQ_VABASEDTO) | - LS_64(info->read_fence, I40IWQPSQ_READFENCE) | - LS_64(local_fence, I40IWQPSQ_LOCALFENCE) | - LS_64(info->signaled, I40IWQPSQ_SIGCOMPL) | - LS_64(qp->swqe_polarity, I40IWQPSQ_VALID); - - wmb(); /* make sure WQE is populated before valid bit is set */ - - set_64bit_val(wqe, 24, header); - - if (post_sq) - i40iw_qp_post_wr(qp); - - return 0; -} - -/** - * i40iw_post_receive - post receive wqe - * @qp: hw qp ptr - * @info: post rq information - */ -static enum i40iw_status_code i40iw_post_receive(struct i40iw_qp_uk *qp, - struct i40iw_post_rq_info *info) -{ - u64 *wqe; - u64 header; - u32 total_size = 0, wqe_idx, i, byte_off; - - if (qp->max_rq_frag_cnt < info->num_sges) - return I40IW_ERR_INVALID_FRAG_COUNT; - for (i = 0; i < info->num_sges; i++) - total_size += info->sg_list[i].len; - wqe = i40iw_qp_get_next_recv_wqe(qp, &wqe_idx); - if (!wqe) - return I40IW_ERR_QP_TOOMANY_WRS_POSTED; - - qp->rq_wrid_array[wqe_idx] = info->wr_id; - set_64bit_val(wqe, 16, 0); - - header = LS_64((info->num_sges > 1 ? (info->num_sges - 1) : 0), - I40IWQPSQ_ADDFRAGCNT) | - LS_64(qp->rwqe_polarity, I40IWQPSQ_VALID); - - i40iw_set_fragment(wqe, 0, info->sg_list); - - for (i = 1, byte_off = 32; i < info->num_sges; i++) { - i40iw_set_fragment(wqe, byte_off, &info->sg_list[i]); - byte_off += 16; - } - - wmb(); /* make sure WQE is populated before valid bit is set */ - - set_64bit_val(wqe, 24, header); - - return 0; -} - -/** - * i40iw_cq_request_notification - cq notification request (door bell) - * @cq: hw cq - * @cq_notify: notification type - */ -static void i40iw_cq_request_notification(struct i40iw_cq_uk *cq, - enum i40iw_completion_notify cq_notify) -{ - u64 temp_val; - u16 sw_cq_sel; - u8 arm_next_se = 0; - u8 arm_next = 0; - u8 arm_seq_num; - - get_64bit_val(cq->shadow_area, 32, &temp_val); - arm_seq_num = (u8)RS_64(temp_val, I40IW_CQ_DBSA_ARM_SEQ_NUM); - arm_seq_num++; - - sw_cq_sel = (u16)RS_64(temp_val, I40IW_CQ_DBSA_SW_CQ_SELECT); - arm_next_se = (u8)RS_64(temp_val, I40IW_CQ_DBSA_ARM_NEXT_SE); - arm_next_se |= 1; - if (cq_notify == IW_CQ_COMPL_EVENT) - arm_next = 1; - temp_val = LS_64(arm_seq_num, I40IW_CQ_DBSA_ARM_SEQ_NUM) | - LS_64(sw_cq_sel, I40IW_CQ_DBSA_SW_CQ_SELECT) | - LS_64(arm_next_se, I40IW_CQ_DBSA_ARM_NEXT_SE) | - LS_64(arm_next, I40IW_CQ_DBSA_ARM_NEXT); - - set_64bit_val(cq->shadow_area, 32, temp_val); - - wmb(); /* make sure WQE is populated before valid bit is set */ - - writel(cq->cq_id, cq->cqe_alloc_reg); -} - -/** - * i40iw_cq_post_entries - update tail in shadow memory - * @cq: hw cq - * @count: # of entries processed - */ -static enum i40iw_status_code i40iw_cq_post_entries(struct i40iw_cq_uk *cq, - u8 count) -{ - I40IW_RING_MOVE_TAIL_BY_COUNT(cq->cq_ring, count); - set_64bit_val(cq->shadow_area, 0, - I40IW_RING_GETCURRENT_HEAD(cq->cq_ring)); - return 0; -} - -/** - * i40iw_cq_poll_completion - get cq completion info - * @cq: hw cq - * @info: cq poll information returned - */ -static enum i40iw_status_code i40iw_cq_poll_completion(struct i40iw_cq_uk *cq, - struct i40iw_cq_poll_info *info) -{ - u64 comp_ctx, qword0, qword2, qword3, wqe_qword; - u64 *cqe, *sw_wqe; - struct i40iw_qp_uk *qp; - struct i40iw_ring *pring = NULL; - u32 wqe_idx, q_type, array_idx = 0; - enum i40iw_status_code ret_code = 0; - bool move_cq_head = true; - u8 polarity; - u8 addl_wqes = 0; - - if (cq->avoid_mem_cflct) - cqe = (u64 *)I40IW_GET_CURRENT_EXTENDED_CQ_ELEMENT(cq); - else - cqe = (u64 *)I40IW_GET_CURRENT_CQ_ELEMENT(cq); - - get_64bit_val(cqe, 24, &qword3); - polarity = (u8)RS_64(qword3, I40IW_CQ_VALID); - - if (polarity != cq->polarity) - return I40IW_ERR_QUEUE_EMPTY; - - q_type = (u8)RS_64(qword3, I40IW_CQ_SQ); - info->error = (bool)RS_64(qword3, I40IW_CQ_ERROR); - if (info->error) { - info->comp_status = I40IW_COMPL_STATUS_FLUSHED; - info->major_err = (bool)RS_64(qword3, I40IW_CQ_MAJERR); - info->minor_err = (bool)RS_64(qword3, I40IW_CQ_MINERR); - } else { - info->comp_status = I40IW_COMPL_STATUS_SUCCESS; - } - - get_64bit_val(cqe, 0, &qword0); - get_64bit_val(cqe, 16, &qword2); - - info->tcp_seq_num = (u32)RS_64(qword0, I40IWCQ_TCPSEQNUM); - - info->qp_id = (u32)RS_64(qword2, I40IWCQ_QPID); - - get_64bit_val(cqe, 8, &comp_ctx); - - info->solicited_event = (bool)RS_64(qword3, I40IWCQ_SOEVENT); - info->is_srq = (bool)RS_64(qword3, I40IWCQ_SRQ); - - qp = (struct i40iw_qp_uk *)(unsigned long)comp_ctx; - if (!qp) { - ret_code = I40IW_ERR_QUEUE_DESTROYED; - goto exit; - } - wqe_idx = (u32)RS_64(qword3, I40IW_CQ_WQEIDX); - info->qp_handle = (i40iw_qp_handle)(unsigned long)qp; - - if (q_type == I40IW_CQE_QTYPE_RQ) { - array_idx = (wqe_idx * 4) / qp->rq_wqe_size_multiplier; - if (info->comp_status == I40IW_COMPL_STATUS_FLUSHED) { - info->wr_id = qp->rq_wrid_array[qp->rq_ring.tail]; - array_idx = qp->rq_ring.tail; - } else { - info->wr_id = qp->rq_wrid_array[array_idx]; - } - - info->op_type = I40IW_OP_TYPE_REC; - if (qword3 & I40IWCQ_STAG_MASK) { - info->stag_invalid_set = true; - info->inv_stag = (u32)RS_64(qword2, I40IWCQ_INVSTAG); - } else { - info->stag_invalid_set = false; - } - info->bytes_xfered = (u32)RS_64(qword0, I40IWCQ_PAYLDLEN); - I40IW_RING_SET_TAIL(qp->rq_ring, array_idx + 1); - pring = &qp->rq_ring; - } else { - if (qp->first_sq_wq) { - qp->first_sq_wq = false; - if (!wqe_idx && (qp->sq_ring.head == qp->sq_ring.tail)) { - I40IW_RING_MOVE_HEAD_NOCHECK(cq->cq_ring); - I40IW_RING_MOVE_TAIL(cq->cq_ring); - set_64bit_val(cq->shadow_area, 0, - I40IW_RING_GETCURRENT_HEAD(cq->cq_ring)); - memset(info, 0, sizeof(struct i40iw_cq_poll_info)); - return i40iw_cq_poll_completion(cq, info); - } - } - - if (info->comp_status != I40IW_COMPL_STATUS_FLUSHED) { - info->wr_id = qp->sq_wrtrk_array[wqe_idx].wrid; - info->bytes_xfered = qp->sq_wrtrk_array[wqe_idx].wr_len; - - info->op_type = (u8)RS_64(qword3, I40IWCQ_OP); - sw_wqe = qp->sq_base[wqe_idx].elem; - get_64bit_val(sw_wqe, 24, &wqe_qword); - - addl_wqes = qp->sq_wrtrk_array[wqe_idx].wqe_size / I40IW_QP_WQE_MIN_SIZE; - I40IW_RING_SET_TAIL(qp->sq_ring, (wqe_idx + addl_wqes)); - } else { - do { - u8 op_type; - u32 tail; - - tail = qp->sq_ring.tail; - sw_wqe = qp->sq_base[tail].elem; - get_64bit_val(sw_wqe, 24, &wqe_qword); - op_type = (u8)RS_64(wqe_qword, I40IWQPSQ_OPCODE); - info->op_type = op_type; - addl_wqes = qp->sq_wrtrk_array[tail].wqe_size / I40IW_QP_WQE_MIN_SIZE; - I40IW_RING_SET_TAIL(qp->sq_ring, (tail + addl_wqes)); - if (op_type != I40IWQP_OP_NOP) { - info->wr_id = qp->sq_wrtrk_array[tail].wrid; - info->bytes_xfered = qp->sq_wrtrk_array[tail].wr_len; - break; - } - } while (1); - } - pring = &qp->sq_ring; - } - - ret_code = 0; - -exit: - if (!ret_code && - (info->comp_status == I40IW_COMPL_STATUS_FLUSHED)) - if (pring && (I40IW_RING_MORE_WORK(*pring))) - move_cq_head = false; - - if (move_cq_head) { - I40IW_RING_MOVE_HEAD_NOCHECK(cq->cq_ring); - - if (I40IW_RING_GETCURRENT_HEAD(cq->cq_ring) == 0) - cq->polarity ^= 1; - - I40IW_RING_MOVE_TAIL(cq->cq_ring); - set_64bit_val(cq->shadow_area, 0, - I40IW_RING_GETCURRENT_HEAD(cq->cq_ring)); - } else { - if (info->is_srq) - return ret_code; - qword3 &= ~I40IW_CQ_WQEIDX_MASK; - qword3 |= LS_64(pring->tail, I40IW_CQ_WQEIDX); - set_64bit_val(cqe, 24, qword3); - } - - return ret_code; -} - -/** - * i40iw_get_wqe_shift - get shift count for maximum wqe size - * @sge: Maximum Scatter Gather Elements wqe - * @inline_data: Maximum inline data size - * @shift: Returns the shift needed based on sge - * - * Shift can be used to left shift the wqe size based on number of SGEs and inlind data size. - * For 1 SGE or inline data <= 16, shift = 0 (wqe size of 32 bytes). - * For 2 or 3 SGEs or inline data <= 48, shift = 1 (wqe size of 64 bytes). - * Shift of 2 otherwise (wqe size of 128 bytes). - */ -void i40iw_get_wqe_shift(u32 sge, u32 inline_data, u8 *shift) -{ - *shift = 0; - if (sge > 1 || inline_data > 16) - *shift = (sge < 4 && inline_data <= 48) ? 1 : 2; -} - -/* - * i40iw_get_sqdepth - get SQ depth (quantas) - * @sq_size: SQ size - * @shift: shift which determines size of WQE - * @sqdepth: depth of SQ - * - */ -enum i40iw_status_code i40iw_get_sqdepth(u32 sq_size, u8 shift, u32 *sqdepth) -{ - *sqdepth = roundup_pow_of_two((sq_size << shift) + I40IW_SQ_RSVD); - - if (*sqdepth < (I40IW_QP_SW_MIN_WQSIZE << shift)) - *sqdepth = I40IW_QP_SW_MIN_WQSIZE << shift; - else if (*sqdepth > I40IW_QP_SW_MAX_SQ_QUANTAS) - return I40IW_ERR_INVALID_SIZE; - - return 0; -} - -/* - * i40iw_get_rq_depth - get RQ depth (quantas) - * @rq_size: RQ size - * @shift: shift which determines size of WQE - * @rqdepth: depth of RQ - * - */ -enum i40iw_status_code i40iw_get_rqdepth(u32 rq_size, u8 shift, u32 *rqdepth) -{ - *rqdepth = roundup_pow_of_two((rq_size << shift) + I40IW_RQ_RSVD); - - if (*rqdepth < (I40IW_QP_SW_MIN_WQSIZE << shift)) - *rqdepth = I40IW_QP_SW_MIN_WQSIZE << shift; - else if (*rqdepth > I40IW_QP_SW_MAX_RQ_QUANTAS) - return I40IW_ERR_INVALID_SIZE; - - return 0; -} - -static const struct i40iw_qp_uk_ops iw_qp_uk_ops = { - .iw_qp_post_wr = i40iw_qp_post_wr, - .iw_rdma_write = i40iw_rdma_write, - .iw_rdma_read = i40iw_rdma_read, - .iw_send = i40iw_send, - .iw_inline_rdma_write = i40iw_inline_rdma_write, - .iw_inline_send = i40iw_inline_send, - .iw_stag_local_invalidate = i40iw_stag_local_invalidate, - .iw_mw_bind = i40iw_mw_bind, - .iw_post_receive = i40iw_post_receive, - .iw_post_nop = i40iw_nop -}; - -static const struct i40iw_cq_ops iw_cq_ops = { - .iw_cq_request_notification = i40iw_cq_request_notification, - .iw_cq_poll_completion = i40iw_cq_poll_completion, - .iw_cq_post_entries = i40iw_cq_post_entries, - .iw_cq_clean = i40iw_clean_cq -}; - -static const struct i40iw_device_uk_ops iw_device_uk_ops = { - .iwarp_cq_uk_init = i40iw_cq_uk_init, - .iwarp_qp_uk_init = i40iw_qp_uk_init, -}; - -/** - * i40iw_qp_uk_init - initialize shared qp - * @qp: hw qp (user and kernel) - * @info: qp initialization info - * - * initializes the vars used in both user and kernel mode. - * size of the wqe depends on numbers of max. fragements - * allowed. Then size of wqe * the number of wqes should be the - * amount of memory allocated for sq and rq. If srq is used, - * then rq_base will point to one rq wqe only (not the whole - * array of wqes) - */ -enum i40iw_status_code i40iw_qp_uk_init(struct i40iw_qp_uk *qp, - struct i40iw_qp_uk_init_info *info) -{ - enum i40iw_status_code ret_code = 0; - u32 sq_ring_size; - u8 sqshift, rqshift; - - if (info->max_sq_frag_cnt > I40IW_MAX_WQ_FRAGMENT_COUNT) - return I40IW_ERR_INVALID_FRAG_COUNT; - - if (info->max_rq_frag_cnt > I40IW_MAX_WQ_FRAGMENT_COUNT) - return I40IW_ERR_INVALID_FRAG_COUNT; - i40iw_get_wqe_shift(info->max_sq_frag_cnt, info->max_inline_data, &sqshift); - - qp->sq_base = info->sq; - qp->rq_base = info->rq; - qp->shadow_area = info->shadow_area; - qp->sq_wrtrk_array = info->sq_wrtrk_array; - qp->rq_wrid_array = info->rq_wrid_array; - - qp->wqe_alloc_reg = info->wqe_alloc_reg; - qp->qp_id = info->qp_id; - qp->sq_size = info->sq_size; - qp->max_sq_frag_cnt = info->max_sq_frag_cnt; - sq_ring_size = qp->sq_size << sqshift; - - I40IW_RING_INIT(qp->sq_ring, sq_ring_size); - I40IW_RING_INIT(qp->initial_ring, sq_ring_size); - I40IW_RING_MOVE_HEAD(qp->sq_ring, ret_code); - I40IW_RING_MOVE_TAIL(qp->sq_ring); - I40IW_RING_MOVE_HEAD(qp->initial_ring, ret_code); - qp->swqe_polarity = 1; - qp->first_sq_wq = true; - qp->swqe_polarity_deferred = 1; - qp->rwqe_polarity = 0; - - if (!qp->use_srq) { - qp->rq_size = info->rq_size; - qp->max_rq_frag_cnt = info->max_rq_frag_cnt; - I40IW_RING_INIT(qp->rq_ring, qp->rq_size); - switch (info->abi_ver) { - case 4: - i40iw_get_wqe_shift(info->max_rq_frag_cnt, 0, &rqshift); - break; - case 5: /* fallthrough until next ABI version */ - default: - rqshift = I40IW_MAX_RQ_WQE_SHIFT; - break; - } - qp->rq_wqe_size = rqshift; - qp->rq_wqe_size_multiplier = 4 << rqshift; - } - qp->ops = iw_qp_uk_ops; - - return ret_code; -} - -/** - * i40iw_cq_uk_init - initialize shared cq (user and kernel) - * @cq: hw cq - * @info: hw cq initialization info - */ -enum i40iw_status_code i40iw_cq_uk_init(struct i40iw_cq_uk *cq, - struct i40iw_cq_uk_init_info *info) -{ - if ((info->cq_size < I40IW_MIN_CQ_SIZE) || - (info->cq_size > I40IW_MAX_CQ_SIZE)) - return I40IW_ERR_INVALID_SIZE; - cq->cq_base = (struct i40iw_cqe *)info->cq_base; - cq->cq_id = info->cq_id; - cq->cq_size = info->cq_size; - cq->cqe_alloc_reg = info->cqe_alloc_reg; - cq->shadow_area = info->shadow_area; - cq->avoid_mem_cflct = info->avoid_mem_cflct; - - I40IW_RING_INIT(cq->cq_ring, cq->cq_size); - cq->polarity = 1; - cq->ops = iw_cq_ops; - - return 0; -} - -/** - * i40iw_device_init_uk - setup routines for iwarp shared device - * @dev: iwarp shared (user and kernel) - */ -void i40iw_device_init_uk(struct i40iw_dev_uk *dev) -{ - dev->ops_uk = iw_device_uk_ops; -} - -/** - * i40iw_clean_cq - clean cq entries - * @queue: completion context - * @cq: cq to clean - */ -void i40iw_clean_cq(void *queue, struct i40iw_cq_uk *cq) -{ - u64 *cqe; - u64 qword3, comp_ctx; - u32 cq_head; - u8 polarity, temp; - - cq_head = cq->cq_ring.head; - temp = cq->polarity; - do { - if (cq->avoid_mem_cflct) - cqe = (u64 *)&(((struct i40iw_extended_cqe *)cq->cq_base)[cq_head]); - else - cqe = (u64 *)&cq->cq_base[cq_head]; - get_64bit_val(cqe, 24, &qword3); - polarity = (u8)RS_64(qword3, I40IW_CQ_VALID); - - if (polarity != temp) - break; - - get_64bit_val(cqe, 8, &comp_ctx); - if ((void *)(unsigned long)comp_ctx == queue) - set_64bit_val(cqe, 8, 0); - - cq_head = (cq_head + 1) % cq->cq_ring.size; - if (!cq_head) - temp ^= 1; - } while (true); -} - -/** - * i40iw_nop - send a nop - * @qp: hw qp ptr - * @wr_id: work request id - * @signaled: flag if signaled for completion - * @post_sq: flag to post sq - */ -enum i40iw_status_code i40iw_nop(struct i40iw_qp_uk *qp, - u64 wr_id, - bool signaled, - bool post_sq) -{ - u64 header, *wqe; - u32 wqe_idx; - - wqe = i40iw_qp_get_next_send_wqe(qp, &wqe_idx, I40IW_QP_WQE_MIN_SIZE, 0, wr_id); - if (!wqe) - return I40IW_ERR_QP_TOOMANY_WRS_POSTED; - set_64bit_val(wqe, 0, 0); - set_64bit_val(wqe, 8, 0); - set_64bit_val(wqe, 16, 0); - - header = LS_64(I40IWQP_OP_NOP, I40IWQPSQ_OPCODE) | - LS_64(signaled, I40IWQPSQ_SIGCOMPL) | - LS_64(qp->swqe_polarity, I40IWQPSQ_VALID); - - wmb(); /* make sure WQE is populated before valid bit is set */ - - set_64bit_val(wqe, 24, header); - if (post_sq) - i40iw_qp_post_wr(qp); - - return 0; -} - -/** - * i40iw_fragcnt_to_wqesize_sq - calculate wqe size based on fragment count for SQ - * @frag_cnt: number of fragments - * @wqe_size: size of sq wqe returned - */ -enum i40iw_status_code i40iw_fragcnt_to_wqesize_sq(u32 frag_cnt, u8 *wqe_size) -{ - switch (frag_cnt) { - case 0: - case 1: - *wqe_size = I40IW_QP_WQE_MIN_SIZE; - break; - case 2: - case 3: - *wqe_size = 64; - break; - case 4: - case 5: - *wqe_size = 96; - break; - case 6: - case 7: - *wqe_size = 128; - break; - default: - return I40IW_ERR_INVALID_FRAG_COUNT; - } - - return 0; -} - -/** - * i40iw_fragcnt_to_wqesize_rq - calculate wqe size based on fragment count for RQ - * @frag_cnt: number of fragments - * @wqe_size: size of rq wqe returned - */ -enum i40iw_status_code i40iw_fragcnt_to_wqesize_rq(u32 frag_cnt, u8 *wqe_size) -{ - switch (frag_cnt) { - case 0: - case 1: - *wqe_size = 32; - break; - case 2: - case 3: - *wqe_size = 64; - break; - case 4: - case 5: - case 6: - case 7: - *wqe_size = 128; - break; - default: - return I40IW_ERR_INVALID_FRAG_COUNT; - } - - return 0; -} - -/** - * i40iw_inline_data_size_to_wqesize - based on inline data, wqe size - * @data_size: data size for inline - * @wqe_size: size of sq wqe returned - */ -enum i40iw_status_code i40iw_inline_data_size_to_wqesize(u32 data_size, - u8 *wqe_size) -{ - if (data_size > I40IW_MAX_INLINE_DATA_SIZE) - return I40IW_ERR_INVALID_INLINE_DATA_SIZE; - - if (data_size <= 16) - *wqe_size = I40IW_QP_WQE_MIN_SIZE; - else - *wqe_size = 64; - - return 0; -} diff --git a/drivers/infiniband/hw/i40iw/i40iw_user.h b/drivers/infiniband/hw/i40iw/i40iw_user.h deleted file mode 100644 index 93fc3081dd65..000000000000 --- a/drivers/infiniband/hw/i40iw/i40iw_user.h +++ /dev/null @@ -1,422 +0,0 @@ -/******************************************************************************* -* -* Copyright (c) 2015-2016 Intel Corporation. All rights reserved. -* -* This software is available to you under a choice of one of two -* licenses. You may choose to be licensed under the terms of the GNU -* General Public License (GPL) Version 2, available from the file -* COPYING in the main directory of this source tree, or the -* OpenFabrics.org BSD license below: -* -* Redistribution and use in source and binary forms, with or -* without modification, are permitted provided that the following -* conditions are met: -* -* - Redistributions of source code must retain the above -* copyright notice, this list of conditions and the following -* disclaimer. -* -* - Redistributions in binary form must reproduce the above -* copyright notice, this list of conditions and the following -* disclaimer in the documentation and/or other materials -* provided with the distribution. -* -* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS -* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN -* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -* SOFTWARE. -* -*******************************************************************************/ - -#ifndef I40IW_USER_H -#define I40IW_USER_H - -enum i40iw_device_capabilities_const { - I40IW_WQE_SIZE = 4, - I40IW_CQP_WQE_SIZE = 8, - I40IW_CQE_SIZE = 4, - I40IW_EXTENDED_CQE_SIZE = 8, - I40IW_AEQE_SIZE = 2, - I40IW_CEQE_SIZE = 1, - I40IW_CQP_CTX_SIZE = 8, - I40IW_SHADOW_AREA_SIZE = 8, - I40IW_CEQ_MAX_COUNT = 256, - I40IW_QUERY_FPM_BUF_SIZE = 128, - I40IW_COMMIT_FPM_BUF_SIZE = 128, - I40IW_MIN_IW_QP_ID = 1, - I40IW_MAX_IW_QP_ID = 262143, - I40IW_MIN_CEQID = 0, - I40IW_MAX_CEQID = 256, - I40IW_MIN_CQID = 0, - I40IW_MAX_CQID = 131071, - I40IW_MIN_AEQ_ENTRIES = 1, - I40IW_MAX_AEQ_ENTRIES = 524287, - I40IW_MIN_CEQ_ENTRIES = 1, - I40IW_MAX_CEQ_ENTRIES = 131071, - I40IW_MIN_CQ_SIZE = 1, - I40IW_MAX_CQ_SIZE = 1048575, - I40IW_DB_ID_ZERO = 0, - I40IW_MAX_WQ_FRAGMENT_COUNT = 3, - I40IW_MAX_SGE_RD = 1, - I40IW_MAX_OUTBOUND_MESSAGE_SIZE = 2147483647, - I40IW_MAX_INBOUND_MESSAGE_SIZE = 2147483647, - I40IW_MAX_PE_ENABLED_VF_COUNT = 32, - I40IW_MAX_VF_FPM_ID = 47, - I40IW_MAX_VF_PER_PF = 127, - I40IW_MAX_SQ_PAYLOAD_SIZE = 2145386496, - I40IW_MAX_INLINE_DATA_SIZE = 48, - I40IW_MAX_IRD_SIZE = 64, - I40IW_MAX_ORD_SIZE = 127, - I40IW_MAX_WQ_ENTRIES = 2048, - I40IW_Q2_BUFFER_SIZE = (248 + 100), - I40IW_MAX_WQE_SIZE_RQ = 128, - I40IW_QP_CTX_SIZE = 248, - I40IW_MAX_PDS = 32768 -}; - -#define i40iw_handle void * -#define i40iw_adapter_handle i40iw_handle -#define i40iw_qp_handle i40iw_handle -#define i40iw_cq_handle i40iw_handle -#define i40iw_srq_handle i40iw_handle -#define i40iw_pd_id i40iw_handle -#define i40iw_stag_handle i40iw_handle -#define i40iw_stag_index u32 -#define i40iw_stag u32 -#define i40iw_stag_key u8 - -#define i40iw_tagged_offset u64 -#define i40iw_access_privileges u32 -#define i40iw_physical_fragment u64 -#define i40iw_address_list u64 * - -#define I40IW_MAX_MR_SIZE 0x10000000000L -#define I40IW_MAX_RQ_WQE_SHIFT 2 - -struct i40iw_qp_uk; -struct i40iw_cq_uk; -struct i40iw_srq_uk; -struct i40iw_qp_uk_init_info; -struct i40iw_cq_uk_init_info; -struct i40iw_srq_uk_init_info; - -struct i40iw_sge { - i40iw_tagged_offset tag_off; - u32 len; - i40iw_stag stag; -}; - -#define i40iw_sgl struct i40iw_sge * - -struct i40iw_ring { - u32 head; - u32 tail; - u32 size; -}; - -struct i40iw_cqe { - u64 buf[I40IW_CQE_SIZE]; -}; - -struct i40iw_extended_cqe { - u64 buf[I40IW_EXTENDED_CQE_SIZE]; -}; - -struct i40iw_wqe { - u64 buf[I40IW_WQE_SIZE]; -}; - -struct i40iw_qp_uk_ops; - -enum i40iw_addressing_type { - I40IW_ADDR_TYPE_ZERO_BASED = 0, - I40IW_ADDR_TYPE_VA_BASED = 1, -}; - -#define I40IW_ACCESS_FLAGS_LOCALREAD 0x01 -#define I40IW_ACCESS_FLAGS_LOCALWRITE 0x02 -#define I40IW_ACCESS_FLAGS_REMOTEREAD_ONLY 0x04 -#define I40IW_ACCESS_FLAGS_REMOTEREAD 0x05 -#define I40IW_ACCESS_FLAGS_REMOTEWRITE_ONLY 0x08 -#define I40IW_ACCESS_FLAGS_REMOTEWRITE 0x0a -#define I40IW_ACCESS_FLAGS_BIND_WINDOW 0x10 -#define I40IW_ACCESS_FLAGS_ALL 0x1F - -#define I40IW_OP_TYPE_RDMA_WRITE 0 -#define I40IW_OP_TYPE_RDMA_READ 1 -#define I40IW_OP_TYPE_SEND 3 -#define I40IW_OP_TYPE_SEND_INV 4 -#define I40IW_OP_TYPE_SEND_SOL 5 -#define I40IW_OP_TYPE_SEND_SOL_INV 6 -#define I40IW_OP_TYPE_REC 7 -#define I40IW_OP_TYPE_BIND_MW 8 -#define I40IW_OP_TYPE_FAST_REG_NSMR 9 -#define I40IW_OP_TYPE_INV_STAG 10 -#define I40IW_OP_TYPE_RDMA_READ_INV_STAG 11 -#define I40IW_OP_TYPE_NOP 12 - -enum i40iw_completion_status { - I40IW_COMPL_STATUS_SUCCESS = 0, - I40IW_COMPL_STATUS_FLUSHED, - I40IW_COMPL_STATUS_INVALID_WQE, - I40IW_COMPL_STATUS_QP_CATASTROPHIC, - I40IW_COMPL_STATUS_REMOTE_TERMINATION, - I40IW_COMPL_STATUS_INVALID_STAG, - I40IW_COMPL_STATUS_BASE_BOUND_VIOLATION, - I40IW_COMPL_STATUS_ACCESS_VIOLATION, - I40IW_COMPL_STATUS_INVALID_PD_ID, - I40IW_COMPL_STATUS_WRAP_ERROR, - I40IW_COMPL_STATUS_STAG_INVALID_PDID, - I40IW_COMPL_STATUS_RDMA_READ_ZERO_ORD, - I40IW_COMPL_STATUS_QP_NOT_PRIVLEDGED, - I40IW_COMPL_STATUS_STAG_NOT_INVALID, - I40IW_COMPL_STATUS_INVALID_PHYS_BUFFER_SIZE, - I40IW_COMPL_STATUS_INVALID_PHYS_BUFFER_ENTRY, - I40IW_COMPL_STATUS_INVALID_FBO, - I40IW_COMPL_STATUS_INVALID_LENGTH, - I40IW_COMPL_STATUS_INVALID_ACCESS, - I40IW_COMPL_STATUS_PHYS_BUFFER_LIST_TOO_LONG, - I40IW_COMPL_STATUS_INVALID_VIRT_ADDRESS, - I40IW_COMPL_STATUS_INVALID_REGION, - I40IW_COMPL_STATUS_INVALID_WINDOW, - I40IW_COMPL_STATUS_INVALID_TOTAL_LENGTH -}; - -enum i40iw_completion_notify { - IW_CQ_COMPL_EVENT = 0, - IW_CQ_COMPL_SOLICITED = 1 -}; - -struct i40iw_post_send { - i40iw_sgl sg_list; - u32 num_sges; -}; - -struct i40iw_post_inline_send { - void *data; - u32 len; -}; - -struct i40iw_rdma_write { - i40iw_sgl lo_sg_list; - u32 num_lo_sges; - struct i40iw_sge rem_addr; -}; - -struct i40iw_inline_rdma_write { - void *data; - u32 len; - struct i40iw_sge rem_addr; -}; - -struct i40iw_rdma_read { - struct i40iw_sge lo_addr; - struct i40iw_sge rem_addr; -}; - -struct i40iw_bind_window { - i40iw_stag mr_stag; - u64 bind_length; - void *va; - enum i40iw_addressing_type addressing_type; - bool enable_reads; - bool enable_writes; - i40iw_stag mw_stag; -}; - -struct i40iw_inv_local_stag { - i40iw_stag target_stag; -}; - -struct i40iw_post_sq_info { - u64 wr_id; - u8 op_type; - bool signaled; - bool read_fence; - bool local_fence; - bool inline_data; - bool defer_flag; - union { - struct i40iw_post_send send; - struct i40iw_rdma_write rdma_write; - struct i40iw_rdma_read rdma_read; - struct i40iw_rdma_read rdma_read_inv; - struct i40iw_bind_window bind_window; - struct i40iw_inv_local_stag inv_local_stag; - struct i40iw_inline_rdma_write inline_rdma_write; - struct i40iw_post_inline_send inline_send; - } op; -}; - -struct i40iw_post_rq_info { - u64 wr_id; - i40iw_sgl sg_list; - u32 num_sges; -}; - -struct i40iw_cq_poll_info { - u64 wr_id; - i40iw_qp_handle qp_handle; - u32 bytes_xfered; - u32 tcp_seq_num; - u32 qp_id; - i40iw_stag inv_stag; - enum i40iw_completion_status comp_status; - u16 major_err; - u16 minor_err; - u8 op_type; - bool stag_invalid_set; - bool error; - bool is_srq; - bool solicited_event; -}; - -struct i40iw_qp_uk_ops { - void (*iw_qp_post_wr)(struct i40iw_qp_uk *); - enum i40iw_status_code (*iw_rdma_write)(struct i40iw_qp_uk *, - struct i40iw_post_sq_info *, bool); - enum i40iw_status_code (*iw_rdma_read)(struct i40iw_qp_uk *, - struct i40iw_post_sq_info *, bool, bool); - enum i40iw_status_code (*iw_send)(struct i40iw_qp_uk *, - struct i40iw_post_sq_info *, u32, bool); - enum i40iw_status_code (*iw_inline_rdma_write)(struct i40iw_qp_uk *, - struct i40iw_post_sq_info *, bool); - enum i40iw_status_code (*iw_inline_send)(struct i40iw_qp_uk *, - struct i40iw_post_sq_info *, u32, bool); - enum i40iw_status_code (*iw_stag_local_invalidate)(struct i40iw_qp_uk *, - struct i40iw_post_sq_info *, bool); - enum i40iw_status_code (*iw_mw_bind)(struct i40iw_qp_uk *, - struct i40iw_post_sq_info *, bool); - enum i40iw_status_code (*iw_post_receive)(struct i40iw_qp_uk *, - struct i40iw_post_rq_info *); - enum i40iw_status_code (*iw_post_nop)(struct i40iw_qp_uk *, u64, bool, bool); -}; - -struct i40iw_cq_ops { - void (*iw_cq_request_notification)(struct i40iw_cq_uk *, - enum i40iw_completion_notify); - enum i40iw_status_code (*iw_cq_poll_completion)(struct i40iw_cq_uk *, - struct i40iw_cq_poll_info *); - enum i40iw_status_code (*iw_cq_post_entries)(struct i40iw_cq_uk *, u8 count); - void (*iw_cq_clean)(void *, struct i40iw_cq_uk *); -}; - -struct i40iw_dev_uk; - -struct i40iw_device_uk_ops { - enum i40iw_status_code (*iwarp_cq_uk_init)(struct i40iw_cq_uk *, - struct i40iw_cq_uk_init_info *); - enum i40iw_status_code (*iwarp_qp_uk_init)(struct i40iw_qp_uk *, - struct i40iw_qp_uk_init_info *); -}; - -struct i40iw_dev_uk { - struct i40iw_device_uk_ops ops_uk; -}; - -struct i40iw_sq_uk_wr_trk_info { - u64 wrid; - u32 wr_len; - u8 wqe_size; - u8 reserved[3]; -}; - -struct i40iw_qp_quanta { - u64 elem[I40IW_WQE_SIZE]; -}; - -struct i40iw_qp_uk { - struct i40iw_qp_quanta *sq_base; - struct i40iw_qp_quanta *rq_base; - u32 __iomem *wqe_alloc_reg; - struct i40iw_sq_uk_wr_trk_info *sq_wrtrk_array; - u64 *rq_wrid_array; - u64 *shadow_area; - struct i40iw_ring sq_ring; - struct i40iw_ring rq_ring; - struct i40iw_ring initial_ring; - u32 qp_id; - u32 sq_size; - u32 rq_size; - u32 max_sq_frag_cnt; - u32 max_rq_frag_cnt; - struct i40iw_qp_uk_ops ops; - bool use_srq; - u8 swqe_polarity; - u8 swqe_polarity_deferred; - u8 rwqe_polarity; - u8 rq_wqe_size; - u8 rq_wqe_size_multiplier; - bool first_sq_wq; - bool deferred_flag; -}; - -struct i40iw_cq_uk { - struct i40iw_cqe *cq_base; - u32 __iomem *cqe_alloc_reg; - u64 *shadow_area; - u32 cq_id; - u32 cq_size; - struct i40iw_ring cq_ring; - u8 polarity; - bool avoid_mem_cflct; - - struct i40iw_cq_ops ops; -}; - -struct i40iw_qp_uk_init_info { - struct i40iw_qp_quanta *sq; - struct i40iw_qp_quanta *rq; - u32 __iomem *wqe_alloc_reg; - u64 *shadow_area; - struct i40iw_sq_uk_wr_trk_info *sq_wrtrk_array; - u64 *rq_wrid_array; - u32 qp_id; - u32 sq_size; - u32 rq_size; - u32 max_sq_frag_cnt; - u32 max_rq_frag_cnt; - u32 max_inline_data; - int abi_ver; -}; - -struct i40iw_cq_uk_init_info { - u32 __iomem *cqe_alloc_reg; - struct i40iw_cqe *cq_base; - u64 *shadow_area; - u32 cq_size; - u32 cq_id; - bool avoid_mem_cflct; -}; - -void i40iw_device_init_uk(struct i40iw_dev_uk *dev); - -void i40iw_qp_post_wr(struct i40iw_qp_uk *qp); -u64 *i40iw_qp_get_next_send_wqe(struct i40iw_qp_uk *qp, u32 *wqe_idx, - u8 wqe_size, - u32 total_size, - u64 wr_id - ); -u64 *i40iw_qp_get_next_recv_wqe(struct i40iw_qp_uk *qp, u32 *wqe_idx); -u64 *i40iw_qp_get_next_srq_wqe(struct i40iw_srq_uk *srq, u32 *wqe_idx); - -enum i40iw_status_code i40iw_cq_uk_init(struct i40iw_cq_uk *cq, - struct i40iw_cq_uk_init_info *info); -enum i40iw_status_code i40iw_qp_uk_init(struct i40iw_qp_uk *qp, - struct i40iw_qp_uk_init_info *info); - -void i40iw_clean_cq(void *queue, struct i40iw_cq_uk *cq); -enum i40iw_status_code i40iw_nop(struct i40iw_qp_uk *qp, u64 wr_id, - bool signaled, bool post_sq); -enum i40iw_status_code i40iw_fragcnt_to_wqesize_sq(u32 frag_cnt, u8 *wqe_size); -enum i40iw_status_code i40iw_fragcnt_to_wqesize_rq(u32 frag_cnt, u8 *wqe_size); -enum i40iw_status_code i40iw_inline_data_size_to_wqesize(u32 data_size, - u8 *wqe_size); -void i40iw_get_wqe_shift(u32 sge, u32 inline_data, u8 *shift); -enum i40iw_status_code i40iw_get_sqdepth(u32 sq_size, u8 shift, u32 *sqdepth); -enum i40iw_status_code i40iw_get_rqdepth(u32 rq_size, u8 shift, u32 *rqdepth); -#endif diff --git a/drivers/infiniband/hw/i40iw/i40iw_utils.c b/drivers/infiniband/hw/i40iw/i40iw_utils.c deleted file mode 100644 index 9ff825f7860b..000000000000 --- a/drivers/infiniband/hw/i40iw/i40iw_utils.c +++ /dev/null @@ -1,1518 +0,0 @@ -/******************************************************************************* -* -* Copyright (c) 2015-2016 Intel Corporation. All rights reserved. -* -* This software is available to you under a choice of one of two -* licenses. You may choose to be licensed under the terms of the GNU -* General Public License (GPL) Version 2, available from the file -* COPYING in the main directory of this source tree, or the -* OpenFabrics.org BSD license below: -* -* Redistribution and use in source and binary forms, with or -* without modification, are permitted provided that the following -* conditions are met: -* -* - Redistributions of source code must retain the above -* copyright notice, this list of conditions and the following -* disclaimer. -* -* - Redistributions in binary form must reproduce the above -* copyright notice, this list of conditions and the following -* disclaimer in the documentation and/or other materials -* provided with the distribution. -* -* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS -* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN -* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -* SOFTWARE. -* -*******************************************************************************/ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "i40iw.h" - -/** - * i40iw_arp_table - manage arp table - * @iwdev: iwarp device - * @ip_addr: ip address for device - * @ipv4: flag indicating IPv4 when true - * @mac_addr: mac address ptr - * @action: modify, delete or add - */ -int i40iw_arp_table(struct i40iw_device *iwdev, - u32 *ip_addr, - bool ipv4, - u8 *mac_addr, - u32 action) -{ - int arp_index; - int err; - u32 ip[4]; - - if (ipv4) { - memset(ip, 0, sizeof(ip)); - ip[0] = *ip_addr; - } else { - memcpy(ip, ip_addr, sizeof(ip)); - } - - for (arp_index = 0; (u32)arp_index < iwdev->arp_table_size; arp_index++) - if (memcmp(iwdev->arp_table[arp_index].ip_addr, ip, sizeof(ip)) == 0) - break; - switch (action) { - case I40IW_ARP_ADD: - if (arp_index != iwdev->arp_table_size) - return -1; - - arp_index = 0; - err = i40iw_alloc_resource(iwdev, iwdev->allocated_arps, - iwdev->arp_table_size, - (u32 *)&arp_index, - &iwdev->next_arp_index); - - if (err) - return err; - - memcpy(iwdev->arp_table[arp_index].ip_addr, ip, sizeof(ip)); - ether_addr_copy(iwdev->arp_table[arp_index].mac_addr, mac_addr); - break; - case I40IW_ARP_RESOLVE: - if (arp_index == iwdev->arp_table_size) - return -1; - break; - case I40IW_ARP_DELETE: - if (arp_index == iwdev->arp_table_size) - return -1; - memset(iwdev->arp_table[arp_index].ip_addr, 0, - sizeof(iwdev->arp_table[arp_index].ip_addr)); - eth_zero_addr(iwdev->arp_table[arp_index].mac_addr); - i40iw_free_resource(iwdev, iwdev->allocated_arps, arp_index); - break; - default: - return -1; - } - return arp_index; -} - -/** - * i40iw_wr32 - write 32 bits to hw register - * @hw: hardware information including registers - * @reg: register offset - * @value: vvalue to write to register - */ -inline void i40iw_wr32(struct i40iw_hw *hw, u32 reg, u32 value) -{ - writel(value, hw->hw_addr + reg); -} - -/** - * i40iw_rd32 - read a 32 bit hw register - * @hw: hardware information including registers - * @reg: register offset - * - * Return value of register content - */ -inline u32 i40iw_rd32(struct i40iw_hw *hw, u32 reg) -{ - return readl(hw->hw_addr + reg); -} - -/** - * i40iw_inetaddr_event - system notifier for ipv4 addr events - * @notifier: not used - * @event: event for notifier - * @ptr: if address - */ -int i40iw_inetaddr_event(struct notifier_block *notifier, - unsigned long event, - void *ptr) -{ - struct in_ifaddr *ifa = ptr; - struct net_device *event_netdev = ifa->ifa_dev->dev; - struct net_device *netdev; - struct net_device *upper_dev; - struct i40iw_device *iwdev; - struct i40iw_handler *hdl; - u32 local_ipaddr; - u32 action = I40IW_ARP_ADD; - - hdl = i40iw_find_netdev(event_netdev); - if (!hdl) - return NOTIFY_DONE; - - iwdev = &hdl->device; - if (iwdev->init_state < IP_ADDR_REGISTERED || iwdev->closing) - return NOTIFY_DONE; - - netdev = iwdev->ldev->netdev; - upper_dev = netdev_master_upper_dev_get(netdev); - if (netdev != event_netdev) - return NOTIFY_DONE; - - if (upper_dev) { - struct in_device *in; - - rcu_read_lock(); - in = __in_dev_get_rcu(upper_dev); - - local_ipaddr = 0; - if (in) { - struct in_ifaddr *ifa; - - ifa = rcu_dereference(in->ifa_list); - if (ifa) - local_ipaddr = ntohl(ifa->ifa_address); - } - - rcu_read_unlock(); - } else { - local_ipaddr = ntohl(ifa->ifa_address); - } - switch (event) { - case NETDEV_DOWN: - action = I40IW_ARP_DELETE; - fallthrough; - case NETDEV_UP: - case NETDEV_CHANGEADDR: - - /* Just skip if no need to handle ARP cache */ - if (!local_ipaddr) - break; - - i40iw_manage_arp_cache(iwdev, - netdev->dev_addr, - &local_ipaddr, - true, - action); - i40iw_if_notify(iwdev, netdev, &local_ipaddr, true, - (action == I40IW_ARP_ADD) ? true : false); - break; - default: - break; - } - return NOTIFY_DONE; -} - -/** - * i40iw_inet6addr_event - system notifier for ipv6 addr events - * @notifier: not used - * @event: event for notifier - * @ptr: if address - */ -int i40iw_inet6addr_event(struct notifier_block *notifier, - unsigned long event, - void *ptr) -{ - struct inet6_ifaddr *ifa = (struct inet6_ifaddr *)ptr; - struct net_device *event_netdev = ifa->idev->dev; - struct net_device *netdev; - struct i40iw_device *iwdev; - struct i40iw_handler *hdl; - u32 local_ipaddr6[4]; - u32 action = I40IW_ARP_ADD; - - hdl = i40iw_find_netdev(event_netdev); - if (!hdl) - return NOTIFY_DONE; - - iwdev = &hdl->device; - if (iwdev->init_state < IP_ADDR_REGISTERED || iwdev->closing) - return NOTIFY_DONE; - - netdev = iwdev->ldev->netdev; - if (netdev != event_netdev) - return NOTIFY_DONE; - - i40iw_copy_ip_ntohl(local_ipaddr6, ifa->addr.in6_u.u6_addr32); - switch (event) { - case NETDEV_DOWN: - action = I40IW_ARP_DELETE; - fallthrough; - case NETDEV_UP: - case NETDEV_CHANGEADDR: - i40iw_manage_arp_cache(iwdev, - netdev->dev_addr, - local_ipaddr6, - false, - action); - i40iw_if_notify(iwdev, netdev, local_ipaddr6, false, - (action == I40IW_ARP_ADD) ? true : false); - break; - default: - break; - } - return NOTIFY_DONE; -} - -/** - * i40iw_net_event - system notifier for netevents - * @notifier: not used - * @event: event for notifier - * @ptr: neighbor - */ -int i40iw_net_event(struct notifier_block *notifier, unsigned long event, void *ptr) -{ - struct neighbour *neigh = ptr; - struct i40iw_device *iwdev; - struct i40iw_handler *iwhdl; - __be32 *p; - u32 local_ipaddr[4]; - - switch (event) { - case NETEVENT_NEIGH_UPDATE: - iwhdl = i40iw_find_netdev((struct net_device *)neigh->dev); - if (!iwhdl) - return NOTIFY_DONE; - iwdev = &iwhdl->device; - if (iwdev->init_state < IP_ADDR_REGISTERED || iwdev->closing) - return NOTIFY_DONE; - p = (__be32 *)neigh->primary_key; - i40iw_copy_ip_ntohl(local_ipaddr, p); - if (neigh->nud_state & NUD_VALID) { - i40iw_manage_arp_cache(iwdev, - neigh->ha, - local_ipaddr, - false, - I40IW_ARP_ADD); - - } else { - i40iw_manage_arp_cache(iwdev, - neigh->ha, - local_ipaddr, - false, - I40IW_ARP_DELETE); - } - break; - default: - break; - } - return NOTIFY_DONE; -} - -/** - * i40iw_netdevice_event - system notifier for netdev events - * @notifier: not used - * @event: event for notifier - * @ptr: netdev - */ -int i40iw_netdevice_event(struct notifier_block *notifier, - unsigned long event, - void *ptr) -{ - struct net_device *event_netdev; - struct net_device *netdev; - struct i40iw_device *iwdev; - struct i40iw_handler *hdl; - - event_netdev = netdev_notifier_info_to_dev(ptr); - - hdl = i40iw_find_netdev(event_netdev); - if (!hdl) - return NOTIFY_DONE; - - iwdev = &hdl->device; - if (iwdev->init_state < RDMA_DEV_REGISTERED || iwdev->closing) - return NOTIFY_DONE; - - netdev = iwdev->ldev->netdev; - if (netdev != event_netdev) - return NOTIFY_DONE; - - iwdev->iw_status = 1; - - switch (event) { - case NETDEV_DOWN: - iwdev->iw_status = 0; - fallthrough; - case NETDEV_UP: - i40iw_port_ibevent(iwdev); - break; - default: - break; - } - return NOTIFY_DONE; -} - -/** - * i40iw_get_cqp_request - get cqp struct - * @cqp: device cqp ptr - * @wait: cqp to be used in wait mode - */ -struct i40iw_cqp_request *i40iw_get_cqp_request(struct i40iw_cqp *cqp, bool wait) -{ - struct i40iw_cqp_request *cqp_request = NULL; - unsigned long flags; - - spin_lock_irqsave(&cqp->req_lock, flags); - if (!list_empty(&cqp->cqp_avail_reqs)) { - cqp_request = list_entry(cqp->cqp_avail_reqs.next, - struct i40iw_cqp_request, list); - list_del_init(&cqp_request->list); - } - spin_unlock_irqrestore(&cqp->req_lock, flags); - if (!cqp_request) { - cqp_request = kzalloc(sizeof(*cqp_request), GFP_ATOMIC); - if (cqp_request) { - cqp_request->dynamic = true; - INIT_LIST_HEAD(&cqp_request->list); - init_waitqueue_head(&cqp_request->waitq); - } - } - if (!cqp_request) { - i40iw_pr_err("CQP Request Fail: No Memory"); - return NULL; - } - - if (wait) { - atomic_set(&cqp_request->refcount, 2); - cqp_request->waiting = true; - } else { - atomic_set(&cqp_request->refcount, 1); - } - return cqp_request; -} - -/** - * i40iw_free_cqp_request - free cqp request - * @cqp: cqp ptr - * @cqp_request: to be put back in cqp list - */ -void i40iw_free_cqp_request(struct i40iw_cqp *cqp, struct i40iw_cqp_request *cqp_request) -{ - struct i40iw_device *iwdev = container_of(cqp, struct i40iw_device, cqp); - unsigned long flags; - - if (cqp_request->dynamic) { - kfree(cqp_request); - } else { - cqp_request->request_done = false; - cqp_request->callback_fcn = NULL; - cqp_request->waiting = false; - - spin_lock_irqsave(&cqp->req_lock, flags); - list_add_tail(&cqp_request->list, &cqp->cqp_avail_reqs); - spin_unlock_irqrestore(&cqp->req_lock, flags); - } - wake_up(&iwdev->close_wq); -} - -/** - * i40iw_put_cqp_request - dec ref count and free if 0 - * @cqp: cqp ptr - * @cqp_request: to be put back in cqp list - */ -void i40iw_put_cqp_request(struct i40iw_cqp *cqp, - struct i40iw_cqp_request *cqp_request) -{ - if (atomic_dec_and_test(&cqp_request->refcount)) - i40iw_free_cqp_request(cqp, cqp_request); -} - -/** - * i40iw_free_pending_cqp_request -free pending cqp request objs - * @cqp: cqp ptr - * @cqp_request: to be put back in cqp list - */ -static void i40iw_free_pending_cqp_request(struct i40iw_cqp *cqp, - struct i40iw_cqp_request *cqp_request) -{ - struct i40iw_device *iwdev = container_of(cqp, struct i40iw_device, cqp); - - if (cqp_request->waiting) { - cqp_request->compl_info.error = true; - cqp_request->request_done = true; - wake_up(&cqp_request->waitq); - } - i40iw_put_cqp_request(cqp, cqp_request); - wait_event_timeout(iwdev->close_wq, - !atomic_read(&cqp_request->refcount), - 1000); -} - -/** - * i40iw_cleanup_pending_cqp_op - clean-up cqp with no completions - * @iwdev: iwarp device - */ -void i40iw_cleanup_pending_cqp_op(struct i40iw_device *iwdev) -{ - struct i40iw_sc_dev *dev = &iwdev->sc_dev; - struct i40iw_cqp *cqp = &iwdev->cqp; - struct i40iw_cqp_request *cqp_request = NULL; - struct cqp_commands_info *pcmdinfo = NULL; - u32 i, pending_work, wqe_idx; - - pending_work = I40IW_RING_WORK_AVAILABLE(cqp->sc_cqp.sq_ring); - wqe_idx = I40IW_RING_GETCURRENT_TAIL(cqp->sc_cqp.sq_ring); - for (i = 0; i < pending_work; i++) { - cqp_request = (struct i40iw_cqp_request *)(unsigned long)cqp->scratch_array[wqe_idx]; - if (cqp_request) - i40iw_free_pending_cqp_request(cqp, cqp_request); - wqe_idx = (wqe_idx + 1) % I40IW_RING_GETSIZE(cqp->sc_cqp.sq_ring); - } - - while (!list_empty(&dev->cqp_cmd_head)) { - pcmdinfo = (struct cqp_commands_info *)i40iw_remove_head(&dev->cqp_cmd_head); - cqp_request = container_of(pcmdinfo, struct i40iw_cqp_request, info); - if (cqp_request) - i40iw_free_pending_cqp_request(cqp, cqp_request); - } -} - -/** - * i40iw_wait_event - wait for completion - * @iwdev: iwarp device - * @cqp_request: cqp request to wait - */ -static int i40iw_wait_event(struct i40iw_device *iwdev, - struct i40iw_cqp_request *cqp_request) -{ - struct cqp_commands_info *info = &cqp_request->info; - struct i40iw_cqp *iwcqp = &iwdev->cqp; - struct i40iw_cqp_timeout cqp_timeout; - bool cqp_error = false; - int err_code = 0; - memset(&cqp_timeout, 0, sizeof(cqp_timeout)); - cqp_timeout.compl_cqp_cmds = iwdev->sc_dev.cqp_cmd_stats[OP_COMPLETED_COMMANDS]; - do { - if (wait_event_timeout(cqp_request->waitq, - cqp_request->request_done, CQP_COMPL_WAIT_TIME)) - break; - - i40iw_check_cqp_progress(&cqp_timeout, &iwdev->sc_dev); - - if (cqp_timeout.count < CQP_TIMEOUT_THRESHOLD) - continue; - - i40iw_pr_err("error cqp command 0x%x timed out", info->cqp_cmd); - err_code = -ETIME; - if (!iwdev->reset) { - iwdev->reset = true; - i40iw_request_reset(iwdev); - } - goto done; - } while (1); - cqp_error = cqp_request->compl_info.error; - if (cqp_error) { - i40iw_pr_err("error cqp command 0x%x completion maj = 0x%x min=0x%x\n", - info->cqp_cmd, cqp_request->compl_info.maj_err_code, - cqp_request->compl_info.min_err_code); - err_code = -EPROTO; - goto done; - } -done: - i40iw_put_cqp_request(iwcqp, cqp_request); - return err_code; -} - -/** - * i40iw_handle_cqp_op - process cqp command - * @iwdev: iwarp device - * @cqp_request: cqp request to process - */ -enum i40iw_status_code i40iw_handle_cqp_op(struct i40iw_device *iwdev, - struct i40iw_cqp_request - *cqp_request) -{ - struct i40iw_sc_dev *dev = &iwdev->sc_dev; - enum i40iw_status_code status; - struct cqp_commands_info *info = &cqp_request->info; - int err_code = 0; - - if (iwdev->reset) { - i40iw_free_cqp_request(&iwdev->cqp, cqp_request); - return I40IW_ERR_CQP_COMPL_ERROR; - } - - status = i40iw_process_cqp_cmd(dev, info); - if (status) { - i40iw_pr_err("error cqp command 0x%x failed\n", info->cqp_cmd); - i40iw_free_cqp_request(&iwdev->cqp, cqp_request); - return status; - } - if (cqp_request->waiting) - err_code = i40iw_wait_event(iwdev, cqp_request); - if (err_code) - status = I40IW_ERR_CQP_COMPL_ERROR; - return status; -} - -/** - * i40iw_add_devusecount - add dev refcount - * @iwdev: dev for refcount - */ -void i40iw_add_devusecount(struct i40iw_device *iwdev) -{ - atomic64_inc(&iwdev->use_count); -} - -/** - * i40iw_rem_devusecount - decrement refcount for dev - * @iwdev: device - */ -void i40iw_rem_devusecount(struct i40iw_device *iwdev) -{ - if (!atomic64_dec_and_test(&iwdev->use_count)) - return; - wake_up(&iwdev->close_wq); -} - -/** - * i40iw_add_pdusecount - add pd refcount - * @iwpd: pd for refcount - */ -void i40iw_add_pdusecount(struct i40iw_pd *iwpd) -{ - atomic_inc(&iwpd->usecount); -} - -/** - * i40iw_rem_pdusecount - decrement refcount for pd and free if 0 - * @iwpd: pd for refcount - * @iwdev: iwarp device - */ -void i40iw_rem_pdusecount(struct i40iw_pd *iwpd, struct i40iw_device *iwdev) -{ - if (!atomic_dec_and_test(&iwpd->usecount)) - return; - i40iw_free_resource(iwdev, iwdev->allocated_pds, iwpd->sc_pd.pd_id); -} - -/** - * i40iw_qp_add_ref - add refcount for qp - * @ibqp: iqarp qp - */ -void i40iw_qp_add_ref(struct ib_qp *ibqp) -{ - struct i40iw_qp *iwqp = (struct i40iw_qp *)ibqp; - - refcount_inc(&iwqp->refcount); -} - -/** - * i40iw_qp_rem_ref - rem refcount for qp and free if 0 - * @ibqp: iqarp qp - */ -void i40iw_qp_rem_ref(struct ib_qp *ibqp) -{ - struct i40iw_qp *iwqp; - struct i40iw_device *iwdev; - u32 qp_num; - unsigned long flags; - - iwqp = to_iwqp(ibqp); - iwdev = iwqp->iwdev; - spin_lock_irqsave(&iwdev->qptable_lock, flags); - if (!refcount_dec_and_test(&iwqp->refcount)) { - spin_unlock_irqrestore(&iwdev->qptable_lock, flags); - return; - } - - qp_num = iwqp->ibqp.qp_num; - iwdev->qp_table[qp_num] = NULL; - spin_unlock_irqrestore(&iwdev->qptable_lock, flags); - complete(&iwqp->free_qp); - -} - -/** - * i40iw_get_qp - get qp address - * @device: iwarp device - * @qpn: qp number - */ -struct ib_qp *i40iw_get_qp(struct ib_device *device, int qpn) -{ - struct i40iw_device *iwdev = to_iwdev(device); - - if ((qpn < IW_FIRST_QPN) || (qpn >= iwdev->max_qp)) - return NULL; - - return &iwdev->qp_table[qpn]->ibqp; -} - -/** - * i40iw_debug_buf - print debug msg and buffer is mask set - * @dev: hardware control device structure - * @mask: mask to compare if to print debug buffer - * @desc: identifying string - * @buf: points buffer addr - * @size: saize of buffer to print - */ -void i40iw_debug_buf(struct i40iw_sc_dev *dev, - enum i40iw_debug_flag mask, - char *desc, - u64 *buf, - u32 size) -{ - u32 i; - - if (!(dev->debug_mask & mask)) - return; - i40iw_debug(dev, mask, "%s\n", desc); - i40iw_debug(dev, mask, "starting address virt=%p phy=%llxh\n", buf, - (unsigned long long)virt_to_phys(buf)); - - for (i = 0; i < size; i += 8) - i40iw_debug(dev, mask, "index %03d val: %016llx\n", i, buf[i / 8]); -} - -/** - * i40iw_get_hw_addr - return hw addr - * @par: points to shared dev - */ -u8 __iomem *i40iw_get_hw_addr(void *par) -{ - struct i40iw_sc_dev *dev = (struct i40iw_sc_dev *)par; - - return dev->hw->hw_addr; -} - -/** - * i40iw_remove_head - return head entry and remove from list - * @list: list for entry - */ -void *i40iw_remove_head(struct list_head *list) -{ - struct list_head *entry; - - if (list_empty(list)) - return NULL; - - entry = (void *)list->next; - list_del(entry); - return (void *)entry; -} - -/** - * i40iw_allocate_dma_mem - Memory alloc helper fn - * @hw: pointer to the HW structure - * @mem: ptr to mem struct to fill out - * @size: size of memory requested - * @alignment: what to align the allocation to - */ -enum i40iw_status_code i40iw_allocate_dma_mem(struct i40iw_hw *hw, - struct i40iw_dma_mem *mem, - u64 size, - u32 alignment) -{ - struct pci_dev *pcidev = hw->pcidev; - - if (!mem) - return I40IW_ERR_PARAM; - mem->size = ALIGN(size, alignment); - mem->va = dma_alloc_coherent(&pcidev->dev, mem->size, - (dma_addr_t *)&mem->pa, GFP_KERNEL); - if (!mem->va) - return I40IW_ERR_NO_MEMORY; - return 0; -} - -/** - * i40iw_free_dma_mem - Memory free helper fn - * @hw: pointer to the HW structure - * @mem: ptr to mem struct to free - */ -void i40iw_free_dma_mem(struct i40iw_hw *hw, struct i40iw_dma_mem *mem) -{ - struct pci_dev *pcidev = hw->pcidev; - - if (!mem || !mem->va) - return; - - dma_free_coherent(&pcidev->dev, mem->size, - mem->va, (dma_addr_t)mem->pa); - mem->va = NULL; -} - -/** - * i40iw_allocate_virt_mem - virtual memory alloc helper fn - * @hw: pointer to the HW structure - * @mem: ptr to mem struct to fill out - * @size: size of memory requested - */ -enum i40iw_status_code i40iw_allocate_virt_mem(struct i40iw_hw *hw, - struct i40iw_virt_mem *mem, - u32 size) -{ - if (!mem) - return I40IW_ERR_PARAM; - - mem->size = size; - mem->va = kzalloc(size, GFP_KERNEL); - - if (mem->va) - return 0; - else - return I40IW_ERR_NO_MEMORY; -} - -/** - * i40iw_free_virt_mem - virtual memory free helper fn - * @hw: pointer to the HW structure - * @mem: ptr to mem struct to free - */ -enum i40iw_status_code i40iw_free_virt_mem(struct i40iw_hw *hw, - struct i40iw_virt_mem *mem) -{ - if (!mem) - return I40IW_ERR_PARAM; - /* - * mem->va points to the parent of mem, so both mem and mem->va - * can not be touched once mem->va is freed - */ - kfree(mem->va); - return 0; -} - -/** - * i40iw_cqp_sds_cmd - create cqp command for sd - * @dev: hardware control device structure - * @sdinfo: information for sd cqp - * - */ -enum i40iw_status_code i40iw_cqp_sds_cmd(struct i40iw_sc_dev *dev, - struct i40iw_update_sds_info *sdinfo) -{ - enum i40iw_status_code status; - struct i40iw_cqp_request *cqp_request; - struct cqp_commands_info *cqp_info; - struct i40iw_device *iwdev = (struct i40iw_device *)dev->back_dev; - - cqp_request = i40iw_get_cqp_request(&iwdev->cqp, true); - if (!cqp_request) - return I40IW_ERR_NO_MEMORY; - cqp_info = &cqp_request->info; - memcpy(&cqp_info->in.u.update_pe_sds.info, sdinfo, - sizeof(cqp_info->in.u.update_pe_sds.info)); - cqp_info->cqp_cmd = OP_UPDATE_PE_SDS; - cqp_info->post_sq = 1; - cqp_info->in.u.update_pe_sds.dev = dev; - cqp_info->in.u.update_pe_sds.scratch = (uintptr_t)cqp_request; - status = i40iw_handle_cqp_op(iwdev, cqp_request); - if (status) - i40iw_pr_err("CQP-OP Update SD's fail"); - return status; -} - -/** - * i40iw_qp_suspend_resume - cqp command for suspend/resume - * @dev: hardware control device structure - * @qp: hardware control qp - * @suspend: flag if suspend or resume - */ -void i40iw_qp_suspend_resume(struct i40iw_sc_dev *dev, struct i40iw_sc_qp *qp, bool suspend) -{ - struct i40iw_device *iwdev = (struct i40iw_device *)dev->back_dev; - struct i40iw_cqp_request *cqp_request; - struct i40iw_sc_cqp *cqp = dev->cqp; - struct cqp_commands_info *cqp_info; - enum i40iw_status_code status; - - cqp_request = i40iw_get_cqp_request(&iwdev->cqp, false); - if (!cqp_request) - return; - - cqp_info = &cqp_request->info; - cqp_info->cqp_cmd = (suspend) ? OP_SUSPEND : OP_RESUME; - cqp_info->in.u.suspend_resume.cqp = cqp; - cqp_info->in.u.suspend_resume.qp = qp; - cqp_info->in.u.suspend_resume.scratch = (uintptr_t)cqp_request; - status = i40iw_handle_cqp_op(iwdev, cqp_request); - if (status) - i40iw_pr_err("CQP-OP QP Suspend/Resume fail"); -} - -/** - * i40iw_term_modify_qp - modify qp for term message - * @qp: hardware control qp - * @next_state: qp's next state - * @term: terminate code - * @term_len: length - */ -void i40iw_term_modify_qp(struct i40iw_sc_qp *qp, u8 next_state, u8 term, u8 term_len) -{ - struct i40iw_qp *iwqp; - - iwqp = (struct i40iw_qp *)qp->back_qp; - i40iw_next_iw_state(iwqp, next_state, 0, term, term_len); -}; - -/** - * i40iw_terminate_done - after terminate is completed - * @qp: hardware control qp - * @timeout_occurred: indicates if terminate timer expired - */ -void i40iw_terminate_done(struct i40iw_sc_qp *qp, int timeout_occurred) -{ - struct i40iw_qp *iwqp; - u32 next_iwarp_state = I40IW_QP_STATE_ERROR; - u8 hte = 0; - bool first_time; - unsigned long flags; - - iwqp = (struct i40iw_qp *)qp->back_qp; - spin_lock_irqsave(&iwqp->lock, flags); - if (iwqp->hte_added) { - iwqp->hte_added = 0; - hte = 1; - } - first_time = !(qp->term_flags & I40IW_TERM_DONE); - qp->term_flags |= I40IW_TERM_DONE; - spin_unlock_irqrestore(&iwqp->lock, flags); - if (first_time) { - if (!timeout_occurred) - i40iw_terminate_del_timer(qp); - else - next_iwarp_state = I40IW_QP_STATE_CLOSING; - - i40iw_next_iw_state(iwqp, next_iwarp_state, hte, 0, 0); - i40iw_cm_disconn(iwqp); - } -} - -/** - * i40iw_terminate_timeout - timeout happened - * @t: points to iwarp qp - */ -static void i40iw_terminate_timeout(struct timer_list *t) -{ - struct i40iw_qp *iwqp = from_timer(iwqp, t, terminate_timer); - struct i40iw_sc_qp *qp = (struct i40iw_sc_qp *)&iwqp->sc_qp; - - i40iw_terminate_done(qp, 1); - i40iw_qp_rem_ref(&iwqp->ibqp); -} - -/** - * i40iw_terminate_start_timer - start terminate timeout - * @qp: hardware control qp - */ -void i40iw_terminate_start_timer(struct i40iw_sc_qp *qp) -{ - struct i40iw_qp *iwqp; - - iwqp = (struct i40iw_qp *)qp->back_qp; - i40iw_qp_add_ref(&iwqp->ibqp); - timer_setup(&iwqp->terminate_timer, i40iw_terminate_timeout, 0); - iwqp->terminate_timer.expires = jiffies + HZ; - add_timer(&iwqp->terminate_timer); -} - -/** - * i40iw_terminate_del_timer - delete terminate timeout - * @qp: hardware control qp - */ -void i40iw_terminate_del_timer(struct i40iw_sc_qp *qp) -{ - struct i40iw_qp *iwqp; - - iwqp = (struct i40iw_qp *)qp->back_qp; - if (del_timer(&iwqp->terminate_timer)) - i40iw_qp_rem_ref(&iwqp->ibqp); -} - -/** - * i40iw_cqp_generic_worker - generic worker for cqp - * @work: work pointer - */ -static void i40iw_cqp_generic_worker(struct work_struct *work) -{ - struct i40iw_virtchnl_work_info *work_info = - &((struct virtchnl_work *)work)->work_info; - - if (work_info->worker_vf_dev) - work_info->callback_fcn(work_info->worker_vf_dev); -} - -/** - * i40iw_cqp_spawn_worker - spawn worket thread - * @dev: device struct pointer - * @work_info: work request info - * @iw_vf_idx: virtual function index - */ -void i40iw_cqp_spawn_worker(struct i40iw_sc_dev *dev, - struct i40iw_virtchnl_work_info *work_info, - u32 iw_vf_idx) -{ - struct virtchnl_work *work; - struct i40iw_device *iwdev = (struct i40iw_device *)dev->back_dev; - - work = &iwdev->virtchnl_w[iw_vf_idx]; - memcpy(&work->work_info, work_info, sizeof(*work_info)); - INIT_WORK(&work->work, i40iw_cqp_generic_worker); - queue_work(iwdev->virtchnl_wq, &work->work); -} - -/** - * i40iw_cqp_manage_hmc_fcn_worker - - * @work: work pointer for hmc info - */ -static void i40iw_cqp_manage_hmc_fcn_worker(struct work_struct *work) -{ - struct i40iw_cqp_request *cqp_request = - ((struct virtchnl_work *)work)->cqp_request; - struct i40iw_ccq_cqe_info ccq_cqe_info; - struct i40iw_hmc_fcn_info *hmcfcninfo = - &cqp_request->info.in.u.manage_hmc_pm.info; - struct i40iw_device *iwdev = - (struct i40iw_device *)cqp_request->info.in.u.manage_hmc_pm.dev->back_dev; - - ccq_cqe_info.cqp = NULL; - ccq_cqe_info.maj_err_code = cqp_request->compl_info.maj_err_code; - ccq_cqe_info.min_err_code = cqp_request->compl_info.min_err_code; - ccq_cqe_info.op_code = cqp_request->compl_info.op_code; - ccq_cqe_info.op_ret_val = cqp_request->compl_info.op_ret_val; - ccq_cqe_info.scratch = 0; - ccq_cqe_info.error = cqp_request->compl_info.error; - hmcfcninfo->callback_fcn(cqp_request->info.in.u.manage_hmc_pm.dev, - hmcfcninfo->cqp_callback_param, &ccq_cqe_info); - i40iw_put_cqp_request(&iwdev->cqp, cqp_request); -} - -/** - * i40iw_cqp_manage_hmc_fcn_callback - called function after cqp completion - * @cqp_request: cqp request info struct for hmc fun - * @unused: unused param of callback - */ -static void i40iw_cqp_manage_hmc_fcn_callback(struct i40iw_cqp_request *cqp_request, - u32 unused) -{ - struct virtchnl_work *work; - struct i40iw_hmc_fcn_info *hmcfcninfo = - &cqp_request->info.in.u.manage_hmc_pm.info; - struct i40iw_device *iwdev = - (struct i40iw_device *)cqp_request->info.in.u.manage_hmc_pm.dev-> - back_dev; - - if (hmcfcninfo && hmcfcninfo->callback_fcn) { - i40iw_debug(&iwdev->sc_dev, I40IW_DEBUG_HMC, "%s1\n", __func__); - atomic_inc(&cqp_request->refcount); - work = &iwdev->virtchnl_w[hmcfcninfo->iw_vf_idx]; - work->cqp_request = cqp_request; - INIT_WORK(&work->work, i40iw_cqp_manage_hmc_fcn_worker); - queue_work(iwdev->virtchnl_wq, &work->work); - i40iw_debug(&iwdev->sc_dev, I40IW_DEBUG_HMC, "%s2\n", __func__); - } else { - i40iw_debug(&iwdev->sc_dev, I40IW_DEBUG_HMC, "%s: Something wrong\n", __func__); - } -} - -/** - * i40iw_cqp_manage_hmc_fcn_cmd - issue cqp command to manage hmc - * @dev: hardware control device structure - * @hmcfcninfo: info for hmc - */ -enum i40iw_status_code i40iw_cqp_manage_hmc_fcn_cmd(struct i40iw_sc_dev *dev, - struct i40iw_hmc_fcn_info *hmcfcninfo) -{ - enum i40iw_status_code status; - struct i40iw_cqp_request *cqp_request; - struct cqp_commands_info *cqp_info; - struct i40iw_device *iwdev = (struct i40iw_device *)dev->back_dev; - - i40iw_debug(&iwdev->sc_dev, I40IW_DEBUG_HMC, "%s\n", __func__); - cqp_request = i40iw_get_cqp_request(&iwdev->cqp, false); - if (!cqp_request) - return I40IW_ERR_NO_MEMORY; - cqp_info = &cqp_request->info; - cqp_request->callback_fcn = i40iw_cqp_manage_hmc_fcn_callback; - cqp_request->param = hmcfcninfo; - memcpy(&cqp_info->in.u.manage_hmc_pm.info, hmcfcninfo, - sizeof(*hmcfcninfo)); - cqp_info->in.u.manage_hmc_pm.dev = dev; - cqp_info->cqp_cmd = OP_MANAGE_HMC_PM_FUNC_TABLE; - cqp_info->post_sq = 1; - cqp_info->in.u.manage_hmc_pm.scratch = (uintptr_t)cqp_request; - status = i40iw_handle_cqp_op(iwdev, cqp_request); - if (status) - i40iw_pr_err("CQP-OP Manage HMC fail"); - return status; -} - -/** - * i40iw_cqp_query_fpm_values_cmd - send cqp command for fpm - * @dev: function device struct - * @values_mem: buffer for fpm - * @hmc_fn_id: function id for fpm - */ -enum i40iw_status_code i40iw_cqp_query_fpm_values_cmd(struct i40iw_sc_dev *dev, - struct i40iw_dma_mem *values_mem, - u8 hmc_fn_id) -{ - enum i40iw_status_code status; - struct i40iw_cqp_request *cqp_request; - struct cqp_commands_info *cqp_info; - struct i40iw_device *iwdev = (struct i40iw_device *)dev->back_dev; - - cqp_request = i40iw_get_cqp_request(&iwdev->cqp, true); - if (!cqp_request) - return I40IW_ERR_NO_MEMORY; - cqp_info = &cqp_request->info; - cqp_request->param = NULL; - cqp_info->in.u.query_fpm_values.cqp = dev->cqp; - cqp_info->in.u.query_fpm_values.fpm_values_pa = values_mem->pa; - cqp_info->in.u.query_fpm_values.fpm_values_va = values_mem->va; - cqp_info->in.u.query_fpm_values.hmc_fn_id = hmc_fn_id; - cqp_info->cqp_cmd = OP_QUERY_FPM_VALUES; - cqp_info->post_sq = 1; - cqp_info->in.u.query_fpm_values.scratch = (uintptr_t)cqp_request; - status = i40iw_handle_cqp_op(iwdev, cqp_request); - if (status) - i40iw_pr_err("CQP-OP Query FPM fail"); - return status; -} - -/** - * i40iw_cqp_commit_fpm_values_cmd - commit fpm values in hw - * @dev: hardware control device structure - * @values_mem: buffer with fpm values - * @hmc_fn_id: function id for fpm - */ -enum i40iw_status_code i40iw_cqp_commit_fpm_values_cmd(struct i40iw_sc_dev *dev, - struct i40iw_dma_mem *values_mem, - u8 hmc_fn_id) -{ - enum i40iw_status_code status; - struct i40iw_cqp_request *cqp_request; - struct cqp_commands_info *cqp_info; - struct i40iw_device *iwdev = (struct i40iw_device *)dev->back_dev; - - cqp_request = i40iw_get_cqp_request(&iwdev->cqp, true); - if (!cqp_request) - return I40IW_ERR_NO_MEMORY; - cqp_info = &cqp_request->info; - cqp_request->param = NULL; - cqp_info->in.u.commit_fpm_values.cqp = dev->cqp; - cqp_info->in.u.commit_fpm_values.fpm_values_pa = values_mem->pa; - cqp_info->in.u.commit_fpm_values.fpm_values_va = values_mem->va; - cqp_info->in.u.commit_fpm_values.hmc_fn_id = hmc_fn_id; - cqp_info->cqp_cmd = OP_COMMIT_FPM_VALUES; - cqp_info->post_sq = 1; - cqp_info->in.u.commit_fpm_values.scratch = (uintptr_t)cqp_request; - status = i40iw_handle_cqp_op(iwdev, cqp_request); - if (status) - i40iw_pr_err("CQP-OP Commit FPM fail"); - return status; -} - -/** - * i40iw_vf_wait_vchnl_resp - wait for channel msg - * @dev: function's device struct - */ -enum i40iw_status_code i40iw_vf_wait_vchnl_resp(struct i40iw_sc_dev *dev) -{ - struct i40iw_device *iwdev = dev->back_dev; - int timeout_ret; - - i40iw_debug(dev, I40IW_DEBUG_VIRT, "%s[%u] dev %p, iwdev %p\n", - __func__, __LINE__, dev, iwdev); - - atomic_set(&iwdev->vchnl_msgs, 2); - timeout_ret = wait_event_timeout(iwdev->vchnl_waitq, - (atomic_read(&iwdev->vchnl_msgs) == 1), - I40IW_VCHNL_EVENT_TIMEOUT); - atomic_dec(&iwdev->vchnl_msgs); - if (!timeout_ret) { - i40iw_pr_err("virt channel completion timeout = 0x%x\n", timeout_ret); - atomic_set(&iwdev->vchnl_msgs, 0); - dev->vchnl_up = false; - return I40IW_ERR_TIMEOUT; - } - wake_up(&dev->vf_reqs); - return 0; -} - -/** - * i40iw_cqp_cq_create_cmd - create a cq for the cqp - * @dev: device pointer - * @cq: pointer to created cq - */ -enum i40iw_status_code i40iw_cqp_cq_create_cmd(struct i40iw_sc_dev *dev, - struct i40iw_sc_cq *cq) -{ - struct i40iw_device *iwdev = (struct i40iw_device *)dev->back_dev; - struct i40iw_cqp *iwcqp = &iwdev->cqp; - struct i40iw_cqp_request *cqp_request; - struct cqp_commands_info *cqp_info; - enum i40iw_status_code status; - - cqp_request = i40iw_get_cqp_request(iwcqp, true); - if (!cqp_request) - return I40IW_ERR_NO_MEMORY; - - cqp_info = &cqp_request->info; - cqp_info->cqp_cmd = OP_CQ_CREATE; - cqp_info->post_sq = 1; - cqp_info->in.u.cq_create.cq = cq; - cqp_info->in.u.cq_create.scratch = (uintptr_t)cqp_request; - status = i40iw_handle_cqp_op(iwdev, cqp_request); - if (status) - i40iw_pr_err("CQP-OP Create QP fail"); - - return status; -} - -/** - * i40iw_cqp_qp_create_cmd - create a qp for the cqp - * @dev: device pointer - * @qp: pointer to created qp - */ -enum i40iw_status_code i40iw_cqp_qp_create_cmd(struct i40iw_sc_dev *dev, - struct i40iw_sc_qp *qp) -{ - struct i40iw_device *iwdev = (struct i40iw_device *)dev->back_dev; - struct i40iw_cqp *iwcqp = &iwdev->cqp; - struct i40iw_cqp_request *cqp_request; - struct cqp_commands_info *cqp_info; - struct i40iw_create_qp_info *qp_info; - enum i40iw_status_code status; - - cqp_request = i40iw_get_cqp_request(iwcqp, true); - if (!cqp_request) - return I40IW_ERR_NO_MEMORY; - - cqp_info = &cqp_request->info; - qp_info = &cqp_request->info.in.u.qp_create.info; - - memset(qp_info, 0, sizeof(*qp_info)); - - qp_info->cq_num_valid = true; - qp_info->next_iwarp_state = I40IW_QP_STATE_RTS; - - cqp_info->cqp_cmd = OP_QP_CREATE; - cqp_info->post_sq = 1; - cqp_info->in.u.qp_create.qp = qp; - cqp_info->in.u.qp_create.scratch = (uintptr_t)cqp_request; - status = i40iw_handle_cqp_op(iwdev, cqp_request); - if (status) - i40iw_pr_err("CQP-OP QP create fail"); - return status; -} - -/** - * i40iw_cqp_cq_destroy_cmd - destroy the cqp cq - * @dev: device pointer - * @cq: pointer to cq - */ -void i40iw_cqp_cq_destroy_cmd(struct i40iw_sc_dev *dev, struct i40iw_sc_cq *cq) -{ - struct i40iw_device *iwdev = (struct i40iw_device *)dev->back_dev; - - i40iw_cq_wq_destroy(iwdev, cq); -} - -/** - * i40iw_cqp_qp_destroy_cmd - destroy the cqp - * @dev: device pointer - * @qp: pointer to qp - */ -void i40iw_cqp_qp_destroy_cmd(struct i40iw_sc_dev *dev, struct i40iw_sc_qp *qp) -{ - struct i40iw_device *iwdev = (struct i40iw_device *)dev->back_dev; - struct i40iw_cqp *iwcqp = &iwdev->cqp; - struct i40iw_cqp_request *cqp_request; - struct cqp_commands_info *cqp_info; - enum i40iw_status_code status; - - cqp_request = i40iw_get_cqp_request(iwcqp, true); - if (!cqp_request) - return; - - cqp_info = &cqp_request->info; - memset(cqp_info, 0, sizeof(*cqp_info)); - - cqp_info->cqp_cmd = OP_QP_DESTROY; - cqp_info->post_sq = 1; - cqp_info->in.u.qp_destroy.qp = qp; - cqp_info->in.u.qp_destroy.scratch = (uintptr_t)cqp_request; - cqp_info->in.u.qp_destroy.remove_hash_idx = true; - status = i40iw_handle_cqp_op(iwdev, cqp_request); - if (status) - i40iw_pr_err("CQP QP_DESTROY fail"); -} - - -/** - * i40iw_ieq_mpa_crc_ae - generate AE for crc error - * @dev: hardware control device structure - * @qp: hardware control qp - */ -void i40iw_ieq_mpa_crc_ae(struct i40iw_sc_dev *dev, struct i40iw_sc_qp *qp) -{ - struct i40iw_gen_ae_info info; - struct i40iw_device *iwdev = (struct i40iw_device *)dev->back_dev; - - i40iw_debug(dev, I40IW_DEBUG_AEQ, "%s entered\n", __func__); - info.ae_code = I40IW_AE_LLP_RECEIVED_MPA_CRC_ERROR; - info.ae_source = I40IW_AE_SOURCE_RQ; - i40iw_gen_ae(iwdev, qp, &info, false); -} - -/** - * i40iw_init_hash_desc - initialize hash for crc calculation - * @desc: cryption type - */ -enum i40iw_status_code i40iw_init_hash_desc(struct shash_desc **desc) -{ - struct crypto_shash *tfm; - struct shash_desc *tdesc; - - tfm = crypto_alloc_shash("crc32c", 0, 0); - if (IS_ERR(tfm)) - return I40IW_ERR_MPA_CRC; - - tdesc = kzalloc(sizeof(*tdesc) + crypto_shash_descsize(tfm), - GFP_KERNEL); - if (!tdesc) { - crypto_free_shash(tfm); - return I40IW_ERR_MPA_CRC; - } - tdesc->tfm = tfm; - *desc = tdesc; - - return 0; -} - -/** - * i40iw_free_hash_desc - free hash desc - * @desc: to be freed - */ -void i40iw_free_hash_desc(struct shash_desc *desc) -{ - if (desc) { - crypto_free_shash(desc->tfm); - kfree(desc); - } -} - -/** - * i40iw_alloc_query_fpm_buf - allocate buffer for fpm - * @dev: hardware control device structure - * @mem: buffer ptr for fpm to be allocated - * @return: memory allocation status - */ -enum i40iw_status_code i40iw_alloc_query_fpm_buf(struct i40iw_sc_dev *dev, - struct i40iw_dma_mem *mem) -{ - enum i40iw_status_code status; - struct i40iw_device *iwdev = (struct i40iw_device *)dev->back_dev; - - status = i40iw_obj_aligned_mem(iwdev, mem, I40IW_QUERY_FPM_BUF_SIZE, - I40IW_FPM_QUERY_BUF_ALIGNMENT_MASK); - return status; -} - -/** - * i40iw_ieq_check_mpacrc - check if mpa crc is OK - * @desc: desc for hash - * @addr: address of buffer for crc - * @length: length of buffer - * @value: value to be compared - */ -enum i40iw_status_code i40iw_ieq_check_mpacrc(struct shash_desc *desc, - void *addr, - u32 length, - u32 value) -{ - u32 crc = 0; - int ret; - enum i40iw_status_code ret_code = 0; - - crypto_shash_init(desc); - ret = crypto_shash_update(desc, addr, length); - if (!ret) - crypto_shash_final(desc, (u8 *)&crc); - if (crc != value) { - i40iw_pr_err("mpa crc check fail\n"); - ret_code = I40IW_ERR_MPA_CRC; - } - return ret_code; -} - -/** - * i40iw_ieq_get_qp - get qp based on quad in puda buffer - * @dev: hardware control device structure - * @buf: receive puda buffer on exception q - */ -struct i40iw_sc_qp *i40iw_ieq_get_qp(struct i40iw_sc_dev *dev, - struct i40iw_puda_buf *buf) -{ - struct i40iw_device *iwdev = (struct i40iw_device *)dev->back_dev; - struct i40iw_qp *iwqp; - struct i40iw_cm_node *cm_node; - u32 loc_addr[4], rem_addr[4]; - u16 loc_port, rem_port; - struct ipv6hdr *ip6h; - struct iphdr *iph = (struct iphdr *)buf->iph; - struct tcphdr *tcph = (struct tcphdr *)buf->tcph; - - if (iph->version == 4) { - memset(loc_addr, 0, sizeof(loc_addr)); - loc_addr[0] = ntohl(iph->daddr); - memset(rem_addr, 0, sizeof(rem_addr)); - rem_addr[0] = ntohl(iph->saddr); - } else { - ip6h = (struct ipv6hdr *)buf->iph; - i40iw_copy_ip_ntohl(loc_addr, ip6h->daddr.in6_u.u6_addr32); - i40iw_copy_ip_ntohl(rem_addr, ip6h->saddr.in6_u.u6_addr32); - } - loc_port = ntohs(tcph->dest); - rem_port = ntohs(tcph->source); - - cm_node = i40iw_find_node(&iwdev->cm_core, rem_port, rem_addr, loc_port, - loc_addr, false, true); - if (!cm_node) - return NULL; - iwqp = cm_node->iwqp; - return &iwqp->sc_qp; -} - -/** - * i40iw_ieq_update_tcpip_info - update tcpip in the buffer - * @buf: puda to update - * @length: length of buffer - * @seqnum: seq number for tcp - */ -void i40iw_ieq_update_tcpip_info(struct i40iw_puda_buf *buf, u16 length, u32 seqnum) -{ - struct tcphdr *tcph; - struct iphdr *iph; - u16 iphlen; - u16 packetsize; - u8 *addr = (u8 *)buf->mem.va; - - iphlen = (buf->ipv4) ? 20 : 40; - iph = (struct iphdr *)(addr + buf->maclen); - tcph = (struct tcphdr *)(addr + buf->maclen + iphlen); - packetsize = length + buf->tcphlen + iphlen; - - iph->tot_len = htons(packetsize); - tcph->seq = htonl(seqnum); -} - -/** - * i40iw_puda_get_tcpip_info - get tcpip info from puda buffer - * @info: to get information - * @buf: puda buffer - */ -enum i40iw_status_code i40iw_puda_get_tcpip_info(struct i40iw_puda_completion_info *info, - struct i40iw_puda_buf *buf) -{ - struct iphdr *iph; - struct ipv6hdr *ip6h; - struct tcphdr *tcph; - u16 iphlen; - u16 pkt_len; - u8 *mem = (u8 *)buf->mem.va; - struct ethhdr *ethh = (struct ethhdr *)buf->mem.va; - - if (ethh->h_proto == htons(0x8100)) { - info->vlan_valid = true; - buf->vlan_id = ntohs(((struct vlan_ethhdr *)ethh)->h_vlan_TCI) & VLAN_VID_MASK; - } - buf->maclen = (info->vlan_valid) ? 18 : 14; - iphlen = (info->l3proto) ? 40 : 20; - buf->ipv4 = (info->l3proto) ? false : true; - buf->iph = mem + buf->maclen; - iph = (struct iphdr *)buf->iph; - - buf->tcph = buf->iph + iphlen; - tcph = (struct tcphdr *)buf->tcph; - - if (buf->ipv4) { - pkt_len = ntohs(iph->tot_len); - } else { - ip6h = (struct ipv6hdr *)buf->iph; - pkt_len = ntohs(ip6h->payload_len) + iphlen; - } - - buf->totallen = pkt_len + buf->maclen; - - if (info->payload_len < buf->totallen) { - i40iw_pr_err("payload_len = 0x%x totallen expected0x%x\n", - info->payload_len, buf->totallen); - return I40IW_ERR_INVALID_SIZE; - } - - buf->tcphlen = (tcph->doff) << 2; - buf->datalen = pkt_len - iphlen - buf->tcphlen; - buf->data = (buf->datalen) ? buf->tcph + buf->tcphlen : NULL; - buf->hdrlen = buf->maclen + iphlen + buf->tcphlen; - buf->seqnum = ntohl(tcph->seq); - return 0; -} - -/** - * i40iw_hw_stats_timeout - Stats timer-handler which updates all HW stats - * @t: Timer context containing pointer to the vsi structure - */ -static void i40iw_hw_stats_timeout(struct timer_list *t) -{ - struct i40iw_vsi_pestat *pf_devstat = from_timer(pf_devstat, t, - stats_timer); - struct i40iw_sc_vsi *sc_vsi = pf_devstat->vsi; - struct i40iw_sc_dev *pf_dev = sc_vsi->dev; - struct i40iw_vsi_pestat *vf_devstat = NULL; - u16 iw_vf_idx; - unsigned long flags; - - /*PF*/ - i40iw_hw_stats_read_all(pf_devstat, &pf_devstat->hw_stats); - - for (iw_vf_idx = 0; iw_vf_idx < I40IW_MAX_PE_ENABLED_VF_COUNT; iw_vf_idx++) { - spin_lock_irqsave(&pf_devstat->lock, flags); - if (pf_dev->vf_dev[iw_vf_idx]) { - if (pf_dev->vf_dev[iw_vf_idx]->stats_initialized) { - vf_devstat = &pf_dev->vf_dev[iw_vf_idx]->pestat; - i40iw_hw_stats_read_all(vf_devstat, &vf_devstat->hw_stats); - } - } - spin_unlock_irqrestore(&pf_devstat->lock, flags); - } - - mod_timer(&pf_devstat->stats_timer, - jiffies + msecs_to_jiffies(STATS_TIMER_DELAY)); -} - -/** - * i40iw_hw_stats_start_timer - Start periodic stats timer - * @vsi: pointer to the vsi structure - */ -void i40iw_hw_stats_start_timer(struct i40iw_sc_vsi *vsi) -{ - struct i40iw_vsi_pestat *devstat = vsi->pestat; - - timer_setup(&devstat->stats_timer, i40iw_hw_stats_timeout, 0); - mod_timer(&devstat->stats_timer, - jiffies + msecs_to_jiffies(STATS_TIMER_DELAY)); -} - -/** - * i40iw_hw_stats_stop_timer - Delete periodic stats timer - * @vsi: pointer to the vsi structure - */ -void i40iw_hw_stats_stop_timer(struct i40iw_sc_vsi *vsi) -{ - struct i40iw_vsi_pestat *devstat = vsi->pestat; - - del_timer_sync(&devstat->stats_timer); -} diff --git a/drivers/infiniband/hw/i40iw/i40iw_verbs.c b/drivers/infiniband/hw/i40iw/i40iw_verbs.c deleted file mode 100644 index b876d722fcc8..000000000000 --- a/drivers/infiniband/hw/i40iw/i40iw_verbs.c +++ /dev/null @@ -1,2652 +0,0 @@ -/******************************************************************************* -* -* Copyright (c) 2015-2016 Intel Corporation. All rights reserved. -* -* This software is available to you under a choice of one of two -* licenses. You may choose to be licensed under the terms of the GNU -* General Public License (GPL) Version 2, available from the file -* COPYING in the main directory of this source tree, or the -* OpenFabrics.org BSD license below: -* -* Redistribution and use in source and binary forms, with or -* without modification, are permitted provided that the following -* conditions are met: -* -* - Redistributions of source code must retain the above -* copyright notice, this list of conditions and the following -* disclaimer. -* -* - Redistributions in binary form must reproduce the above -* copyright notice, this list of conditions and the following -* disclaimer in the documentation and/or other materials -* provided with the distribution. -* -* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS -* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN -* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -* SOFTWARE. -* -*******************************************************************************/ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "i40iw.h" - -/** - * i40iw_query_device - get device attributes - * @ibdev: device pointer from stack - * @props: returning device attributes - * @udata: user data - */ -static int i40iw_query_device(struct ib_device *ibdev, - struct ib_device_attr *props, - struct ib_udata *udata) -{ - struct i40iw_device *iwdev = to_iwdev(ibdev); - - if (udata->inlen || udata->outlen) - return -EINVAL; - memset(props, 0, sizeof(*props)); - ether_addr_copy((u8 *)&props->sys_image_guid, iwdev->netdev->dev_addr); - props->fw_ver = i40iw_fw_major_ver(&iwdev->sc_dev) << 32 | - i40iw_fw_minor_ver(&iwdev->sc_dev); - props->device_cap_flags = iwdev->device_cap_flags; - props->vendor_id = iwdev->ldev->pcidev->vendor; - props->vendor_part_id = iwdev->ldev->pcidev->device; - props->hw_ver = (u32)iwdev->sc_dev.hw_rev; - props->max_mr_size = I40IW_MAX_OUTBOUND_MESSAGE_SIZE; - props->max_qp = iwdev->max_qp - iwdev->used_qps; - props->max_qp_wr = I40IW_MAX_QP_WRS; - props->max_send_sge = I40IW_MAX_WQ_FRAGMENT_COUNT; - props->max_recv_sge = I40IW_MAX_WQ_FRAGMENT_COUNT; - props->max_cq = iwdev->max_cq - iwdev->used_cqs; - props->max_cqe = iwdev->max_cqe; - props->max_mr = iwdev->max_mr - iwdev->used_mrs; - props->max_pd = iwdev->max_pd - iwdev->used_pds; - props->max_sge_rd = I40IW_MAX_SGE_RD; - props->max_qp_rd_atom = I40IW_MAX_IRD_SIZE; - props->max_qp_init_rd_atom = props->max_qp_rd_atom; - props->atomic_cap = IB_ATOMIC_NONE; - props->max_fast_reg_page_list_len = I40IW_MAX_PAGES_PER_FMR; - return 0; -} - -/** - * i40iw_query_port - get port attrubutes - * @ibdev: device pointer from stack - * @port: port number for query - * @props: returning device attributes - */ -static int i40iw_query_port(struct ib_device *ibdev, - u32 port, - struct ib_port_attr *props) -{ - props->lid = 1; - props->port_cap_flags = IB_PORT_CM_SUP | IB_PORT_REINIT_SUP | - IB_PORT_VENDOR_CLASS_SUP | IB_PORT_BOOT_MGMT_SUP; - props->gid_tbl_len = 1; - props->active_width = IB_WIDTH_4X; - props->active_speed = 1; - props->max_msg_sz = I40IW_MAX_OUTBOUND_MESSAGE_SIZE; - return 0; -} - -/** - * i40iw_alloc_ucontext - Allocate the user context data structure - * @uctx: Uverbs context pointer from stack - * @udata: user data - * - * This keeps track of all objects associated with a particular - * user-mode client. - */ -static int i40iw_alloc_ucontext(struct ib_ucontext *uctx, - struct ib_udata *udata) -{ - struct ib_device *ibdev = uctx->device; - struct i40iw_device *iwdev = to_iwdev(ibdev); - struct i40iw_alloc_ucontext_req req; - struct i40iw_alloc_ucontext_resp uresp = {}; - struct i40iw_ucontext *ucontext = to_ucontext(uctx); - - if (ib_copy_from_udata(&req, udata, sizeof(req))) - return -EINVAL; - - if (req.userspace_ver < 4 || req.userspace_ver > I40IW_ABI_VER) { - i40iw_pr_err("Unsupported provider library version %u.\n", req.userspace_ver); - return -EINVAL; - } - - uresp.max_qps = iwdev->max_qp; - uresp.max_pds = iwdev->max_pd; - uresp.wq_size = iwdev->max_qp_wr * 2; - uresp.kernel_ver = req.userspace_ver; - - ucontext->iwdev = iwdev; - ucontext->abi_ver = req.userspace_ver; - - if (ib_copy_to_udata(udata, &uresp, sizeof(uresp))) - return -EFAULT; - - INIT_LIST_HEAD(&ucontext->cq_reg_mem_list); - spin_lock_init(&ucontext->cq_reg_mem_list_lock); - INIT_LIST_HEAD(&ucontext->qp_reg_mem_list); - spin_lock_init(&ucontext->qp_reg_mem_list_lock); - - return 0; -} - -/** - * i40iw_dealloc_ucontext - deallocate the user context data structure - * @context: user context created during alloc - */ -static void i40iw_dealloc_ucontext(struct ib_ucontext *context) -{ - return; -} - -/** - * i40iw_mmap - user memory map - * @context: context created during alloc - * @vma: kernel info for user memory map - */ -static int i40iw_mmap(struct ib_ucontext *context, struct vm_area_struct *vma) -{ - struct i40iw_ucontext *ucontext = to_ucontext(context); - u64 dbaddr; - - if (vma->vm_pgoff || vma->vm_end - vma->vm_start != PAGE_SIZE) - return -EINVAL; - - dbaddr = I40IW_DB_ADDR_OFFSET + pci_resource_start(ucontext->iwdev->ldev->pcidev, 0); - - return rdma_user_mmap_io(context, vma, dbaddr >> PAGE_SHIFT, PAGE_SIZE, - pgprot_noncached(vma->vm_page_prot), NULL); -} - -/** - * i40iw_alloc_pd - allocate protection domain - * @pd: PD pointer - * @udata: user data - */ -static int i40iw_alloc_pd(struct ib_pd *pd, struct ib_udata *udata) -{ - struct i40iw_pd *iwpd = to_iwpd(pd); - struct i40iw_device *iwdev = to_iwdev(pd->device); - struct i40iw_sc_dev *dev = &iwdev->sc_dev; - struct i40iw_alloc_pd_resp uresp; - struct i40iw_sc_pd *sc_pd; - u32 pd_id = 0; - int err; - - if (iwdev->closing) - return -ENODEV; - - err = i40iw_alloc_resource(iwdev, iwdev->allocated_pds, - iwdev->max_pd, &pd_id, &iwdev->next_pd); - if (err) { - i40iw_pr_err("alloc resource failed\n"); - return err; - } - - sc_pd = &iwpd->sc_pd; - - if (udata) { - struct i40iw_ucontext *ucontext = rdma_udata_to_drv_context( - udata, struct i40iw_ucontext, ibucontext); - dev->iw_pd_ops->pd_init(dev, sc_pd, pd_id, ucontext->abi_ver); - memset(&uresp, 0, sizeof(uresp)); - uresp.pd_id = pd_id; - if (ib_copy_to_udata(udata, &uresp, sizeof(uresp))) { - err = -EFAULT; - goto error; - } - } else { - dev->iw_pd_ops->pd_init(dev, sc_pd, pd_id, -1); - } - - i40iw_add_pdusecount(iwpd); - return 0; - -error: - i40iw_free_resource(iwdev, iwdev->allocated_pds, pd_id); - return err; -} - -/** - * i40iw_dealloc_pd - deallocate pd - * @ibpd: ptr of pd to be deallocated - * @udata: user data or null for kernel object - */ -static int i40iw_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) -{ - struct i40iw_pd *iwpd = to_iwpd(ibpd); - struct i40iw_device *iwdev = to_iwdev(ibpd->device); - - i40iw_rem_pdusecount(iwpd, iwdev); - return 0; -} - -/** - * i40iw_get_pbl - Retrieve pbl from a list given a virtual - * address - * @va: user virtual address - * @pbl_list: pbl list to search in (QP's or CQ's) - */ -static struct i40iw_pbl *i40iw_get_pbl(unsigned long va, - struct list_head *pbl_list) -{ - struct i40iw_pbl *iwpbl; - - list_for_each_entry(iwpbl, pbl_list, list) { - if (iwpbl->user_base == va) { - iwpbl->on_list = false; - list_del(&iwpbl->list); - return iwpbl; - } - } - return NULL; -} - -/** - * i40iw_free_qp_resources - free up memory resources for qp - * @iwqp: qp ptr (user or kernel) - */ -void i40iw_free_qp_resources(struct i40iw_qp *iwqp) -{ - struct i40iw_pbl *iwpbl = &iwqp->iwpbl; - struct i40iw_device *iwdev = iwqp->iwdev; - u32 qp_num = iwqp->ibqp.qp_num; - - i40iw_ieq_cleanup_qp(iwdev->vsi.ieq, &iwqp->sc_qp); - if (qp_num) - i40iw_free_resource(iwdev, iwdev->allocated_qps, qp_num); - if (iwpbl->pbl_allocated) - i40iw_free_pble(iwdev->pble_rsrc, &iwpbl->pble_alloc); - i40iw_free_dma_mem(iwdev->sc_dev.hw, &iwqp->q2_ctx_mem); - i40iw_free_dma_mem(iwdev->sc_dev.hw, &iwqp->kqp.dma_mem); - kfree(iwqp->kqp.wrid_mem); - iwqp->kqp.wrid_mem = NULL; - kfree(iwqp); -} - -/** - * i40iw_clean_cqes - clean cq entries for qp - * @iwqp: qp ptr (user or kernel) - * @iwcq: cq ptr - */ -static void i40iw_clean_cqes(struct i40iw_qp *iwqp, struct i40iw_cq *iwcq) -{ - struct i40iw_cq_uk *ukcq = &iwcq->sc_cq.cq_uk; - - ukcq->ops.iw_cq_clean(&iwqp->sc_qp.qp_uk, ukcq); -} - -/** - * i40iw_destroy_qp - destroy qp - * @ibqp: qp's ib pointer also to get to device's qp address - * @udata: user data - */ -static int i40iw_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata) -{ - struct i40iw_qp *iwqp = to_iwqp(ibqp); - struct ib_qp_attr attr; - struct i40iw_device *iwdev = iwqp->iwdev; - - memset(&attr, 0, sizeof(attr)); - - iwqp->destroyed = 1; - - if (iwqp->ibqp_state >= IB_QPS_INIT && iwqp->ibqp_state < IB_QPS_RTS) - i40iw_next_iw_state(iwqp, I40IW_QP_STATE_ERROR, 0, 0, 0); - - if (!iwqp->user_mode) { - if (iwqp->iwscq) { - i40iw_clean_cqes(iwqp, iwqp->iwscq); - if (iwqp->iwrcq != iwqp->iwscq) - i40iw_clean_cqes(iwqp, iwqp->iwrcq); - } - } - - attr.qp_state = IB_QPS_ERR; - i40iw_modify_qp(&iwqp->ibqp, &attr, IB_QP_STATE, NULL); - i40iw_qp_rem_ref(&iwqp->ibqp); - wait_for_completion(&iwqp->free_qp); - i40iw_cqp_qp_destroy_cmd(&iwdev->sc_dev, &iwqp->sc_qp); - i40iw_rem_pdusecount(iwqp->iwpd, iwdev); - i40iw_free_qp_resources(iwqp); - i40iw_rem_devusecount(iwdev); - - return 0; -} - -/** - * i40iw_setup_virt_qp - setup for allocation of virtual qp - * @iwdev: iwarp device - * @iwqp: qp ptr - * @init_info: initialize info to return - */ -static int i40iw_setup_virt_qp(struct i40iw_device *iwdev, - struct i40iw_qp *iwqp, - struct i40iw_qp_init_info *init_info) -{ - struct i40iw_pbl *iwpbl = &iwqp->iwpbl; - struct i40iw_qp_mr *qpmr = &iwpbl->qp_mr; - - iwqp->page = qpmr->sq_page; - init_info->shadow_area_pa = cpu_to_le64(qpmr->shadow); - if (iwpbl->pbl_allocated) { - init_info->virtual_map = true; - init_info->sq_pa = qpmr->sq_pbl.idx; - init_info->rq_pa = qpmr->rq_pbl.idx; - } else { - init_info->sq_pa = qpmr->sq_pbl.addr; - init_info->rq_pa = qpmr->rq_pbl.addr; - } - return 0; -} - -/** - * i40iw_setup_kmode_qp - setup initialization for kernel mode qp - * @iwdev: iwarp device - * @iwqp: qp ptr (user or kernel) - * @info: initialize info to return - */ -static int i40iw_setup_kmode_qp(struct i40iw_device *iwdev, - struct i40iw_qp *iwqp, - struct i40iw_qp_init_info *info) -{ - struct i40iw_dma_mem *mem = &iwqp->kqp.dma_mem; - u32 sqdepth, rqdepth; - u8 sqshift; - u32 size; - enum i40iw_status_code status; - struct i40iw_qp_uk_init_info *ukinfo = &info->qp_uk_init_info; - - i40iw_get_wqe_shift(ukinfo->max_sq_frag_cnt, ukinfo->max_inline_data, &sqshift); - status = i40iw_get_sqdepth(ukinfo->sq_size, sqshift, &sqdepth); - if (status) - return -ENOMEM; - - status = i40iw_get_rqdepth(ukinfo->rq_size, I40IW_MAX_RQ_WQE_SHIFT, &rqdepth); - if (status) - return -ENOMEM; - - size = sqdepth * sizeof(struct i40iw_sq_uk_wr_trk_info) + (rqdepth << 3); - iwqp->kqp.wrid_mem = kzalloc(size, GFP_KERNEL); - - ukinfo->sq_wrtrk_array = (struct i40iw_sq_uk_wr_trk_info *)iwqp->kqp.wrid_mem; - if (!ukinfo->sq_wrtrk_array) - return -ENOMEM; - - ukinfo->rq_wrid_array = (u64 *)&ukinfo->sq_wrtrk_array[sqdepth]; - - size = (sqdepth + rqdepth) * I40IW_QP_WQE_MIN_SIZE; - size += (I40IW_SHADOW_AREA_SIZE << 3); - - status = i40iw_allocate_dma_mem(iwdev->sc_dev.hw, mem, size, 256); - if (status) { - kfree(ukinfo->sq_wrtrk_array); - ukinfo->sq_wrtrk_array = NULL; - return -ENOMEM; - } - - ukinfo->sq = mem->va; - info->sq_pa = mem->pa; - - ukinfo->rq = &ukinfo->sq[sqdepth]; - info->rq_pa = info->sq_pa + (sqdepth * I40IW_QP_WQE_MIN_SIZE); - - ukinfo->shadow_area = ukinfo->rq[rqdepth].elem; - info->shadow_area_pa = info->rq_pa + (rqdepth * I40IW_QP_WQE_MIN_SIZE); - - ukinfo->sq_size = sqdepth >> sqshift; - ukinfo->rq_size = rqdepth >> I40IW_MAX_RQ_WQE_SHIFT; - ukinfo->qp_id = iwqp->ibqp.qp_num; - return 0; -} - -/** - * i40iw_create_qp - create qp - * @ibpd: ptr of pd - * @init_attr: attributes for qp - * @udata: user data for create qp - */ -static struct ib_qp *i40iw_create_qp(struct ib_pd *ibpd, - struct ib_qp_init_attr *init_attr, - struct ib_udata *udata) -{ - struct i40iw_pd *iwpd = to_iwpd(ibpd); - struct i40iw_device *iwdev = to_iwdev(ibpd->device); - struct i40iw_cqp *iwcqp = &iwdev->cqp; - struct i40iw_qp *iwqp; - struct i40iw_ucontext *ucontext = rdma_udata_to_drv_context( - udata, struct i40iw_ucontext, ibucontext); - struct i40iw_create_qp_req req; - struct i40iw_create_qp_resp uresp; - u32 qp_num = 0; - enum i40iw_status_code ret; - int err_code; - int sq_size; - int rq_size; - struct i40iw_sc_qp *qp; - struct i40iw_sc_dev *dev = &iwdev->sc_dev; - struct i40iw_qp_init_info init_info; - struct i40iw_create_qp_info *qp_info; - struct i40iw_cqp_request *cqp_request; - struct cqp_commands_info *cqp_info; - - struct i40iw_qp_host_ctx_info *ctx_info; - struct i40iwarp_offload_info *iwarp_info; - unsigned long flags; - - if (iwdev->closing) - return ERR_PTR(-ENODEV); - - if (init_attr->create_flags) - return ERR_PTR(-EOPNOTSUPP); - if (init_attr->cap.max_inline_data > I40IW_MAX_INLINE_DATA_SIZE) - init_attr->cap.max_inline_data = I40IW_MAX_INLINE_DATA_SIZE; - - if (init_attr->cap.max_send_sge > I40IW_MAX_WQ_FRAGMENT_COUNT) - init_attr->cap.max_send_sge = I40IW_MAX_WQ_FRAGMENT_COUNT; - - if (init_attr->cap.max_recv_sge > I40IW_MAX_WQ_FRAGMENT_COUNT) - init_attr->cap.max_recv_sge = I40IW_MAX_WQ_FRAGMENT_COUNT; - - memset(&init_info, 0, sizeof(init_info)); - - sq_size = init_attr->cap.max_send_wr; - rq_size = init_attr->cap.max_recv_wr; - - init_info.vsi = &iwdev->vsi; - init_info.qp_uk_init_info.sq_size = sq_size; - init_info.qp_uk_init_info.rq_size = rq_size; - init_info.qp_uk_init_info.max_sq_frag_cnt = init_attr->cap.max_send_sge; - init_info.qp_uk_init_info.max_rq_frag_cnt = init_attr->cap.max_recv_sge; - init_info.qp_uk_init_info.max_inline_data = init_attr->cap.max_inline_data; - - iwqp = kzalloc(sizeof(*iwqp), GFP_KERNEL); - if (!iwqp) - return ERR_PTR(-ENOMEM); - - qp = &iwqp->sc_qp; - qp->back_qp = (void *)iwqp; - iwqp->iwdev = iwdev; - iwqp->ctx_info.iwarp_info = &iwqp->iwarp_info; - - if (i40iw_allocate_dma_mem(dev->hw, - &iwqp->q2_ctx_mem, - I40IW_Q2_BUFFER_SIZE + I40IW_QP_CTX_SIZE, - 256)) { - i40iw_pr_err("dma_mem failed\n"); - err_code = -ENOMEM; - goto error; - } - - init_info.q2 = iwqp->q2_ctx_mem.va; - init_info.q2_pa = iwqp->q2_ctx_mem.pa; - - init_info.host_ctx = (void *)init_info.q2 + I40IW_Q2_BUFFER_SIZE; - init_info.host_ctx_pa = init_info.q2_pa + I40IW_Q2_BUFFER_SIZE; - - err_code = i40iw_alloc_resource(iwdev, iwdev->allocated_qps, iwdev->max_qp, - &qp_num, &iwdev->next_qp); - if (err_code) { - i40iw_pr_err("qp resource\n"); - goto error; - } - - iwqp->iwpd = iwpd; - iwqp->ibqp.qp_num = qp_num; - qp = &iwqp->sc_qp; - iwqp->iwscq = to_iwcq(init_attr->send_cq); - iwqp->iwrcq = to_iwcq(init_attr->recv_cq); - - iwqp->host_ctx.va = init_info.host_ctx; - iwqp->host_ctx.pa = init_info.host_ctx_pa; - iwqp->host_ctx.size = I40IW_QP_CTX_SIZE; - - init_info.pd = &iwpd->sc_pd; - init_info.qp_uk_init_info.qp_id = iwqp->ibqp.qp_num; - iwqp->ctx_info.qp_compl_ctx = (uintptr_t)qp; - - if (init_attr->qp_type != IB_QPT_RC) { - err_code = -EOPNOTSUPP; - goto error; - } - if (udata) { - err_code = ib_copy_from_udata(&req, udata, sizeof(req)); - if (err_code) { - i40iw_pr_err("ib_copy_from_data\n"); - goto error; - } - iwqp->ctx_info.qp_compl_ctx = req.user_compl_ctx; - iwqp->user_mode = 1; - - if (req.user_wqe_buffers) { - struct i40iw_pbl *iwpbl; - - spin_lock_irqsave( - &ucontext->qp_reg_mem_list_lock, flags); - iwpbl = i40iw_get_pbl( - (unsigned long)req.user_wqe_buffers, - &ucontext->qp_reg_mem_list); - spin_unlock_irqrestore( - &ucontext->qp_reg_mem_list_lock, flags); - - if (!iwpbl) { - err_code = -ENODATA; - i40iw_pr_err("no pbl info\n"); - goto error; - } - memcpy(&iwqp->iwpbl, iwpbl, sizeof(iwqp->iwpbl)); - } - err_code = i40iw_setup_virt_qp(iwdev, iwqp, &init_info); - } else { - err_code = i40iw_setup_kmode_qp(iwdev, iwqp, &init_info); - } - - if (err_code) { - i40iw_pr_err("setup qp failed\n"); - goto error; - } - - init_info.type = I40IW_QP_TYPE_IWARP; - ret = dev->iw_priv_qp_ops->qp_init(qp, &init_info); - if (ret) { - err_code = -EPROTO; - i40iw_pr_err("qp_init fail\n"); - goto error; - } - ctx_info = &iwqp->ctx_info; - iwarp_info = &iwqp->iwarp_info; - iwarp_info->rd_enable = true; - iwarp_info->wr_rdresp_en = true; - if (!iwqp->user_mode) { - iwarp_info->fast_reg_en = true; - iwarp_info->priv_mode_en = true; - } - iwarp_info->ddp_ver = 1; - iwarp_info->rdmap_ver = 1; - - ctx_info->iwarp_info_valid = true; - ctx_info->send_cq_num = iwqp->iwscq->sc_cq.cq_uk.cq_id; - ctx_info->rcv_cq_num = iwqp->iwrcq->sc_cq.cq_uk.cq_id; - ret = dev->iw_priv_qp_ops->qp_setctx(&iwqp->sc_qp, - (u64 *)iwqp->host_ctx.va, - ctx_info); - ctx_info->iwarp_info_valid = false; - cqp_request = i40iw_get_cqp_request(iwcqp, true); - if (!cqp_request) { - err_code = -ENOMEM; - goto error; - } - cqp_info = &cqp_request->info; - qp_info = &cqp_request->info.in.u.qp_create.info; - - memset(qp_info, 0, sizeof(*qp_info)); - - qp_info->cq_num_valid = true; - qp_info->next_iwarp_state = I40IW_QP_STATE_IDLE; - - cqp_info->cqp_cmd = OP_QP_CREATE; - cqp_info->post_sq = 1; - cqp_info->in.u.qp_create.qp = qp; - cqp_info->in.u.qp_create.scratch = (uintptr_t)cqp_request; - ret = i40iw_handle_cqp_op(iwdev, cqp_request); - if (ret) { - i40iw_pr_err("CQP-OP QP create fail"); - err_code = -EACCES; - goto error; - } - - refcount_set(&iwqp->refcount, 1); - spin_lock_init(&iwqp->lock); - iwqp->sig_all = (init_attr->sq_sig_type == IB_SIGNAL_ALL_WR) ? 1 : 0; - iwdev->qp_table[qp_num] = iwqp; - i40iw_add_pdusecount(iwqp->iwpd); - i40iw_add_devusecount(iwdev); - if (udata) { - memset(&uresp, 0, sizeof(uresp)); - uresp.actual_sq_size = sq_size; - uresp.actual_rq_size = rq_size; - uresp.qp_id = qp_num; - uresp.push_idx = I40IW_INVALID_PUSH_PAGE_INDEX; - err_code = ib_copy_to_udata(udata, &uresp, sizeof(uresp)); - if (err_code) { - i40iw_pr_err("copy_to_udata failed\n"); - i40iw_destroy_qp(&iwqp->ibqp, udata); - /* let the completion of the qp destroy free the qp */ - return ERR_PTR(err_code); - } - } - init_completion(&iwqp->sq_drained); - init_completion(&iwqp->rq_drained); - init_completion(&iwqp->free_qp); - - return &iwqp->ibqp; -error: - i40iw_free_qp_resources(iwqp); - return ERR_PTR(err_code); -} - -/** - * i40iw_query_qp - query qp attributes - * @ibqp: qp pointer - * @attr: attributes pointer - * @attr_mask: Not used - * @init_attr: qp attributes to return - */ -static int i40iw_query_qp(struct ib_qp *ibqp, - struct ib_qp_attr *attr, - int attr_mask, - struct ib_qp_init_attr *init_attr) -{ - struct i40iw_qp *iwqp = to_iwqp(ibqp); - struct i40iw_sc_qp *qp = &iwqp->sc_qp; - - attr->qp_state = iwqp->ibqp_state; - attr->cur_qp_state = attr->qp_state; - attr->qp_access_flags = 0; - attr->cap.max_send_wr = qp->qp_uk.sq_size; - attr->cap.max_recv_wr = qp->qp_uk.rq_size; - attr->cap.max_inline_data = I40IW_MAX_INLINE_DATA_SIZE; - attr->cap.max_send_sge = I40IW_MAX_WQ_FRAGMENT_COUNT; - attr->cap.max_recv_sge = I40IW_MAX_WQ_FRAGMENT_COUNT; - attr->port_num = 1; - init_attr->event_handler = iwqp->ibqp.event_handler; - init_attr->qp_context = iwqp->ibqp.qp_context; - init_attr->send_cq = iwqp->ibqp.send_cq; - init_attr->recv_cq = iwqp->ibqp.recv_cq; - init_attr->srq = iwqp->ibqp.srq; - init_attr->cap = attr->cap; - init_attr->port_num = 1; - return 0; -} - -/** - * i40iw_hw_modify_qp - setup cqp for modify qp - * @iwdev: iwarp device - * @iwqp: qp ptr (user or kernel) - * @info: info for modify qp - * @wait: flag to wait or not for modify qp completion - */ -void i40iw_hw_modify_qp(struct i40iw_device *iwdev, struct i40iw_qp *iwqp, - struct i40iw_modify_qp_info *info, bool wait) -{ - struct i40iw_cqp_request *cqp_request; - struct cqp_commands_info *cqp_info; - struct i40iw_modify_qp_info *m_info; - struct i40iw_gen_ae_info ae_info; - - cqp_request = i40iw_get_cqp_request(&iwdev->cqp, wait); - if (!cqp_request) - return; - - cqp_info = &cqp_request->info; - m_info = &cqp_info->in.u.qp_modify.info; - memcpy(m_info, info, sizeof(*m_info)); - cqp_info->cqp_cmd = OP_QP_MODIFY; - cqp_info->post_sq = 1; - cqp_info->in.u.qp_modify.qp = &iwqp->sc_qp; - cqp_info->in.u.qp_modify.scratch = (uintptr_t)cqp_request; - if (!i40iw_handle_cqp_op(iwdev, cqp_request)) - return; - - switch (m_info->next_iwarp_state) { - case I40IW_QP_STATE_RTS: - if (iwqp->iwarp_state == I40IW_QP_STATE_IDLE) - i40iw_send_reset(iwqp->cm_node); - fallthrough; - case I40IW_QP_STATE_IDLE: - case I40IW_QP_STATE_TERMINATE: - case I40IW_QP_STATE_CLOSING: - ae_info.ae_code = I40IW_AE_BAD_CLOSE; - ae_info.ae_source = 0; - i40iw_gen_ae(iwdev, &iwqp->sc_qp, &ae_info, false); - break; - case I40IW_QP_STATE_ERROR: - default: - break; - } -} - -/** - * i40iw_modify_qp - modify qp request - * @ibqp: qp's pointer for modify - * @attr: access attributes - * @attr_mask: state mask - * @udata: user data - */ -int i40iw_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, - int attr_mask, struct ib_udata *udata) -{ - struct i40iw_qp *iwqp = to_iwqp(ibqp); - struct i40iw_device *iwdev = iwqp->iwdev; - struct i40iw_qp_host_ctx_info *ctx_info; - struct i40iwarp_offload_info *iwarp_info; - struct i40iw_modify_qp_info info; - u8 issue_modify_qp = 0; - u8 dont_wait = 0; - u32 err; - unsigned long flags; - - if (attr_mask & ~IB_QP_ATTR_STANDARD_BITS) - return -EOPNOTSUPP; - - memset(&info, 0, sizeof(info)); - ctx_info = &iwqp->ctx_info; - iwarp_info = &iwqp->iwarp_info; - - spin_lock_irqsave(&iwqp->lock, flags); - - if (attr_mask & IB_QP_STATE) { - if (iwdev->closing && attr->qp_state != IB_QPS_ERR) { - err = -EINVAL; - goto exit; - } - - switch (attr->qp_state) { - case IB_QPS_INIT: - case IB_QPS_RTR: - if (iwqp->iwarp_state > (u32)I40IW_QP_STATE_IDLE) { - err = -EINVAL; - goto exit; - } - if (iwqp->iwarp_state == I40IW_QP_STATE_INVALID) { - info.next_iwarp_state = I40IW_QP_STATE_IDLE; - issue_modify_qp = 1; - } - break; - case IB_QPS_RTS: - if ((iwqp->iwarp_state > (u32)I40IW_QP_STATE_RTS) || - (!iwqp->cm_id)) { - err = -EINVAL; - goto exit; - } - - issue_modify_qp = 1; - iwqp->hw_tcp_state = I40IW_TCP_STATE_ESTABLISHED; - iwqp->hte_added = 1; - info.next_iwarp_state = I40IW_QP_STATE_RTS; - info.tcp_ctx_valid = true; - info.ord_valid = true; - info.arp_cache_idx_valid = true; - info.cq_num_valid = true; - break; - case IB_QPS_SQD: - if (iwqp->hw_iwarp_state > (u32)I40IW_QP_STATE_RTS) { - err = 0; - goto exit; - } - if ((iwqp->iwarp_state == (u32)I40IW_QP_STATE_CLOSING) || - (iwqp->iwarp_state < (u32)I40IW_QP_STATE_RTS)) { - err = 0; - goto exit; - } - if (iwqp->iwarp_state > (u32)I40IW_QP_STATE_CLOSING) { - err = -EINVAL; - goto exit; - } - info.next_iwarp_state = I40IW_QP_STATE_CLOSING; - issue_modify_qp = 1; - break; - case IB_QPS_SQE: - if (iwqp->iwarp_state >= (u32)I40IW_QP_STATE_TERMINATE) { - err = -EINVAL; - goto exit; - } - info.next_iwarp_state = I40IW_QP_STATE_TERMINATE; - issue_modify_qp = 1; - break; - case IB_QPS_ERR: - case IB_QPS_RESET: - if (iwqp->iwarp_state == (u32)I40IW_QP_STATE_ERROR) { - err = -EINVAL; - goto exit; - } - if (iwqp->sc_qp.term_flags) - i40iw_terminate_del_timer(&iwqp->sc_qp); - info.next_iwarp_state = I40IW_QP_STATE_ERROR; - if ((iwqp->hw_tcp_state > I40IW_TCP_STATE_CLOSED) && - iwdev->iw_status && - (iwqp->hw_tcp_state != I40IW_TCP_STATE_TIME_WAIT)) - info.reset_tcp_conn = true; - else - dont_wait = 1; - issue_modify_qp = 1; - info.next_iwarp_state = I40IW_QP_STATE_ERROR; - break; - default: - err = -EINVAL; - goto exit; - } - - iwqp->ibqp_state = attr->qp_state; - - } - if (attr_mask & IB_QP_ACCESS_FLAGS) { - ctx_info->iwarp_info_valid = true; - if (attr->qp_access_flags & IB_ACCESS_LOCAL_WRITE) - iwarp_info->wr_rdresp_en = true; - if (attr->qp_access_flags & IB_ACCESS_REMOTE_WRITE) - iwarp_info->wr_rdresp_en = true; - if (attr->qp_access_flags & IB_ACCESS_REMOTE_READ) - iwarp_info->rd_enable = true; - if (attr->qp_access_flags & IB_ACCESS_MW_BIND) - iwarp_info->bind_en = true; - - if (iwqp->user_mode) { - iwarp_info->rd_enable = true; - iwarp_info->wr_rdresp_en = true; - iwarp_info->priv_mode_en = false; - } - } - - if (ctx_info->iwarp_info_valid) { - struct i40iw_sc_dev *dev = &iwdev->sc_dev; - int ret; - - ctx_info->send_cq_num = iwqp->iwscq->sc_cq.cq_uk.cq_id; - ctx_info->rcv_cq_num = iwqp->iwrcq->sc_cq.cq_uk.cq_id; - ret = dev->iw_priv_qp_ops->qp_setctx(&iwqp->sc_qp, - (u64 *)iwqp->host_ctx.va, - ctx_info); - if (ret) { - i40iw_pr_err("setting QP context\n"); - err = -EINVAL; - goto exit; - } - } - - spin_unlock_irqrestore(&iwqp->lock, flags); - - if (issue_modify_qp) { - i40iw_hw_modify_qp(iwdev, iwqp, &info, true); - - spin_lock_irqsave(&iwqp->lock, flags); - iwqp->iwarp_state = info.next_iwarp_state; - spin_unlock_irqrestore(&iwqp->lock, flags); - } - - if (issue_modify_qp && (iwqp->ibqp_state > IB_QPS_RTS)) { - if (dont_wait) { - if (iwqp->cm_id && iwqp->hw_tcp_state) { - spin_lock_irqsave(&iwqp->lock, flags); - iwqp->hw_tcp_state = I40IW_TCP_STATE_CLOSED; - iwqp->last_aeq = I40IW_AE_RESET_SENT; - spin_unlock_irqrestore(&iwqp->lock, flags); - i40iw_cm_disconn(iwqp); - } - } else { - spin_lock_irqsave(&iwqp->lock, flags); - if (iwqp->cm_id) { - if (atomic_inc_return(&iwqp->close_timer_started) == 1) { - iwqp->cm_id->add_ref(iwqp->cm_id); - i40iw_schedule_cm_timer(iwqp->cm_node, - (struct i40iw_puda_buf *)iwqp, - I40IW_TIMER_TYPE_CLOSE, 1, 0); - } - } - spin_unlock_irqrestore(&iwqp->lock, flags); - } - } - return 0; -exit: - spin_unlock_irqrestore(&iwqp->lock, flags); - return err; -} - -/** - * cq_free_resources - free up recources for cq - * @iwdev: iwarp device - * @iwcq: cq ptr - */ -static void cq_free_resources(struct i40iw_device *iwdev, struct i40iw_cq *iwcq) -{ - struct i40iw_sc_cq *cq = &iwcq->sc_cq; - - if (!iwcq->user_mode) - i40iw_free_dma_mem(iwdev->sc_dev.hw, &iwcq->kmem); - i40iw_free_resource(iwdev, iwdev->allocated_cqs, cq->cq_uk.cq_id); -} - -/** - * i40iw_cq_wq_destroy - send cq destroy cqp - * @iwdev: iwarp device - * @cq: hardware control cq - */ -void i40iw_cq_wq_destroy(struct i40iw_device *iwdev, struct i40iw_sc_cq *cq) -{ - enum i40iw_status_code status; - struct i40iw_cqp_request *cqp_request; - struct cqp_commands_info *cqp_info; - - cqp_request = i40iw_get_cqp_request(&iwdev->cqp, true); - if (!cqp_request) - return; - - cqp_info = &cqp_request->info; - - cqp_info->cqp_cmd = OP_CQ_DESTROY; - cqp_info->post_sq = 1; - cqp_info->in.u.cq_destroy.cq = cq; - cqp_info->in.u.cq_destroy.scratch = (uintptr_t)cqp_request; - status = i40iw_handle_cqp_op(iwdev, cqp_request); - if (status) - i40iw_pr_err("CQP-OP Destroy QP fail"); -} - -/** - * i40iw_destroy_cq - destroy cq - * @ib_cq: cq pointer - * @udata: user data or NULL for kernel object - */ -static int i40iw_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata) -{ - struct i40iw_cq *iwcq; - struct i40iw_device *iwdev; - struct i40iw_sc_cq *cq; - - iwcq = to_iwcq(ib_cq); - iwdev = to_iwdev(ib_cq->device); - cq = &iwcq->sc_cq; - i40iw_cq_wq_destroy(iwdev, cq); - cq_free_resources(iwdev, iwcq); - i40iw_rem_devusecount(iwdev); - return 0; -} - -/** - * i40iw_create_cq - create cq - * @ibcq: CQ allocated - * @attr: attributes for cq - * @udata: user data - */ -static int i40iw_create_cq(struct ib_cq *ibcq, - const struct ib_cq_init_attr *attr, - struct ib_udata *udata) -{ - struct ib_device *ibdev = ibcq->device; - struct i40iw_device *iwdev = to_iwdev(ibdev); - struct i40iw_cq *iwcq = to_iwcq(ibcq); - struct i40iw_pbl *iwpbl; - u32 cq_num = 0; - struct i40iw_sc_cq *cq; - struct i40iw_sc_dev *dev = &iwdev->sc_dev; - struct i40iw_cq_init_info info = {}; - enum i40iw_status_code status; - struct i40iw_cqp_request *cqp_request; - struct cqp_commands_info *cqp_info; - struct i40iw_cq_uk_init_info *ukinfo = &info.cq_uk_init_info; - unsigned long flags; - int err_code; - int entries = attr->cqe; - - if (attr->flags) - return -EOPNOTSUPP; - - if (iwdev->closing) - return -ENODEV; - - if (entries > iwdev->max_cqe) - return -EINVAL; - - err_code = i40iw_alloc_resource(iwdev, iwdev->allocated_cqs, - iwdev->max_cq, &cq_num, - &iwdev->next_cq); - if (err_code) - return err_code; - - cq = &iwcq->sc_cq; - cq->back_cq = (void *)iwcq; - spin_lock_init(&iwcq->lock); - - info.dev = dev; - ukinfo->cq_size = max(entries, 4); - ukinfo->cq_id = cq_num; - iwcq->ibcq.cqe = info.cq_uk_init_info.cq_size; - info.ceqe_mask = 0; - if (attr->comp_vector < iwdev->ceqs_count) - info.ceq_id = attr->comp_vector; - info.ceq_id_valid = true; - info.ceqe_mask = 1; - info.type = I40IW_CQ_TYPE_IWARP; - if (udata) { - struct i40iw_ucontext *ucontext = rdma_udata_to_drv_context( - udata, struct i40iw_ucontext, ibucontext); - struct i40iw_create_cq_req req; - struct i40iw_cq_mr *cqmr; - - memset(&req, 0, sizeof(req)); - iwcq->user_mode = true; - if (ib_copy_from_udata(&req, udata, sizeof(struct i40iw_create_cq_req))) { - err_code = -EFAULT; - goto cq_free_resources; - } - - spin_lock_irqsave(&ucontext->cq_reg_mem_list_lock, flags); - iwpbl = i40iw_get_pbl((unsigned long)req.user_cq_buffer, - &ucontext->cq_reg_mem_list); - spin_unlock_irqrestore(&ucontext->cq_reg_mem_list_lock, flags); - if (!iwpbl) { - err_code = -EPROTO; - goto cq_free_resources; - } - - iwcq->iwpbl = iwpbl; - iwcq->cq_mem_size = 0; - cqmr = &iwpbl->cq_mr; - info.shadow_area_pa = cpu_to_le64(cqmr->shadow); - if (iwpbl->pbl_allocated) { - info.virtual_map = true; - info.pbl_chunk_size = 1; - info.first_pm_pbl_idx = cqmr->cq_pbl.idx; - } else { - info.cq_base_pa = cqmr->cq_pbl.addr; - } - } else { - /* Kmode allocations */ - int rsize; - int shadow; - - rsize = info.cq_uk_init_info.cq_size * sizeof(struct i40iw_cqe); - rsize = round_up(rsize, 256); - shadow = I40IW_SHADOW_AREA_SIZE << 3; - status = i40iw_allocate_dma_mem(dev->hw, &iwcq->kmem, - rsize + shadow, 256); - if (status) { - err_code = -ENOMEM; - goto cq_free_resources; - } - ukinfo->cq_base = iwcq->kmem.va; - info.cq_base_pa = iwcq->kmem.pa; - info.shadow_area_pa = info.cq_base_pa + rsize; - ukinfo->shadow_area = iwcq->kmem.va + rsize; - } - - if (dev->iw_priv_cq_ops->cq_init(cq, &info)) { - i40iw_pr_err("init cq fail\n"); - err_code = -EPROTO; - goto cq_free_resources; - } - - cqp_request = i40iw_get_cqp_request(&iwdev->cqp, true); - if (!cqp_request) { - err_code = -ENOMEM; - goto cq_free_resources; - } - - cqp_info = &cqp_request->info; - cqp_info->cqp_cmd = OP_CQ_CREATE; - cqp_info->post_sq = 1; - cqp_info->in.u.cq_create.cq = cq; - cqp_info->in.u.cq_create.scratch = (uintptr_t)cqp_request; - status = i40iw_handle_cqp_op(iwdev, cqp_request); - if (status) { - i40iw_pr_err("CQP-OP Create QP fail"); - err_code = -EPROTO; - goto cq_free_resources; - } - - if (udata) { - struct i40iw_create_cq_resp resp; - - memset(&resp, 0, sizeof(resp)); - resp.cq_id = info.cq_uk_init_info.cq_id; - resp.cq_size = info.cq_uk_init_info.cq_size; - if (ib_copy_to_udata(udata, &resp, sizeof(resp))) { - i40iw_pr_err("copy to user data\n"); - err_code = -EPROTO; - goto cq_destroy; - } - } - - i40iw_add_devusecount(iwdev); - return 0; - -cq_destroy: - i40iw_cq_wq_destroy(iwdev, cq); -cq_free_resources: - cq_free_resources(iwdev, iwcq); - return err_code; -} - -/** - * i40iw_get_user_access - get hw access from IB access - * @acc: IB access to return hw access - */ -static inline u16 i40iw_get_user_access(int acc) -{ - u16 access = 0; - - access |= (acc & IB_ACCESS_LOCAL_WRITE) ? I40IW_ACCESS_FLAGS_LOCALWRITE : 0; - access |= (acc & IB_ACCESS_REMOTE_WRITE) ? I40IW_ACCESS_FLAGS_REMOTEWRITE : 0; - access |= (acc & IB_ACCESS_REMOTE_READ) ? I40IW_ACCESS_FLAGS_REMOTEREAD : 0; - access |= (acc & IB_ACCESS_MW_BIND) ? I40IW_ACCESS_FLAGS_BIND_WINDOW : 0; - return access; -} - -/** - * i40iw_free_stag - free stag resource - * @iwdev: iwarp device - * @stag: stag to free - */ -static void i40iw_free_stag(struct i40iw_device *iwdev, u32 stag) -{ - u32 stag_idx; - - stag_idx = (stag & iwdev->mr_stagmask) >> I40IW_CQPSQ_STAG_IDX_SHIFT; - i40iw_free_resource(iwdev, iwdev->allocated_mrs, stag_idx); - i40iw_rem_devusecount(iwdev); -} - -/** - * i40iw_create_stag - create random stag - * @iwdev: iwarp device - */ -static u32 i40iw_create_stag(struct i40iw_device *iwdev) -{ - u32 stag = 0; - u32 stag_index = 0; - u32 next_stag_index; - u32 driver_key; - u32 random; - u8 consumer_key; - int ret; - - get_random_bytes(&random, sizeof(random)); - consumer_key = (u8)random; - - driver_key = random & ~iwdev->mr_stagmask; - next_stag_index = (random & iwdev->mr_stagmask) >> 8; - next_stag_index %= iwdev->max_mr; - - ret = i40iw_alloc_resource(iwdev, - iwdev->allocated_mrs, iwdev->max_mr, - &stag_index, &next_stag_index); - if (!ret) { - stag = stag_index << I40IW_CQPSQ_STAG_IDX_SHIFT; - stag |= driver_key; - stag += (u32)consumer_key; - i40iw_add_devusecount(iwdev); - } - return stag; -} - -/** - * i40iw_next_pbl_addr - Get next pbl address - * @pbl: pointer to a pble - * @pinfo: info pointer - * @idx: index - */ -static inline u64 *i40iw_next_pbl_addr(u64 *pbl, - struct i40iw_pble_info **pinfo, - u32 *idx) -{ - *idx += 1; - if ((!(*pinfo)) || (*idx != (*pinfo)->cnt)) - return ++pbl; - *idx = 0; - (*pinfo)++; - return (u64 *)(*pinfo)->addr; -} - -/** - * i40iw_copy_user_pgaddrs - copy user page address to pble's os locally - * @iwmr: iwmr for IB's user page addresses - * @pbl: ple pointer to save 1 level or 0 level pble - * @level: indicated level 0, 1 or 2 - */ -static void i40iw_copy_user_pgaddrs(struct i40iw_mr *iwmr, - u64 *pbl, - enum i40iw_pble_level level) -{ - struct ib_umem *region = iwmr->region; - struct i40iw_pbl *iwpbl = &iwmr->iwpbl; - struct i40iw_pble_alloc *palloc = &iwpbl->pble_alloc; - struct i40iw_pble_info *pinfo; - struct ib_block_iter biter; - u32 idx = 0; - - pinfo = (level == I40IW_LEVEL_1) ? NULL : palloc->level2.leaf; - - if (iwmr->type == IW_MEMREG_TYPE_QP) - iwpbl->qp_mr.sq_page = sg_page(region->sg_head.sgl); - - rdma_umem_for_each_dma_block(region, &biter, iwmr->page_size) { - *pbl = rdma_block_iter_dma_address(&biter); - pbl = i40iw_next_pbl_addr(pbl, &pinfo, &idx); - } -} - -/** - * i40iw_check_mem_contiguous - check if pbls stored in arr are contiguous - * @arr: lvl1 pbl array - * @npages: page count - * @pg_size: page size - * - */ -static bool i40iw_check_mem_contiguous(u64 *arr, u32 npages, u32 pg_size) -{ - u32 pg_idx; - - for (pg_idx = 0; pg_idx < npages; pg_idx++) { - if ((*arr + (pg_size * pg_idx)) != arr[pg_idx]) - return false; - } - return true; -} - -/** - * i40iw_check_mr_contiguous - check if MR is physically contiguous - * @palloc: pbl allocation struct - * @pg_size: page size - */ -static bool i40iw_check_mr_contiguous(struct i40iw_pble_alloc *palloc, u32 pg_size) -{ - struct i40iw_pble_level2 *lvl2 = &palloc->level2; - struct i40iw_pble_info *leaf = lvl2->leaf; - u64 *arr = NULL; - u64 *start_addr = NULL; - int i; - bool ret; - - if (palloc->level == I40IW_LEVEL_1) { - arr = (u64 *)palloc->level1.addr; - ret = i40iw_check_mem_contiguous(arr, palloc->total_cnt, pg_size); - return ret; - } - - start_addr = (u64 *)leaf->addr; - - for (i = 0; i < lvl2->leaf_cnt; i++, leaf++) { - arr = (u64 *)leaf->addr; - if ((*start_addr + (i * pg_size * PBLE_PER_PAGE)) != *arr) - return false; - ret = i40iw_check_mem_contiguous(arr, leaf->cnt, pg_size); - if (!ret) - return false; - } - - return true; -} - -/** - * i40iw_setup_pbles - copy user pg address to pble's - * @iwdev: iwarp device - * @iwmr: mr pointer for this memory registration - * @use_pbles: flag if to use pble's - */ -static int i40iw_setup_pbles(struct i40iw_device *iwdev, - struct i40iw_mr *iwmr, - bool use_pbles) -{ - struct i40iw_pbl *iwpbl = &iwmr->iwpbl; - struct i40iw_pble_alloc *palloc = &iwpbl->pble_alloc; - struct i40iw_pble_info *pinfo; - u64 *pbl; - enum i40iw_status_code status; - enum i40iw_pble_level level = I40IW_LEVEL_1; - - if (use_pbles) { - mutex_lock(&iwdev->pbl_mutex); - status = i40iw_get_pble(&iwdev->sc_dev, iwdev->pble_rsrc, palloc, iwmr->page_cnt); - mutex_unlock(&iwdev->pbl_mutex); - if (status) - return -ENOMEM; - - iwpbl->pbl_allocated = true; - level = palloc->level; - pinfo = (level == I40IW_LEVEL_1) ? &palloc->level1 : palloc->level2.leaf; - pbl = (u64 *)pinfo->addr; - } else { - pbl = iwmr->pgaddrmem; - } - - i40iw_copy_user_pgaddrs(iwmr, pbl, level); - - if (use_pbles) - iwmr->pgaddrmem[0] = *pbl; - - return 0; -} - -/** - * i40iw_handle_q_mem - handle memory for qp and cq - * @iwdev: iwarp device - * @req: information for q memory management - * @iwpbl: pble struct - * @use_pbles: flag to use pble - */ -static int i40iw_handle_q_mem(struct i40iw_device *iwdev, - struct i40iw_mem_reg_req *req, - struct i40iw_pbl *iwpbl, - bool use_pbles) -{ - struct i40iw_pble_alloc *palloc = &iwpbl->pble_alloc; - struct i40iw_mr *iwmr = iwpbl->iwmr; - struct i40iw_qp_mr *qpmr = &iwpbl->qp_mr; - struct i40iw_cq_mr *cqmr = &iwpbl->cq_mr; - struct i40iw_hmc_pble *hmc_p; - u64 *arr = iwmr->pgaddrmem; - u32 pg_size; - int err; - int total; - bool ret = true; - - total = req->sq_pages + req->rq_pages + req->cq_pages; - pg_size = iwmr->page_size; - - err = i40iw_setup_pbles(iwdev, iwmr, use_pbles); - if (err) - return err; - - if (use_pbles && (palloc->level != I40IW_LEVEL_1)) { - i40iw_free_pble(iwdev->pble_rsrc, palloc); - iwpbl->pbl_allocated = false; - return -ENOMEM; - } - - if (use_pbles) - arr = (u64 *)palloc->level1.addr; - - if (iwmr->type == IW_MEMREG_TYPE_QP) { - hmc_p = &qpmr->sq_pbl; - qpmr->shadow = (dma_addr_t)arr[total]; - - if (use_pbles) { - ret = i40iw_check_mem_contiguous(arr, req->sq_pages, pg_size); - if (ret) - ret = i40iw_check_mem_contiguous(&arr[req->sq_pages], req->rq_pages, pg_size); - } - - if (!ret) { - hmc_p->idx = palloc->level1.idx; - hmc_p = &qpmr->rq_pbl; - hmc_p->idx = palloc->level1.idx + req->sq_pages; - } else { - hmc_p->addr = arr[0]; - hmc_p = &qpmr->rq_pbl; - hmc_p->addr = arr[req->sq_pages]; - } - } else { /* CQ */ - hmc_p = &cqmr->cq_pbl; - cqmr->shadow = (dma_addr_t)arr[total]; - - if (use_pbles) - ret = i40iw_check_mem_contiguous(arr, req->cq_pages, pg_size); - - if (!ret) - hmc_p->idx = palloc->level1.idx; - else - hmc_p->addr = arr[0]; - } - - if (use_pbles && ret) { - i40iw_free_pble(iwdev->pble_rsrc, palloc); - iwpbl->pbl_allocated = false; - } - - return err; -} - -/** - * i40iw_hw_alloc_stag - cqp command to allocate stag - * @iwdev: iwarp device - * @iwmr: iwarp mr pointer - */ -static int i40iw_hw_alloc_stag(struct i40iw_device *iwdev, struct i40iw_mr *iwmr) -{ - struct i40iw_allocate_stag_info *info; - struct i40iw_pd *iwpd = to_iwpd(iwmr->ibmr.pd); - enum i40iw_status_code status; - int err = 0; - struct i40iw_cqp_request *cqp_request; - struct cqp_commands_info *cqp_info; - - cqp_request = i40iw_get_cqp_request(&iwdev->cqp, true); - if (!cqp_request) - return -ENOMEM; - - cqp_info = &cqp_request->info; - info = &cqp_info->in.u.alloc_stag.info; - memset(info, 0, sizeof(*info)); - info->page_size = PAGE_SIZE; - info->stag_idx = iwmr->stag >> I40IW_CQPSQ_STAG_IDX_SHIFT; - info->pd_id = iwpd->sc_pd.pd_id; - info->total_len = iwmr->length; - info->remote_access = true; - cqp_info->cqp_cmd = OP_ALLOC_STAG; - cqp_info->post_sq = 1; - cqp_info->in.u.alloc_stag.dev = &iwdev->sc_dev; - cqp_info->in.u.alloc_stag.scratch = (uintptr_t)cqp_request; - - status = i40iw_handle_cqp_op(iwdev, cqp_request); - if (status) { - err = -ENOMEM; - i40iw_pr_err("CQP-OP MR Reg fail"); - } - return err; -} - -/** - * i40iw_alloc_mr - register stag for fast memory registration - * @pd: ibpd pointer - * @mr_type: memory for stag registrion - * @max_num_sg: man number of pages - */ -static struct ib_mr *i40iw_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type, - u32 max_num_sg) -{ - struct i40iw_pd *iwpd = to_iwpd(pd); - struct i40iw_device *iwdev = to_iwdev(pd->device); - struct i40iw_pble_alloc *palloc; - struct i40iw_pbl *iwpbl; - struct i40iw_mr *iwmr; - enum i40iw_status_code status; - u32 stag; - int err_code = -ENOMEM; - - iwmr = kzalloc(sizeof(*iwmr), GFP_KERNEL); - if (!iwmr) - return ERR_PTR(-ENOMEM); - - stag = i40iw_create_stag(iwdev); - if (!stag) { - err_code = -EOVERFLOW; - goto err; - } - stag &= ~I40IW_CQPSQ_STAG_KEY_MASK; - iwmr->stag = stag; - iwmr->ibmr.rkey = stag; - iwmr->ibmr.lkey = stag; - iwmr->ibmr.pd = pd; - iwmr->ibmr.device = pd->device; - iwpbl = &iwmr->iwpbl; - iwpbl->iwmr = iwmr; - iwmr->type = IW_MEMREG_TYPE_MEM; - palloc = &iwpbl->pble_alloc; - iwmr->page_cnt = max_num_sg; - mutex_lock(&iwdev->pbl_mutex); - status = i40iw_get_pble(&iwdev->sc_dev, iwdev->pble_rsrc, palloc, iwmr->page_cnt); - mutex_unlock(&iwdev->pbl_mutex); - if (status) - goto err1; - - if (palloc->level != I40IW_LEVEL_1) - goto err2; - err_code = i40iw_hw_alloc_stag(iwdev, iwmr); - if (err_code) - goto err2; - iwpbl->pbl_allocated = true; - i40iw_add_pdusecount(iwpd); - return &iwmr->ibmr; -err2: - i40iw_free_pble(iwdev->pble_rsrc, palloc); -err1: - i40iw_free_stag(iwdev, stag); -err: - kfree(iwmr); - return ERR_PTR(err_code); -} - -/** - * i40iw_set_page - populate pbl list for fmr - * @ibmr: ib mem to access iwarp mr pointer - * @addr: page dma address fro pbl list - */ -static int i40iw_set_page(struct ib_mr *ibmr, u64 addr) -{ - struct i40iw_mr *iwmr = to_iwmr(ibmr); - struct i40iw_pbl *iwpbl = &iwmr->iwpbl; - struct i40iw_pble_alloc *palloc = &iwpbl->pble_alloc; - u64 *pbl; - - if (unlikely(iwmr->npages == iwmr->page_cnt)) - return -ENOMEM; - - pbl = (u64 *)palloc->level1.addr; - pbl[iwmr->npages++] = cpu_to_le64(addr); - return 0; -} - -/** - * i40iw_map_mr_sg - map of sg list for fmr - * @ibmr: ib mem to access iwarp mr pointer - * @sg: scatter gather list for fmr - * @sg_nents: number of sg pages - * @sg_offset: scatter gather offset - */ -static int i40iw_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, - int sg_nents, unsigned int *sg_offset) -{ - struct i40iw_mr *iwmr = to_iwmr(ibmr); - - iwmr->npages = 0; - return ib_sg_to_pages(ibmr, sg, sg_nents, sg_offset, i40iw_set_page); -} - -/** - * i40iw_drain_sq - drain the send queue - * @ibqp: ib qp pointer - */ -static void i40iw_drain_sq(struct ib_qp *ibqp) -{ - struct i40iw_qp *iwqp = to_iwqp(ibqp); - struct i40iw_sc_qp *qp = &iwqp->sc_qp; - - if (I40IW_RING_MORE_WORK(qp->qp_uk.sq_ring)) - wait_for_completion(&iwqp->sq_drained); -} - -/** - * i40iw_drain_rq - drain the receive queue - * @ibqp: ib qp pointer - */ -static void i40iw_drain_rq(struct ib_qp *ibqp) -{ - struct i40iw_qp *iwqp = to_iwqp(ibqp); - struct i40iw_sc_qp *qp = &iwqp->sc_qp; - - if (I40IW_RING_MORE_WORK(qp->qp_uk.rq_ring)) - wait_for_completion(&iwqp->rq_drained); -} - -/** - * i40iw_hwreg_mr - send cqp command for memory registration - * @iwdev: iwarp device - * @iwmr: iwarp mr pointer - * @access: access for MR - */ -static int i40iw_hwreg_mr(struct i40iw_device *iwdev, - struct i40iw_mr *iwmr, - u16 access) -{ - struct i40iw_pbl *iwpbl = &iwmr->iwpbl; - struct i40iw_reg_ns_stag_info *stag_info; - struct i40iw_pd *iwpd = to_iwpd(iwmr->ibmr.pd); - struct i40iw_pble_alloc *palloc = &iwpbl->pble_alloc; - enum i40iw_status_code status; - int err = 0; - struct i40iw_cqp_request *cqp_request; - struct cqp_commands_info *cqp_info; - - cqp_request = i40iw_get_cqp_request(&iwdev->cqp, true); - if (!cqp_request) - return -ENOMEM; - - cqp_info = &cqp_request->info; - stag_info = &cqp_info->in.u.mr_reg_non_shared.info; - memset(stag_info, 0, sizeof(*stag_info)); - stag_info->va = (void *)(unsigned long)iwpbl->user_base; - stag_info->stag_idx = iwmr->stag >> I40IW_CQPSQ_STAG_IDX_SHIFT; - stag_info->stag_key = (u8)iwmr->stag; - stag_info->total_len = iwmr->length; - stag_info->access_rights = access; - stag_info->pd_id = iwpd->sc_pd.pd_id; - stag_info->addr_type = I40IW_ADDR_TYPE_VA_BASED; - stag_info->page_size = iwmr->page_size; - - if (iwpbl->pbl_allocated) { - if (palloc->level == I40IW_LEVEL_1) { - stag_info->first_pm_pbl_index = palloc->level1.idx; - stag_info->chunk_size = 1; - } else { - stag_info->first_pm_pbl_index = palloc->level2.root.idx; - stag_info->chunk_size = 3; - } - } else { - stag_info->reg_addr_pa = iwmr->pgaddrmem[0]; - } - - cqp_info->cqp_cmd = OP_MR_REG_NON_SHARED; - cqp_info->post_sq = 1; - cqp_info->in.u.mr_reg_non_shared.dev = &iwdev->sc_dev; - cqp_info->in.u.mr_reg_non_shared.scratch = (uintptr_t)cqp_request; - - status = i40iw_handle_cqp_op(iwdev, cqp_request); - if (status) { - err = -ENOMEM; - i40iw_pr_err("CQP-OP MR Reg fail"); - } - return err; -} - -/** - * i40iw_reg_user_mr - Register a user memory region - * @pd: ptr of pd - * @start: virtual start address - * @length: length of mr - * @virt: virtual address - * @acc: access of mr - * @udata: user data - */ -static struct ib_mr *i40iw_reg_user_mr(struct ib_pd *pd, - u64 start, - u64 length, - u64 virt, - int acc, - struct ib_udata *udata) -{ - struct i40iw_pd *iwpd = to_iwpd(pd); - struct i40iw_device *iwdev = to_iwdev(pd->device); - struct i40iw_ucontext *ucontext = rdma_udata_to_drv_context( - udata, struct i40iw_ucontext, ibucontext); - struct i40iw_pble_alloc *palloc; - struct i40iw_pbl *iwpbl; - struct i40iw_mr *iwmr; - struct ib_umem *region; - struct i40iw_mem_reg_req req; - u32 stag = 0; - u16 access; - bool use_pbles = false; - unsigned long flags; - int err = -ENOSYS; - int ret; - - if (!udata) - return ERR_PTR(-EOPNOTSUPP); - - if (iwdev->closing) - return ERR_PTR(-ENODEV); - - if (length > I40IW_MAX_MR_SIZE) - return ERR_PTR(-EINVAL); - region = ib_umem_get(pd->device, start, length, acc); - if (IS_ERR(region)) - return (struct ib_mr *)region; - - if (ib_copy_from_udata(&req, udata, sizeof(req))) { - ib_umem_release(region); - return ERR_PTR(-EFAULT); - } - - iwmr = kzalloc(sizeof(*iwmr), GFP_KERNEL); - if (!iwmr) { - ib_umem_release(region); - return ERR_PTR(-ENOMEM); - } - - iwpbl = &iwmr->iwpbl; - iwpbl->iwmr = iwmr; - iwmr->region = region; - iwmr->ibmr.pd = pd; - iwmr->ibmr.device = pd->device; - - iwmr->page_size = PAGE_SIZE; - if (req.reg_type == IW_MEMREG_TYPE_MEM) - iwmr->page_size = ib_umem_find_best_pgsz(region, SZ_4K | SZ_2M, - virt); - iwmr->length = region->length; - - iwpbl->user_base = virt; - palloc = &iwpbl->pble_alloc; - - iwmr->type = req.reg_type; - iwmr->page_cnt = ib_umem_num_dma_blocks(region, iwmr->page_size); - - switch (req.reg_type) { - case IW_MEMREG_TYPE_QP: - use_pbles = ((req.sq_pages + req.rq_pages) > 2); - err = i40iw_handle_q_mem(iwdev, &req, iwpbl, use_pbles); - if (err) - goto error; - spin_lock_irqsave(&ucontext->qp_reg_mem_list_lock, flags); - list_add_tail(&iwpbl->list, &ucontext->qp_reg_mem_list); - iwpbl->on_list = true; - spin_unlock_irqrestore(&ucontext->qp_reg_mem_list_lock, flags); - break; - case IW_MEMREG_TYPE_CQ: - use_pbles = (req.cq_pages > 1); - err = i40iw_handle_q_mem(iwdev, &req, iwpbl, use_pbles); - if (err) - goto error; - - spin_lock_irqsave(&ucontext->cq_reg_mem_list_lock, flags); - list_add_tail(&iwpbl->list, &ucontext->cq_reg_mem_list); - iwpbl->on_list = true; - spin_unlock_irqrestore(&ucontext->cq_reg_mem_list_lock, flags); - break; - case IW_MEMREG_TYPE_MEM: - use_pbles = (iwmr->page_cnt != 1); - access = I40IW_ACCESS_FLAGS_LOCALREAD; - - err = i40iw_setup_pbles(iwdev, iwmr, use_pbles); - if (err) - goto error; - - if (use_pbles) { - ret = i40iw_check_mr_contiguous(palloc, iwmr->page_size); - if (ret) { - i40iw_free_pble(iwdev->pble_rsrc, palloc); - iwpbl->pbl_allocated = false; - } - } - - access |= i40iw_get_user_access(acc); - stag = i40iw_create_stag(iwdev); - if (!stag) { - err = -ENOMEM; - goto error; - } - - iwmr->stag = stag; - iwmr->ibmr.rkey = stag; - iwmr->ibmr.lkey = stag; - - err = i40iw_hwreg_mr(iwdev, iwmr, access); - if (err) { - i40iw_free_stag(iwdev, stag); - goto error; - } - - break; - default: - goto error; - } - - iwmr->type = req.reg_type; - if (req.reg_type == IW_MEMREG_TYPE_MEM) - i40iw_add_pdusecount(iwpd); - return &iwmr->ibmr; - -error: - if (palloc->level != I40IW_LEVEL_0 && iwpbl->pbl_allocated) - i40iw_free_pble(iwdev->pble_rsrc, palloc); - ib_umem_release(region); - kfree(iwmr); - return ERR_PTR(err); -} - -/** - * i40iw_reg_phys_mr - register kernel physical memory - * @pd: ibpd pointer - * @addr: physical address of memory to register - * @size: size of memory to register - * @acc: Access rights - * @iova_start: start of virtual address for physical buffers - */ -struct ib_mr *i40iw_reg_phys_mr(struct ib_pd *pd, - u64 addr, - u64 size, - int acc, - u64 *iova_start) -{ - struct i40iw_pd *iwpd = to_iwpd(pd); - struct i40iw_device *iwdev = to_iwdev(pd->device); - struct i40iw_pbl *iwpbl; - struct i40iw_mr *iwmr; - enum i40iw_status_code status; - u32 stag; - u16 access = I40IW_ACCESS_FLAGS_LOCALREAD; - int ret; - - iwmr = kzalloc(sizeof(*iwmr), GFP_KERNEL); - if (!iwmr) - return ERR_PTR(-ENOMEM); - iwmr->ibmr.pd = pd; - iwmr->ibmr.device = pd->device; - iwpbl = &iwmr->iwpbl; - iwpbl->iwmr = iwmr; - iwmr->type = IW_MEMREG_TYPE_MEM; - iwpbl->user_base = *iova_start; - stag = i40iw_create_stag(iwdev); - if (!stag) { - ret = -EOVERFLOW; - goto err; - } - access |= i40iw_get_user_access(acc); - iwmr->stag = stag; - iwmr->ibmr.rkey = stag; - iwmr->ibmr.lkey = stag; - iwmr->page_cnt = 1; - iwmr->pgaddrmem[0] = addr; - iwmr->length = size; - status = i40iw_hwreg_mr(iwdev, iwmr, access); - if (status) { - i40iw_free_stag(iwdev, stag); - ret = -ENOMEM; - goto err; - } - - i40iw_add_pdusecount(iwpd); - return &iwmr->ibmr; - err: - kfree(iwmr); - return ERR_PTR(ret); -} - -/** - * i40iw_get_dma_mr - register physical mem - * @pd: ptr of pd - * @acc: access for memory - */ -static struct ib_mr *i40iw_get_dma_mr(struct ib_pd *pd, int acc) -{ - u64 kva = 0; - - return i40iw_reg_phys_mr(pd, 0, 0, acc, &kva); -} - -/** - * i40iw_del_memlist - Deleting pbl list entries for CQ/QP - * @iwmr: iwmr for IB's user page addresses - * @ucontext: ptr to user context - */ -static void i40iw_del_memlist(struct i40iw_mr *iwmr, - struct i40iw_ucontext *ucontext) -{ - struct i40iw_pbl *iwpbl = &iwmr->iwpbl; - unsigned long flags; - - switch (iwmr->type) { - case IW_MEMREG_TYPE_CQ: - spin_lock_irqsave(&ucontext->cq_reg_mem_list_lock, flags); - if (iwpbl->on_list) { - iwpbl->on_list = false; - list_del(&iwpbl->list); - } - spin_unlock_irqrestore(&ucontext->cq_reg_mem_list_lock, flags); - break; - case IW_MEMREG_TYPE_QP: - spin_lock_irqsave(&ucontext->qp_reg_mem_list_lock, flags); - if (iwpbl->on_list) { - iwpbl->on_list = false; - list_del(&iwpbl->list); - } - spin_unlock_irqrestore(&ucontext->qp_reg_mem_list_lock, flags); - break; - default: - break; - } -} - -/** - * i40iw_dereg_mr - deregister mr - * @ib_mr: mr ptr for dereg - * @udata: user data - */ -static int i40iw_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata) -{ - struct ib_pd *ibpd = ib_mr->pd; - struct i40iw_pd *iwpd = to_iwpd(ibpd); - struct i40iw_mr *iwmr = to_iwmr(ib_mr); - struct i40iw_device *iwdev = to_iwdev(ib_mr->device); - enum i40iw_status_code status; - struct i40iw_dealloc_stag_info *info; - struct i40iw_pbl *iwpbl = &iwmr->iwpbl; - struct i40iw_pble_alloc *palloc = &iwpbl->pble_alloc; - struct i40iw_cqp_request *cqp_request; - struct cqp_commands_info *cqp_info; - u32 stag_idx; - - ib_umem_release(iwmr->region); - - if (iwmr->type != IW_MEMREG_TYPE_MEM) { - /* region is released. only test for userness. */ - if (iwmr->region) { - struct i40iw_ucontext *ucontext = - rdma_udata_to_drv_context( - udata, - struct i40iw_ucontext, - ibucontext); - - i40iw_del_memlist(iwmr, ucontext); - } - if (iwpbl->pbl_allocated && iwmr->type != IW_MEMREG_TYPE_QP) - i40iw_free_pble(iwdev->pble_rsrc, palloc); - kfree(iwmr); - return 0; - } - - cqp_request = i40iw_get_cqp_request(&iwdev->cqp, true); - if (!cqp_request) - return -ENOMEM; - - cqp_info = &cqp_request->info; - info = &cqp_info->in.u.dealloc_stag.info; - memset(info, 0, sizeof(*info)); - - info->pd_id = cpu_to_le32(iwpd->sc_pd.pd_id & 0x00007fff); - info->stag_idx = RS_64_1(ib_mr->rkey, I40IW_CQPSQ_STAG_IDX_SHIFT); - stag_idx = info->stag_idx; - info->mr = true; - if (iwpbl->pbl_allocated) - info->dealloc_pbl = true; - - cqp_info->cqp_cmd = OP_DEALLOC_STAG; - cqp_info->post_sq = 1; - cqp_info->in.u.dealloc_stag.dev = &iwdev->sc_dev; - cqp_info->in.u.dealloc_stag.scratch = (uintptr_t)cqp_request; - status = i40iw_handle_cqp_op(iwdev, cqp_request); - if (status) - i40iw_pr_err("CQP-OP dealloc failed for stag_idx = 0x%x\n", stag_idx); - i40iw_rem_pdusecount(iwpd, iwdev); - i40iw_free_stag(iwdev, iwmr->stag); - if (iwpbl->pbl_allocated) - i40iw_free_pble(iwdev->pble_rsrc, palloc); - kfree(iwmr); - return 0; -} - -/* - * hw_rev_show - */ -static ssize_t hw_rev_show(struct device *dev, - struct device_attribute *attr, char *buf) -{ - struct i40iw_ib_device *iwibdev = - rdma_device_to_drv_device(dev, struct i40iw_ib_device, ibdev); - u32 hw_rev = iwibdev->iwdev->sc_dev.hw_rev; - - return sysfs_emit(buf, "%x\n", hw_rev); -} -static DEVICE_ATTR_RO(hw_rev); - -/* - * hca_type_show - */ -static ssize_t hca_type_show(struct device *dev, - struct device_attribute *attr, char *buf) -{ - return sysfs_emit(buf, "I40IW\n"); -} -static DEVICE_ATTR_RO(hca_type); - -/* - * board_id_show - */ -static ssize_t board_id_show(struct device *dev, - struct device_attribute *attr, char *buf) -{ - return sysfs_emit(buf, "%.*s\n", 32, "I40IW Board ID"); -} -static DEVICE_ATTR_RO(board_id); - -static struct attribute *i40iw_dev_attributes[] = { - &dev_attr_hw_rev.attr, - &dev_attr_hca_type.attr, - &dev_attr_board_id.attr, - NULL -}; - -static const struct attribute_group i40iw_attr_group = { - .attrs = i40iw_dev_attributes, -}; - -/** - * i40iw_copy_sg_list - copy sg list for qp - * @sg_list: copied into sg_list - * @sgl: copy from sgl - * @num_sges: count of sg entries - */ -static void i40iw_copy_sg_list(struct i40iw_sge *sg_list, struct ib_sge *sgl, int num_sges) -{ - unsigned int i; - - for (i = 0; (i < num_sges) && (i < I40IW_MAX_WQ_FRAGMENT_COUNT); i++) { - sg_list[i].tag_off = sgl[i].addr; - sg_list[i].len = sgl[i].length; - sg_list[i].stag = sgl[i].lkey; - } -} - -/** - * i40iw_post_send - kernel application wr - * @ibqp: qp ptr for wr - * @ib_wr: work request ptr - * @bad_wr: return of bad wr if err - */ -static int i40iw_post_send(struct ib_qp *ibqp, - const struct ib_send_wr *ib_wr, - const struct ib_send_wr **bad_wr) -{ - struct i40iw_qp *iwqp; - struct i40iw_qp_uk *ukqp; - struct i40iw_post_sq_info info; - enum i40iw_status_code ret; - int err = 0; - unsigned long flags; - bool inv_stag; - - iwqp = (struct i40iw_qp *)ibqp; - ukqp = &iwqp->sc_qp.qp_uk; - - spin_lock_irqsave(&iwqp->lock, flags); - - if (iwqp->flush_issued) { - err = -EINVAL; - goto out; - } - - while (ib_wr) { - inv_stag = false; - memset(&info, 0, sizeof(info)); - info.wr_id = (u64)(ib_wr->wr_id); - if ((ib_wr->send_flags & IB_SEND_SIGNALED) || iwqp->sig_all) - info.signaled = true; - if (ib_wr->send_flags & IB_SEND_FENCE) - info.read_fence = true; - - switch (ib_wr->opcode) { - case IB_WR_SEND: - case IB_WR_SEND_WITH_INV: - if (ib_wr->opcode == IB_WR_SEND) { - if (ib_wr->send_flags & IB_SEND_SOLICITED) - info.op_type = I40IW_OP_TYPE_SEND_SOL; - else - info.op_type = I40IW_OP_TYPE_SEND; - } else { - if (ib_wr->send_flags & IB_SEND_SOLICITED) - info.op_type = I40IW_OP_TYPE_SEND_SOL_INV; - else - info.op_type = I40IW_OP_TYPE_SEND_INV; - } - - if (ib_wr->send_flags & IB_SEND_INLINE) { - info.op.inline_send.data = (void *)(unsigned long)ib_wr->sg_list[0].addr; - info.op.inline_send.len = ib_wr->sg_list[0].length; - ret = ukqp->ops.iw_inline_send(ukqp, &info, ib_wr->ex.invalidate_rkey, false); - } else { - info.op.send.num_sges = ib_wr->num_sge; - info.op.send.sg_list = (struct i40iw_sge *)ib_wr->sg_list; - ret = ukqp->ops.iw_send(ukqp, &info, ib_wr->ex.invalidate_rkey, false); - } - - if (ret) { - if (ret == I40IW_ERR_QP_TOOMANY_WRS_POSTED) - err = -ENOMEM; - else - err = -EINVAL; - } - break; - case IB_WR_RDMA_WRITE: - info.op_type = I40IW_OP_TYPE_RDMA_WRITE; - - if (ib_wr->send_flags & IB_SEND_INLINE) { - info.op.inline_rdma_write.data = (void *)(unsigned long)ib_wr->sg_list[0].addr; - info.op.inline_rdma_write.len = ib_wr->sg_list[0].length; - info.op.inline_rdma_write.rem_addr.tag_off = rdma_wr(ib_wr)->remote_addr; - info.op.inline_rdma_write.rem_addr.stag = rdma_wr(ib_wr)->rkey; - ret = ukqp->ops.iw_inline_rdma_write(ukqp, &info, false); - } else { - info.op.rdma_write.lo_sg_list = (void *)ib_wr->sg_list; - info.op.rdma_write.num_lo_sges = ib_wr->num_sge; - info.op.rdma_write.rem_addr.tag_off = rdma_wr(ib_wr)->remote_addr; - info.op.rdma_write.rem_addr.stag = rdma_wr(ib_wr)->rkey; - ret = ukqp->ops.iw_rdma_write(ukqp, &info, false); - } - - if (ret) { - if (ret == I40IW_ERR_QP_TOOMANY_WRS_POSTED) - err = -ENOMEM; - else - err = -EINVAL; - } - break; - case IB_WR_RDMA_READ_WITH_INV: - inv_stag = true; - fallthrough; - case IB_WR_RDMA_READ: - if (ib_wr->num_sge > I40IW_MAX_SGE_RD) { - err = -EINVAL; - break; - } - info.op_type = I40IW_OP_TYPE_RDMA_READ; - info.op.rdma_read.rem_addr.tag_off = rdma_wr(ib_wr)->remote_addr; - info.op.rdma_read.rem_addr.stag = rdma_wr(ib_wr)->rkey; - info.op.rdma_read.lo_addr.tag_off = ib_wr->sg_list->addr; - info.op.rdma_read.lo_addr.stag = ib_wr->sg_list->lkey; - info.op.rdma_read.lo_addr.len = ib_wr->sg_list->length; - ret = ukqp->ops.iw_rdma_read(ukqp, &info, inv_stag, false); - if (ret) { - if (ret == I40IW_ERR_QP_TOOMANY_WRS_POSTED) - err = -ENOMEM; - else - err = -EINVAL; - } - break; - case IB_WR_LOCAL_INV: - info.op_type = I40IW_OP_TYPE_INV_STAG; - info.op.inv_local_stag.target_stag = ib_wr->ex.invalidate_rkey; - ret = ukqp->ops.iw_stag_local_invalidate(ukqp, &info, true); - if (ret) - err = -ENOMEM; - break; - case IB_WR_REG_MR: - { - struct i40iw_mr *iwmr = to_iwmr(reg_wr(ib_wr)->mr); - int flags = reg_wr(ib_wr)->access; - struct i40iw_pble_alloc *palloc = &iwmr->iwpbl.pble_alloc; - struct i40iw_sc_dev *dev = &iwqp->iwdev->sc_dev; - struct i40iw_fast_reg_stag_info info; - - memset(&info, 0, sizeof(info)); - info.access_rights = I40IW_ACCESS_FLAGS_LOCALREAD; - info.access_rights |= i40iw_get_user_access(flags); - info.stag_key = reg_wr(ib_wr)->key & 0xff; - info.stag_idx = reg_wr(ib_wr)->key >> 8; - info.page_size = reg_wr(ib_wr)->mr->page_size; - info.wr_id = ib_wr->wr_id; - - info.addr_type = I40IW_ADDR_TYPE_VA_BASED; - info.va = (void *)(uintptr_t)iwmr->ibmr.iova; - info.total_len = iwmr->ibmr.length; - info.reg_addr_pa = *(u64 *)palloc->level1.addr; - info.first_pm_pbl_index = palloc->level1.idx; - info.local_fence = ib_wr->send_flags & IB_SEND_FENCE; - info.signaled = ib_wr->send_flags & IB_SEND_SIGNALED; - - if (iwmr->npages > I40IW_MIN_PAGES_PER_FMR) - info.chunk_size = 1; - - ret = dev->iw_priv_qp_ops->iw_mr_fast_register(&iwqp->sc_qp, &info, true); - if (ret) - err = -ENOMEM; - break; - } - default: - err = -EINVAL; - i40iw_pr_err(" upost_send bad opcode = 0x%x\n", - ib_wr->opcode); - break; - } - - if (err) - break; - ib_wr = ib_wr->next; - } - -out: - if (err) - *bad_wr = ib_wr; - else - ukqp->ops.iw_qp_post_wr(ukqp); - spin_unlock_irqrestore(&iwqp->lock, flags); - - return err; -} - -/** - * i40iw_post_recv - post receive wr for kernel application - * @ibqp: ib qp pointer - * @ib_wr: work request for receive - * @bad_wr: bad wr caused an error - */ -static int i40iw_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *ib_wr, - const struct ib_recv_wr **bad_wr) -{ - struct i40iw_qp *iwqp; - struct i40iw_qp_uk *ukqp; - struct i40iw_post_rq_info post_recv; - struct i40iw_sge sg_list[I40IW_MAX_WQ_FRAGMENT_COUNT]; - enum i40iw_status_code ret = 0; - unsigned long flags; - int err = 0; - - iwqp = (struct i40iw_qp *)ibqp; - ukqp = &iwqp->sc_qp.qp_uk; - - memset(&post_recv, 0, sizeof(post_recv)); - spin_lock_irqsave(&iwqp->lock, flags); - - if (iwqp->flush_issued) { - err = -EINVAL; - goto out; - } - - while (ib_wr) { - post_recv.num_sges = ib_wr->num_sge; - post_recv.wr_id = ib_wr->wr_id; - i40iw_copy_sg_list(sg_list, ib_wr->sg_list, ib_wr->num_sge); - post_recv.sg_list = sg_list; - ret = ukqp->ops.iw_post_receive(ukqp, &post_recv); - if (ret) { - i40iw_pr_err(" post_recv err %d\n", ret); - if (ret == I40IW_ERR_QP_TOOMANY_WRS_POSTED) - err = -ENOMEM; - else - err = -EINVAL; - *bad_wr = ib_wr; - goto out; - } - ib_wr = ib_wr->next; - } - out: - spin_unlock_irqrestore(&iwqp->lock, flags); - return err; -} - -/** - * i40iw_poll_cq - poll cq for completion (kernel apps) - * @ibcq: cq to poll - * @num_entries: number of entries to poll - * @entry: wr of entry completed - */ -static int i40iw_poll_cq(struct ib_cq *ibcq, - int num_entries, - struct ib_wc *entry) -{ - struct i40iw_cq *iwcq; - int cqe_count = 0; - struct i40iw_cq_poll_info cq_poll_info; - enum i40iw_status_code ret; - struct i40iw_cq_uk *ukcq; - struct i40iw_sc_qp *qp; - struct i40iw_qp *iwqp; - unsigned long flags; - - iwcq = (struct i40iw_cq *)ibcq; - ukcq = &iwcq->sc_cq.cq_uk; - - spin_lock_irqsave(&iwcq->lock, flags); - while (cqe_count < num_entries) { - ret = ukcq->ops.iw_cq_poll_completion(ukcq, &cq_poll_info); - if (ret == I40IW_ERR_QUEUE_EMPTY) { - break; - } else if (ret == I40IW_ERR_QUEUE_DESTROYED) { - continue; - } else if (ret) { - if (!cqe_count) - cqe_count = -1; - break; - } - entry->wc_flags = 0; - entry->wr_id = cq_poll_info.wr_id; - if (cq_poll_info.error) { - entry->status = IB_WC_WR_FLUSH_ERR; - entry->vendor_err = cq_poll_info.major_err << 16 | cq_poll_info.minor_err; - } else { - entry->status = IB_WC_SUCCESS; - } - - switch (cq_poll_info.op_type) { - case I40IW_OP_TYPE_RDMA_WRITE: - entry->opcode = IB_WC_RDMA_WRITE; - break; - case I40IW_OP_TYPE_RDMA_READ_INV_STAG: - case I40IW_OP_TYPE_RDMA_READ: - entry->opcode = IB_WC_RDMA_READ; - break; - case I40IW_OP_TYPE_SEND_SOL: - case I40IW_OP_TYPE_SEND_SOL_INV: - case I40IW_OP_TYPE_SEND_INV: - case I40IW_OP_TYPE_SEND: - entry->opcode = IB_WC_SEND; - break; - case I40IW_OP_TYPE_REC: - entry->opcode = IB_WC_RECV; - break; - default: - entry->opcode = IB_WC_RECV; - break; - } - - entry->ex.imm_data = 0; - qp = (struct i40iw_sc_qp *)cq_poll_info.qp_handle; - entry->qp = (struct ib_qp *)qp->back_qp; - entry->src_qp = cq_poll_info.qp_id; - iwqp = (struct i40iw_qp *)qp->back_qp; - if (iwqp->iwarp_state > I40IW_QP_STATE_RTS) { - if (!I40IW_RING_MORE_WORK(qp->qp_uk.sq_ring)) - complete(&iwqp->sq_drained); - if (!I40IW_RING_MORE_WORK(qp->qp_uk.rq_ring)) - complete(&iwqp->rq_drained); - } - entry->byte_len = cq_poll_info.bytes_xfered; - entry++; - cqe_count++; - } - spin_unlock_irqrestore(&iwcq->lock, flags); - return cqe_count; -} - -/** - * i40iw_req_notify_cq - arm cq kernel application - * @ibcq: cq to arm - * @notify_flags: notofication flags - */ -static int i40iw_req_notify_cq(struct ib_cq *ibcq, - enum ib_cq_notify_flags notify_flags) -{ - struct i40iw_cq *iwcq; - struct i40iw_cq_uk *ukcq; - unsigned long flags; - enum i40iw_completion_notify cq_notify = IW_CQ_COMPL_EVENT; - - iwcq = (struct i40iw_cq *)ibcq; - ukcq = &iwcq->sc_cq.cq_uk; - if (notify_flags == IB_CQ_SOLICITED) - cq_notify = IW_CQ_COMPL_SOLICITED; - spin_lock_irqsave(&iwcq->lock, flags); - ukcq->ops.iw_cq_request_notification(ukcq, cq_notify); - spin_unlock_irqrestore(&iwcq->lock, flags); - return 0; -} - -/** - * i40iw_port_immutable - return port's immutable data - * @ibdev: ib dev struct - * @port_num: port number - * @immutable: immutable data for the port return - */ -static int i40iw_port_immutable(struct ib_device *ibdev, u32 port_num, - struct ib_port_immutable *immutable) -{ - struct ib_port_attr attr; - int err; - - immutable->core_cap_flags = RDMA_CORE_PORT_IWARP; - - err = ib_query_port(ibdev, port_num, &attr); - - if (err) - return err; - - immutable->gid_tbl_len = attr.gid_tbl_len; - - return 0; -} - -static const char * const i40iw_hw_stat_names[] = { - // 32bit names - [I40IW_HW_STAT_INDEX_IP4RXDISCARD] = "ip4InDiscards", - [I40IW_HW_STAT_INDEX_IP4RXTRUNC] = "ip4InTruncatedPkts", - [I40IW_HW_STAT_INDEX_IP4TXNOROUTE] = "ip4OutNoRoutes", - [I40IW_HW_STAT_INDEX_IP6RXDISCARD] = "ip6InDiscards", - [I40IW_HW_STAT_INDEX_IP6RXTRUNC] = "ip6InTruncatedPkts", - [I40IW_HW_STAT_INDEX_IP6TXNOROUTE] = "ip6OutNoRoutes", - [I40IW_HW_STAT_INDEX_TCPRTXSEG] = "tcpRetransSegs", - [I40IW_HW_STAT_INDEX_TCPRXOPTERR] = "tcpInOptErrors", - [I40IW_HW_STAT_INDEX_TCPRXPROTOERR] = "tcpInProtoErrors", - // 64bit names - [I40IW_HW_STAT_INDEX_IP4RXOCTS + I40IW_HW_STAT_INDEX_MAX_32] = - "ip4InOctets", - [I40IW_HW_STAT_INDEX_IP4RXPKTS + I40IW_HW_STAT_INDEX_MAX_32] = - "ip4InPkts", - [I40IW_HW_STAT_INDEX_IP4RXFRAGS + I40IW_HW_STAT_INDEX_MAX_32] = - "ip4InReasmRqd", - [I40IW_HW_STAT_INDEX_IP4RXMCPKTS + I40IW_HW_STAT_INDEX_MAX_32] = - "ip4InMcastPkts", - [I40IW_HW_STAT_INDEX_IP4TXOCTS + I40IW_HW_STAT_INDEX_MAX_32] = - "ip4OutOctets", - [I40IW_HW_STAT_INDEX_IP4TXPKTS + I40IW_HW_STAT_INDEX_MAX_32] = - "ip4OutPkts", - [I40IW_HW_STAT_INDEX_IP4TXFRAGS + I40IW_HW_STAT_INDEX_MAX_32] = - "ip4OutSegRqd", - [I40IW_HW_STAT_INDEX_IP4TXMCPKTS + I40IW_HW_STAT_INDEX_MAX_32] = - "ip4OutMcastPkts", - [I40IW_HW_STAT_INDEX_IP6RXOCTS + I40IW_HW_STAT_INDEX_MAX_32] = - "ip6InOctets", - [I40IW_HW_STAT_INDEX_IP6RXPKTS + I40IW_HW_STAT_INDEX_MAX_32] = - "ip6InPkts", - [I40IW_HW_STAT_INDEX_IP6RXFRAGS + I40IW_HW_STAT_INDEX_MAX_32] = - "ip6InReasmRqd", - [I40IW_HW_STAT_INDEX_IP6RXMCPKTS + I40IW_HW_STAT_INDEX_MAX_32] = - "ip6InMcastPkts", - [I40IW_HW_STAT_INDEX_IP6TXOCTS + I40IW_HW_STAT_INDEX_MAX_32] = - "ip6OutOctets", - [I40IW_HW_STAT_INDEX_IP6TXPKTS + I40IW_HW_STAT_INDEX_MAX_32] = - "ip6OutPkts", - [I40IW_HW_STAT_INDEX_IP6TXFRAGS + I40IW_HW_STAT_INDEX_MAX_32] = - "ip6OutSegRqd", - [I40IW_HW_STAT_INDEX_IP6TXMCPKTS + I40IW_HW_STAT_INDEX_MAX_32] = - "ip6OutMcastPkts", - [I40IW_HW_STAT_INDEX_TCPRXSEGS + I40IW_HW_STAT_INDEX_MAX_32] = - "tcpInSegs", - [I40IW_HW_STAT_INDEX_TCPTXSEG + I40IW_HW_STAT_INDEX_MAX_32] = - "tcpOutSegs", - [I40IW_HW_STAT_INDEX_RDMARXRDS + I40IW_HW_STAT_INDEX_MAX_32] = - "iwInRdmaReads", - [I40IW_HW_STAT_INDEX_RDMARXSNDS + I40IW_HW_STAT_INDEX_MAX_32] = - "iwInRdmaSends", - [I40IW_HW_STAT_INDEX_RDMARXWRS + I40IW_HW_STAT_INDEX_MAX_32] = - "iwInRdmaWrites", - [I40IW_HW_STAT_INDEX_RDMATXRDS + I40IW_HW_STAT_INDEX_MAX_32] = - "iwOutRdmaReads", - [I40IW_HW_STAT_INDEX_RDMATXSNDS + I40IW_HW_STAT_INDEX_MAX_32] = - "iwOutRdmaSends", - [I40IW_HW_STAT_INDEX_RDMATXWRS + I40IW_HW_STAT_INDEX_MAX_32] = - "iwOutRdmaWrites", - [I40IW_HW_STAT_INDEX_RDMAVBND + I40IW_HW_STAT_INDEX_MAX_32] = - "iwRdmaBnd", - [I40IW_HW_STAT_INDEX_RDMAVINV + I40IW_HW_STAT_INDEX_MAX_32] = - "iwRdmaInv" -}; - -static void i40iw_get_dev_fw_str(struct ib_device *dev, char *str) -{ - struct i40iw_device *iwdev = to_iwdev(dev); - - snprintf(str, IB_FW_VERSION_NAME_MAX, "%llu.%llu", - i40iw_fw_major_ver(&iwdev->sc_dev), - i40iw_fw_minor_ver(&iwdev->sc_dev)); -} - -/** - * i40iw_alloc_hw_stats - Allocate a hw stats structure - * @ibdev: device pointer from stack - * @port_num: port number - */ -static struct rdma_hw_stats *i40iw_alloc_hw_stats(struct ib_device *ibdev, - u32 port_num) -{ - struct i40iw_device *iwdev = to_iwdev(ibdev); - struct i40iw_sc_dev *dev = &iwdev->sc_dev; - int num_counters = I40IW_HW_STAT_INDEX_MAX_32 + - I40IW_HW_STAT_INDEX_MAX_64; - unsigned long lifespan = RDMA_HW_STATS_DEFAULT_LIFESPAN; - - BUILD_BUG_ON(ARRAY_SIZE(i40iw_hw_stat_names) != - (I40IW_HW_STAT_INDEX_MAX_32 + - I40IW_HW_STAT_INDEX_MAX_64)); - - /* - * PFs get the default update lifespan, but VFs only update once - * per second - */ - if (!dev->is_pf) - lifespan = 1000; - return rdma_alloc_hw_stats_struct(i40iw_hw_stat_names, num_counters, - lifespan); -} - -/** - * i40iw_get_hw_stats - Populates the rdma_hw_stats structure - * @ibdev: device pointer from stack - * @stats: stats pointer from stack - * @port_num: port number - * @index: which hw counter the stack is requesting we update - */ -static int i40iw_get_hw_stats(struct ib_device *ibdev, - struct rdma_hw_stats *stats, - u32 port_num, int index) -{ - struct i40iw_device *iwdev = to_iwdev(ibdev); - struct i40iw_sc_dev *dev = &iwdev->sc_dev; - struct i40iw_vsi_pestat *devstat = iwdev->vsi.pestat; - struct i40iw_dev_hw_stats *hw_stats = &devstat->hw_stats; - - if (dev->is_pf) { - i40iw_hw_stats_read_all(devstat, &devstat->hw_stats); - } else { - if (i40iw_vchnl_vf_get_pe_stats(dev, &devstat->hw_stats)) - return -ENOSYS; - } - - memcpy(&stats->value[0], hw_stats, sizeof(*hw_stats)); - - return stats->num_counters; -} - -/** - * i40iw_query_gid - Query port GID - * @ibdev: device pointer from stack - * @port: port number - * @index: Entry index - * @gid: Global ID - */ -static int i40iw_query_gid(struct ib_device *ibdev, - u32 port, - int index, - union ib_gid *gid) -{ - struct i40iw_device *iwdev = to_iwdev(ibdev); - - memset(gid->raw, 0, sizeof(gid->raw)); - ether_addr_copy(gid->raw, iwdev->netdev->dev_addr); - return 0; -} - -static const struct ib_device_ops i40iw_dev_ops = { - .owner = THIS_MODULE, - .driver_id = RDMA_DRIVER_I40IW, - /* NOTE: Older kernels wrongly use 0 for the uverbs_abi_ver */ - .uverbs_abi_ver = I40IW_ABI_VER, - - .alloc_hw_stats = i40iw_alloc_hw_stats, - .alloc_mr = i40iw_alloc_mr, - .alloc_pd = i40iw_alloc_pd, - .alloc_ucontext = i40iw_alloc_ucontext, - .create_cq = i40iw_create_cq, - .create_qp = i40iw_create_qp, - .dealloc_pd = i40iw_dealloc_pd, - .dealloc_ucontext = i40iw_dealloc_ucontext, - .dereg_mr = i40iw_dereg_mr, - .destroy_cq = i40iw_destroy_cq, - .destroy_qp = i40iw_destroy_qp, - .drain_rq = i40iw_drain_rq, - .drain_sq = i40iw_drain_sq, - .get_dev_fw_str = i40iw_get_dev_fw_str, - .get_dma_mr = i40iw_get_dma_mr, - .get_hw_stats = i40iw_get_hw_stats, - .get_port_immutable = i40iw_port_immutable, - .iw_accept = i40iw_accept, - .iw_add_ref = i40iw_qp_add_ref, - .iw_connect = i40iw_connect, - .iw_create_listen = i40iw_create_listen, - .iw_destroy_listen = i40iw_destroy_listen, - .iw_get_qp = i40iw_get_qp, - .iw_reject = i40iw_reject, - .iw_rem_ref = i40iw_qp_rem_ref, - .map_mr_sg = i40iw_map_mr_sg, - .mmap = i40iw_mmap, - .modify_qp = i40iw_modify_qp, - .poll_cq = i40iw_poll_cq, - .post_recv = i40iw_post_recv, - .post_send = i40iw_post_send, - .query_device = i40iw_query_device, - .query_gid = i40iw_query_gid, - .query_port = i40iw_query_port, - .query_qp = i40iw_query_qp, - .reg_user_mr = i40iw_reg_user_mr, - .req_notify_cq = i40iw_req_notify_cq, - INIT_RDMA_OBJ_SIZE(ib_pd, i40iw_pd, ibpd), - INIT_RDMA_OBJ_SIZE(ib_cq, i40iw_cq, ibcq), - INIT_RDMA_OBJ_SIZE(ib_ucontext, i40iw_ucontext, ibucontext), -}; - -/** - * i40iw_init_rdma_device - initialization of iwarp device - * @iwdev: iwarp device - */ -static struct i40iw_ib_device *i40iw_init_rdma_device(struct i40iw_device *iwdev) -{ - struct i40iw_ib_device *iwibdev; - struct net_device *netdev = iwdev->netdev; - struct pci_dev *pcidev = iwdev->hw.pcidev; - - iwibdev = ib_alloc_device(i40iw_ib_device, ibdev); - if (!iwibdev) { - i40iw_pr_err("iwdev == NULL\n"); - return NULL; - } - iwdev->iwibdev = iwibdev; - iwibdev->iwdev = iwdev; - - iwibdev->ibdev.node_type = RDMA_NODE_RNIC; - ether_addr_copy((u8 *)&iwibdev->ibdev.node_guid, netdev->dev_addr); - - iwibdev->ibdev.phys_port_cnt = 1; - iwibdev->ibdev.num_comp_vectors = iwdev->ceqs_count; - iwibdev->ibdev.dev.parent = &pcidev->dev; - memcpy(iwibdev->ibdev.iw_ifname, netdev->name, - sizeof(iwibdev->ibdev.iw_ifname)); - ib_set_device_ops(&iwibdev->ibdev, &i40iw_dev_ops); - - return iwibdev; -} - -/** - * i40iw_port_ibevent - indicate port event - * @iwdev: iwarp device - */ -void i40iw_port_ibevent(struct i40iw_device *iwdev) -{ - struct i40iw_ib_device *iwibdev = iwdev->iwibdev; - struct ib_event event; - - event.device = &iwibdev->ibdev; - event.element.port_num = 1; - event.event = iwdev->iw_status ? IB_EVENT_PORT_ACTIVE : IB_EVENT_PORT_ERR; - ib_dispatch_event(&event); -} - -/** - * i40iw_destroy_rdma_device - destroy rdma device and free resources - * @iwibdev: IB device ptr - */ -void i40iw_destroy_rdma_device(struct i40iw_ib_device *iwibdev) -{ - ib_unregister_device(&iwibdev->ibdev); - wait_event_timeout(iwibdev->iwdev->close_wq, - !atomic64_read(&iwibdev->iwdev->use_count), - I40IW_EVENT_TIMEOUT); - ib_dealloc_device(&iwibdev->ibdev); -} - -/** - * i40iw_register_rdma_device - register iwarp device to IB - * @iwdev: iwarp device - */ -int i40iw_register_rdma_device(struct i40iw_device *iwdev) -{ - int ret; - struct i40iw_ib_device *iwibdev; - - iwdev->iwibdev = i40iw_init_rdma_device(iwdev); - if (!iwdev->iwibdev) - return -ENOMEM; - iwibdev = iwdev->iwibdev; - rdma_set_device_sysfs_group(&iwibdev->ibdev, &i40iw_attr_group); - ret = ib_device_set_netdev(&iwibdev->ibdev, iwdev->netdev, 1); - if (ret) - goto error; - - dma_set_max_seg_size(&iwdev->hw.pcidev->dev, UINT_MAX); - ret = ib_register_device(&iwibdev->ibdev, "i40iw%d", &iwdev->hw.pcidev->dev); - if (ret) - goto error; - - return 0; -error: - ib_dealloc_device(&iwdev->iwibdev->ibdev); - return ret; -} diff --git a/drivers/infiniband/hw/i40iw/i40iw_verbs.h b/drivers/infiniband/hw/i40iw/i40iw_verbs.h deleted file mode 100644 index bab71f3e5637..000000000000 --- a/drivers/infiniband/hw/i40iw/i40iw_verbs.h +++ /dev/null @@ -1,179 +0,0 @@ -/******************************************************************************* -* -* Copyright (c) 2015-2016 Intel Corporation. All rights reserved. -* -* This software is available to you under a choice of one of two -* licenses. You may choose to be licensed under the terms of the GNU -* General Public License (GPL) Version 2, available from the file -* COPYING in the main directory of this source tree, or the -* OpenFabrics.org BSD license below: -* -* Redistribution and use in source and binary forms, with or -* without modification, are permitted provided that the following -* conditions are met: -* -* - Redistributions of source code must retain the above -* copyright notice, this list of conditions and the following -* disclaimer. -* -* - Redistributions in binary form must reproduce the above -* copyright notice, this list of conditions and the following -* disclaimer in the documentation and/or other materials -* provided with the distribution. -* -* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS -* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN -* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -* SOFTWARE. -* -*******************************************************************************/ - -#ifndef I40IW_VERBS_H -#define I40IW_VERBS_H - -struct i40iw_ucontext { - struct ib_ucontext ibucontext; - struct i40iw_device *iwdev; - struct list_head cq_reg_mem_list; - spinlock_t cq_reg_mem_list_lock; /* memory list for cq's */ - struct list_head qp_reg_mem_list; - spinlock_t qp_reg_mem_list_lock; /* memory list for qp's */ - int abi_ver; -}; - -struct i40iw_pd { - struct ib_pd ibpd; - struct i40iw_sc_pd sc_pd; - atomic_t usecount; -}; - -struct i40iw_hmc_pble { - union { - u32 idx; - dma_addr_t addr; - }; -}; - -struct i40iw_cq_mr { - struct i40iw_hmc_pble cq_pbl; - dma_addr_t shadow; -}; - -struct i40iw_qp_mr { - struct i40iw_hmc_pble sq_pbl; - struct i40iw_hmc_pble rq_pbl; - dma_addr_t shadow; - struct page *sq_page; -}; - -struct i40iw_pbl { - struct list_head list; - union { - struct i40iw_qp_mr qp_mr; - struct i40iw_cq_mr cq_mr; - }; - - bool pbl_allocated; - bool on_list; - u64 user_base; - struct i40iw_pble_alloc pble_alloc; - struct i40iw_mr *iwmr; -}; - -#define MAX_SAVE_PAGE_ADDRS 4 -struct i40iw_mr { - union { - struct ib_mr ibmr; - struct ib_mw ibmw; - }; - struct ib_umem *region; - u16 type; - u32 page_cnt; - u64 page_size; - u32 npages; - u32 stag; - u64 length; - u64 pgaddrmem[MAX_SAVE_PAGE_ADDRS]; - struct i40iw_pbl iwpbl; -}; - -struct i40iw_cq { - struct ib_cq ibcq; - struct i40iw_sc_cq sc_cq; - u16 cq_head; - u16 cq_size; - u16 cq_number; - bool user_mode; - u32 polled_completions; - u32 cq_mem_size; - struct i40iw_dma_mem kmem; - spinlock_t lock; /* for poll cq */ - struct i40iw_pbl *iwpbl; -}; - -struct disconn_work { - struct work_struct work; - struct i40iw_qp *iwqp; -}; - -struct iw_cm_id; -struct ietf_mpa_frame; -struct i40iw_ud_file; - -struct i40iw_qp_kmode { - struct i40iw_dma_mem dma_mem; - u64 *wrid_mem; -}; - -struct i40iw_qp { - struct ib_qp ibqp; - struct i40iw_sc_qp sc_qp; - struct i40iw_device *iwdev; - struct i40iw_cq *iwscq; - struct i40iw_cq *iwrcq; - struct i40iw_pd *iwpd; - struct i40iw_qp_host_ctx_info ctx_info; - struct i40iwarp_offload_info iwarp_info; - void *allocated_buffer; - refcount_t refcount; - struct iw_cm_id *cm_id; - void *cm_node; - struct ib_mr *lsmm_mr; - struct work_struct work; - enum ib_qp_state ibqp_state; - u32 iwarp_state; - u32 qp_mem_size; - u32 last_aeq; - atomic_t close_timer_started; - spinlock_t lock; /* for post work requests */ - struct i40iw_qp_context *iwqp_context; - void *pbl_vbase; - dma_addr_t pbl_pbase; - struct page *page; - u8 active_conn:1; - u8 user_mode:1; - u8 hte_added:1; - u8 flush_issued:1; - u8 destroyed:1; - u8 sig_all:1; - u8 pau_mode:1; - u8 rsvd:1; - u16 term_sq_flush_code; - u16 term_rq_flush_code; - u8 hw_iwarp_state; - u8 hw_tcp_state; - struct i40iw_qp_kmode kqp; - struct i40iw_dma_mem host_ctx; - struct timer_list terminate_timer; - struct i40iw_pbl iwpbl; - struct i40iw_dma_mem q2_ctx_mem; - struct i40iw_dma_mem ietf_mem; - struct completion sq_drained; - struct completion rq_drained; - struct completion free_qp; -}; -#endif diff --git a/drivers/infiniband/hw/i40iw/i40iw_vf.c b/drivers/infiniband/hw/i40iw/i40iw_vf.c deleted file mode 100644 index e33d4810965c..000000000000 --- a/drivers/infiniband/hw/i40iw/i40iw_vf.c +++ /dev/null @@ -1,85 +0,0 @@ -/******************************************************************************* -* -* Copyright (c) 2015-2016 Intel Corporation. All rights reserved. -* -* This software is available to you under a choice of one of two -* licenses. You may choose to be licensed under the terms of the GNU -* General Public License (GPL) Version 2, available from the file -* COPYING in the main directory of this source tree, or the -* OpenFabrics.org BSD license below: -* -* Redistribution and use in source and binary forms, with or -* without modification, are permitted provided that the following -* conditions are met: -* -* - Redistributions of source code must retain the above -* copyright notice, this list of conditions and the following -* disclaimer. -* -* - Redistributions in binary form must reproduce the above -* copyright notice, this list of conditions and the following -* disclaimer in the documentation and/or other materials -* provided with the distribution. -* -* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS -* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN -* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -* SOFTWARE. -* -*******************************************************************************/ - -#include "i40iw_osdep.h" -#include "i40iw_register.h" -#include "i40iw_status.h" -#include "i40iw_hmc.h" -#include "i40iw_d.h" -#include "i40iw_type.h" -#include "i40iw_p.h" -#include "i40iw_vf.h" - -/** - * i40iw_manage_vf_pble_bp - manage vf pble - * @cqp: cqp for cqp' sq wqe - * @info: pble info - * @scratch: pointer for completion - * @post_sq: to post and ring - */ -enum i40iw_status_code i40iw_manage_vf_pble_bp(struct i40iw_sc_cqp *cqp, - struct i40iw_manage_vf_pble_info *info, - u64 scratch, - bool post_sq) -{ - u64 *wqe; - u64 temp, header, pd_pl_pba = 0; - - wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch); - if (!wqe) - return I40IW_ERR_RING_FULL; - - temp = LS_64(info->pd_entry_cnt, I40IW_CQPSQ_MVPBP_PD_ENTRY_CNT) | - LS_64(info->first_pd_index, I40IW_CQPSQ_MVPBP_FIRST_PD_INX) | - LS_64(info->sd_index, I40IW_CQPSQ_MVPBP_SD_INX); - set_64bit_val(wqe, 16, temp); - - header = LS_64((info->inv_pd_ent ? 1 : 0), I40IW_CQPSQ_MVPBP_INV_PD_ENT) | - LS_64(I40IW_CQP_OP_MANAGE_VF_PBLE_BP, I40IW_CQPSQ_OPCODE) | - LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID); - set_64bit_val(wqe, 24, header); - - pd_pl_pba = LS_64(info->pd_pl_pba >> 3, I40IW_CQPSQ_MVPBP_PD_PLPBA); - set_64bit_val(wqe, 32, pd_pl_pba); - - i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "MANAGE VF_PBLE_BP WQE", wqe, I40IW_CQP_WQE_SIZE * 8); - - if (post_sq) - i40iw_sc_cqp_post_sq(cqp); - return 0; -} - -const struct i40iw_vf_cqp_ops iw_vf_cqp_ops = { - i40iw_manage_vf_pble_bp -}; diff --git a/drivers/infiniband/hw/i40iw/i40iw_vf.h b/drivers/infiniband/hw/i40iw/i40iw_vf.h deleted file mode 100644 index 4359559ece9c..000000000000 --- a/drivers/infiniband/hw/i40iw/i40iw_vf.h +++ /dev/null @@ -1,62 +0,0 @@ -/******************************************************************************* -* -* Copyright (c) 2015-2016 Intel Corporation. All rights reserved. -* -* This software is available to you under a choice of one of two -* licenses. You may choose to be licensed under the terms of the GNU -* General Public License (GPL) Version 2, available from the file -* COPYING in the main directory of this source tree, or the -* OpenFabrics.org BSD license below: -* -* Redistribution and use in source and binary forms, with or -* without modification, are permitted provided that the following -* conditions are met: -* -* - Redistributions of source code must retain the above -* copyright notice, this list of conditions and the following -* disclaimer. -* -* - Redistributions in binary form must reproduce the above -* copyright notice, this list of conditions and the following -* disclaimer in the documentation and/or other materials -* provided with the distribution. -* -* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS -* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN -* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -* SOFTWARE. -* -*******************************************************************************/ - -#ifndef I40IW_VF_H -#define I40IW_VF_H - -struct i40iw_sc_cqp; - -struct i40iw_manage_vf_pble_info { - u32 sd_index; - u16 first_pd_index; - u16 pd_entry_cnt; - u8 inv_pd_ent; - u64 pd_pl_pba; -}; - -struct i40iw_vf_cqp_ops { - enum i40iw_status_code (*manage_vf_pble_bp)(struct i40iw_sc_cqp *, - struct i40iw_manage_vf_pble_info *, - u64, - bool); -}; - -enum i40iw_status_code i40iw_manage_vf_pble_bp(struct i40iw_sc_cqp *cqp, - struct i40iw_manage_vf_pble_info *info, - u64 scratch, - bool post_sq); - -extern const struct i40iw_vf_cqp_ops iw_vf_cqp_ops; - -#endif diff --git a/drivers/infiniband/hw/i40iw/i40iw_virtchnl.c b/drivers/infiniband/hw/i40iw/i40iw_virtchnl.c deleted file mode 100644 index e34a1522132c..000000000000 --- a/drivers/infiniband/hw/i40iw/i40iw_virtchnl.c +++ /dev/null @@ -1,759 +0,0 @@ -/******************************************************************************* -* -* Copyright (c) 2015-2016 Intel Corporation. All rights reserved. -* -* This software is available to you under a choice of one of two -* licenses. You may choose to be licensed under the terms of the GNU -* General Public License (GPL) Version 2, available from the file -* COPYING in the main directory of this source tree, or the -* OpenFabrics.org BSD license below: -* -* Redistribution and use in source and binary forms, with or -* without modification, are permitted provided that the following -* conditions are met: -* -* - Redistributions of source code must retain the above -* copyright notice, this list of conditions and the following -* disclaimer. -* -* - Redistributions in binary form must reproduce the above -* copyright notice, this list of conditions and the following -* disclaimer in the documentation and/or other materials -* provided with the distribution. -* -* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS -* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN -* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -* SOFTWARE. -* -*******************************************************************************/ - -#include "i40iw_osdep.h" -#include "i40iw_register.h" -#include "i40iw_status.h" -#include "i40iw_hmc.h" -#include "i40iw_d.h" -#include "i40iw_type.h" -#include "i40iw_p.h" -#include "i40iw_virtchnl.h" - -/** - * vchnl_vf_send_get_ver_req - Request Channel version - * @dev: IWARP device pointer - * @vchnl_req: Virtual channel message request pointer - */ -static enum i40iw_status_code vchnl_vf_send_get_ver_req(struct i40iw_sc_dev *dev, - struct i40iw_virtchnl_req *vchnl_req) -{ - enum i40iw_status_code ret_code = I40IW_ERR_NOT_READY; - struct i40iw_virtchnl_op_buf *vchnl_msg = vchnl_req->vchnl_msg; - - if (!dev->vchnl_up) - return ret_code; - - memset(vchnl_msg, 0, sizeof(*vchnl_msg)); - vchnl_msg->iw_chnl_op_ctx = (uintptr_t)vchnl_req; - vchnl_msg->iw_chnl_buf_len = sizeof(*vchnl_msg); - vchnl_msg->iw_op_code = I40IW_VCHNL_OP_GET_VER; - vchnl_msg->iw_op_ver = I40IW_VCHNL_OP_GET_VER_V0; - ret_code = dev->vchnl_if.vchnl_send(dev, 0, (u8 *)vchnl_msg, vchnl_msg->iw_chnl_buf_len); - if (ret_code) - i40iw_debug(dev, I40IW_DEBUG_VIRT, - "%s: virt channel send failed 0x%x\n", __func__, ret_code); - return ret_code; -} - -/** - * vchnl_vf_send_get_hmc_fcn_req - Request HMC Function from VF - * @dev: IWARP device pointer - * @vchnl_req: Virtual channel message request pointer - */ -static enum i40iw_status_code vchnl_vf_send_get_hmc_fcn_req(struct i40iw_sc_dev *dev, - struct i40iw_virtchnl_req *vchnl_req) -{ - enum i40iw_status_code ret_code = I40IW_ERR_NOT_READY; - struct i40iw_virtchnl_op_buf *vchnl_msg = vchnl_req->vchnl_msg; - - if (!dev->vchnl_up) - return ret_code; - - memset(vchnl_msg, 0, sizeof(*vchnl_msg)); - vchnl_msg->iw_chnl_op_ctx = (uintptr_t)vchnl_req; - vchnl_msg->iw_chnl_buf_len = sizeof(*vchnl_msg); - vchnl_msg->iw_op_code = I40IW_VCHNL_OP_GET_HMC_FCN; - vchnl_msg->iw_op_ver = I40IW_VCHNL_OP_GET_HMC_FCN_V0; - ret_code = dev->vchnl_if.vchnl_send(dev, 0, (u8 *)vchnl_msg, vchnl_msg->iw_chnl_buf_len); - if (ret_code) - i40iw_debug(dev, I40IW_DEBUG_VIRT, - "%s: virt channel send failed 0x%x\n", __func__, ret_code); - return ret_code; -} - -/** - * vchnl_vf_send_get_pe_stats_req - Request PE stats from VF - * @dev: IWARP device pointer - * @vchnl_req: Virtual channel message request pointer - */ -static enum i40iw_status_code vchnl_vf_send_get_pe_stats_req(struct i40iw_sc_dev *dev, - struct i40iw_virtchnl_req *vchnl_req) -{ - enum i40iw_status_code ret_code = I40IW_ERR_NOT_READY; - struct i40iw_virtchnl_op_buf *vchnl_msg = vchnl_req->vchnl_msg; - - if (!dev->vchnl_up) - return ret_code; - - memset(vchnl_msg, 0, sizeof(*vchnl_msg)); - vchnl_msg->iw_chnl_op_ctx = (uintptr_t)vchnl_req; - vchnl_msg->iw_chnl_buf_len = sizeof(*vchnl_msg) + sizeof(struct i40iw_dev_hw_stats) - 1; - vchnl_msg->iw_op_code = I40IW_VCHNL_OP_GET_STATS; - vchnl_msg->iw_op_ver = I40IW_VCHNL_OP_GET_STATS_V0; - ret_code = dev->vchnl_if.vchnl_send(dev, 0, (u8 *)vchnl_msg, vchnl_msg->iw_chnl_buf_len); - if (ret_code) - i40iw_debug(dev, I40IW_DEBUG_VIRT, - "%s: virt channel send failed 0x%x\n", __func__, ret_code); - return ret_code; -} - -/* - * vchnl_vf_send_add_hmc_objs_req - Add HMC objects - * @dev: IWARP device pointer - * @vchnl_req: Virtual channel message request pointer - */ -static enum i40iw_status_code vchnl_vf_send_add_hmc_objs_req(struct i40iw_sc_dev *dev, - struct i40iw_virtchnl_req *vchnl_req, - enum i40iw_hmc_rsrc_type rsrc_type, - u32 start_index, - u32 rsrc_count) -{ - enum i40iw_status_code ret_code = I40IW_ERR_NOT_READY; - struct i40iw_virtchnl_op_buf *vchnl_msg = vchnl_req->vchnl_msg; - struct i40iw_virtchnl_hmc_obj_range *add_hmc_obj; - - if (!dev->vchnl_up) - return ret_code; - - add_hmc_obj = (struct i40iw_virtchnl_hmc_obj_range *)vchnl_msg->iw_chnl_buf; - memset(vchnl_msg, 0, sizeof(*vchnl_msg)); - memset(add_hmc_obj, 0, sizeof(*add_hmc_obj)); - vchnl_msg->iw_chnl_op_ctx = (uintptr_t)vchnl_req; - vchnl_msg->iw_chnl_buf_len = sizeof(*vchnl_msg) + sizeof(struct i40iw_virtchnl_hmc_obj_range) - 1; - vchnl_msg->iw_op_code = I40IW_VCHNL_OP_ADD_HMC_OBJ_RANGE; - vchnl_msg->iw_op_ver = I40IW_VCHNL_OP_ADD_HMC_OBJ_RANGE_V0; - add_hmc_obj->obj_type = (u16)rsrc_type; - add_hmc_obj->start_index = start_index; - add_hmc_obj->obj_count = rsrc_count; - ret_code = dev->vchnl_if.vchnl_send(dev, 0, (u8 *)vchnl_msg, vchnl_msg->iw_chnl_buf_len); - if (ret_code) - i40iw_debug(dev, I40IW_DEBUG_VIRT, - "%s: virt channel send failed 0x%x\n", __func__, ret_code); - return ret_code; -} - -/** - * vchnl_vf_send_del_hmc_objs_req - del HMC objects - * @dev: IWARP device pointer - * @vchnl_req: Virtual channel message request pointer - * @rsrc_type: resource type to delete - * @start_index: starting index for resource - * @rsrc_count: number of resource type to delete - */ -static enum i40iw_status_code vchnl_vf_send_del_hmc_objs_req(struct i40iw_sc_dev *dev, - struct i40iw_virtchnl_req *vchnl_req, - enum i40iw_hmc_rsrc_type rsrc_type, - u32 start_index, - u32 rsrc_count) -{ - enum i40iw_status_code ret_code = I40IW_ERR_NOT_READY; - struct i40iw_virtchnl_op_buf *vchnl_msg = vchnl_req->vchnl_msg; - struct i40iw_virtchnl_hmc_obj_range *add_hmc_obj; - - if (!dev->vchnl_up) - return ret_code; - - add_hmc_obj = (struct i40iw_virtchnl_hmc_obj_range *)vchnl_msg->iw_chnl_buf; - memset(vchnl_msg, 0, sizeof(*vchnl_msg)); - memset(add_hmc_obj, 0, sizeof(*add_hmc_obj)); - vchnl_msg->iw_chnl_op_ctx = (uintptr_t)vchnl_req; - vchnl_msg->iw_chnl_buf_len = sizeof(*vchnl_msg) + sizeof(struct i40iw_virtchnl_hmc_obj_range) - 1; - vchnl_msg->iw_op_code = I40IW_VCHNL_OP_DEL_HMC_OBJ_RANGE; - vchnl_msg->iw_op_ver = I40IW_VCHNL_OP_DEL_HMC_OBJ_RANGE_V0; - add_hmc_obj->obj_type = (u16)rsrc_type; - add_hmc_obj->start_index = start_index; - add_hmc_obj->obj_count = rsrc_count; - ret_code = dev->vchnl_if.vchnl_send(dev, 0, (u8 *)vchnl_msg, vchnl_msg->iw_chnl_buf_len); - if (ret_code) - i40iw_debug(dev, I40IW_DEBUG_VIRT, - "%s: virt channel send failed 0x%x\n", __func__, ret_code); - return ret_code; -} - -/** - * vchnl_pf_send_get_ver_resp - Send channel version to VF - * @dev: IWARP device pointer - * @vf_id: Virtual function ID associated with the message - * @vchnl_msg: Virtual channel message buffer pointer - */ -static void vchnl_pf_send_get_ver_resp(struct i40iw_sc_dev *dev, - u32 vf_id, - struct i40iw_virtchnl_op_buf *vchnl_msg) -{ - enum i40iw_status_code ret_code; - u8 resp_buffer[sizeof(struct i40iw_virtchnl_resp_buf) + sizeof(u32) - 1]; - struct i40iw_virtchnl_resp_buf *vchnl_msg_resp = (struct i40iw_virtchnl_resp_buf *)resp_buffer; - - memset(resp_buffer, 0, sizeof(*resp_buffer)); - vchnl_msg_resp->iw_chnl_op_ctx = vchnl_msg->iw_chnl_op_ctx; - vchnl_msg_resp->iw_chnl_buf_len = sizeof(resp_buffer); - vchnl_msg_resp->iw_op_ret_code = I40IW_SUCCESS; - *((u32 *)vchnl_msg_resp->iw_chnl_buf) = I40IW_VCHNL_CHNL_VER_V0; - ret_code = dev->vchnl_if.vchnl_send(dev, vf_id, resp_buffer, sizeof(resp_buffer)); - if (ret_code) - i40iw_debug(dev, I40IW_DEBUG_VIRT, - "%s: virt channel send failed 0x%x\n", __func__, ret_code); -} - -/** - * vchnl_pf_send_get_hmc_fcn_resp - Send HMC Function to VF - * @dev: IWARP device pointer - * @vf_id: Virtual function ID associated with the message - * @vchnl_msg: Virtual channel message buffer pointer - * @hmc_fcn: HMC function index pointer - */ -static void vchnl_pf_send_get_hmc_fcn_resp(struct i40iw_sc_dev *dev, - u32 vf_id, - struct i40iw_virtchnl_op_buf *vchnl_msg, - u16 hmc_fcn) -{ - enum i40iw_status_code ret_code; - u8 resp_buffer[sizeof(struct i40iw_virtchnl_resp_buf) + sizeof(u16) - 1]; - struct i40iw_virtchnl_resp_buf *vchnl_msg_resp = (struct i40iw_virtchnl_resp_buf *)resp_buffer; - - memset(resp_buffer, 0, sizeof(*resp_buffer)); - vchnl_msg_resp->iw_chnl_op_ctx = vchnl_msg->iw_chnl_op_ctx; - vchnl_msg_resp->iw_chnl_buf_len = sizeof(resp_buffer); - vchnl_msg_resp->iw_op_ret_code = I40IW_SUCCESS; - *((u16 *)vchnl_msg_resp->iw_chnl_buf) = hmc_fcn; - ret_code = dev->vchnl_if.vchnl_send(dev, vf_id, resp_buffer, sizeof(resp_buffer)); - if (ret_code) - i40iw_debug(dev, I40IW_DEBUG_VIRT, - "%s: virt channel send failed 0x%x\n", __func__, ret_code); -} - -/** - * vchnl_pf_send_get_pe_stats_resp - Send PE Stats to VF - * @dev: IWARP device pointer - * @vf_id: Virtual function ID associated with the message - * @vchnl_msg: Virtual channel message buffer pointer - * @hw_stats: HW Stats struct - */ - -static void vchnl_pf_send_get_pe_stats_resp(struct i40iw_sc_dev *dev, - u32 vf_id, - struct i40iw_virtchnl_op_buf *vchnl_msg, - struct i40iw_dev_hw_stats *hw_stats) -{ - enum i40iw_status_code ret_code; - u8 resp_buffer[sizeof(struct i40iw_virtchnl_resp_buf) + sizeof(struct i40iw_dev_hw_stats) - 1]; - struct i40iw_virtchnl_resp_buf *vchnl_msg_resp = (struct i40iw_virtchnl_resp_buf *)resp_buffer; - - memset(resp_buffer, 0, sizeof(*resp_buffer)); - vchnl_msg_resp->iw_chnl_op_ctx = vchnl_msg->iw_chnl_op_ctx; - vchnl_msg_resp->iw_chnl_buf_len = sizeof(resp_buffer); - vchnl_msg_resp->iw_op_ret_code = I40IW_SUCCESS; - *((struct i40iw_dev_hw_stats *)vchnl_msg_resp->iw_chnl_buf) = *hw_stats; - ret_code = dev->vchnl_if.vchnl_send(dev, vf_id, resp_buffer, sizeof(resp_buffer)); - if (ret_code) - i40iw_debug(dev, I40IW_DEBUG_VIRT, - "%s: virt channel send failed 0x%x\n", __func__, ret_code); -} - -/** - * vchnl_pf_send_error_resp - Send an error response to VF - * @dev: IWARP device pointer - * @vf_id: Virtual function ID associated with the message - * @vchnl_msg: Virtual channel message buffer pointer - * @op_ret_code: I40IW_ERR_* status code - */ -static void vchnl_pf_send_error_resp(struct i40iw_sc_dev *dev, u32 vf_id, - struct i40iw_virtchnl_op_buf *vchnl_msg, - u16 op_ret_code) -{ - enum i40iw_status_code ret_code; - u8 resp_buffer[sizeof(struct i40iw_virtchnl_resp_buf)]; - struct i40iw_virtchnl_resp_buf *vchnl_msg_resp = (struct i40iw_virtchnl_resp_buf *)resp_buffer; - - memset(resp_buffer, 0, sizeof(resp_buffer)); - vchnl_msg_resp->iw_chnl_op_ctx = vchnl_msg->iw_chnl_op_ctx; - vchnl_msg_resp->iw_chnl_buf_len = sizeof(resp_buffer); - vchnl_msg_resp->iw_op_ret_code = (u16)op_ret_code; - ret_code = dev->vchnl_if.vchnl_send(dev, vf_id, resp_buffer, sizeof(resp_buffer)); - if (ret_code) - i40iw_debug(dev, I40IW_DEBUG_VIRT, - "%s: virt channel send failed 0x%x\n", __func__, ret_code); -} - -/** - * pf_cqp_get_hmc_fcn_callback - Callback for Get HMC Fcn - * @dev: IWARP device pointer - * @callback_param: unused CQP callback parameter - * @cqe_info: CQE information pointer - */ -static void pf_cqp_get_hmc_fcn_callback(struct i40iw_sc_dev *dev, void *callback_param, - struct i40iw_ccq_cqe_info *cqe_info) -{ - struct i40iw_vfdev *vf_dev = callback_param; - struct i40iw_virt_mem vf_dev_mem; - - if (cqe_info->error) { - i40iw_debug(dev, I40IW_DEBUG_VIRT, - "CQP Completion Error on Get HMC Function. Maj = 0x%04x, Minor = 0x%04x\n", - cqe_info->maj_err_code, cqe_info->min_err_code); - dev->vf_dev[vf_dev->iw_vf_idx] = NULL; - vchnl_pf_send_error_resp(dev, vf_dev->vf_id, &vf_dev->vf_msg_buffer.vchnl_msg, - (u16)I40IW_ERR_CQP_COMPL_ERROR); - vf_dev_mem.va = vf_dev; - vf_dev_mem.size = sizeof(*vf_dev); - i40iw_free_virt_mem(dev->hw, &vf_dev_mem); - } else { - i40iw_debug(dev, I40IW_DEBUG_VIRT, - "CQP Completion Operation Return information = 0x%08x\n", - cqe_info->op_ret_val); - vf_dev->pmf_index = (u16)cqe_info->op_ret_val; - vf_dev->msg_count--; - vchnl_pf_send_get_hmc_fcn_resp(dev, - vf_dev->vf_id, - &vf_dev->vf_msg_buffer.vchnl_msg, - vf_dev->pmf_index); - } -} - -/** - * pf_add_hmc_obj_callback - Callback for Add HMC Object - * @work_vf_dev: pointer to the VF Device - */ -static void pf_add_hmc_obj_callback(void *work_vf_dev) -{ - struct i40iw_vfdev *vf_dev = (struct i40iw_vfdev *)work_vf_dev; - struct i40iw_hmc_info *hmc_info = &vf_dev->hmc_info; - struct i40iw_virtchnl_op_buf *vchnl_msg = &vf_dev->vf_msg_buffer.vchnl_msg; - struct i40iw_hmc_create_obj_info info; - struct i40iw_virtchnl_hmc_obj_range *add_hmc_obj; - enum i40iw_status_code ret_code; - - if (!vf_dev->pf_hmc_initialized) { - ret_code = i40iw_pf_init_vfhmc(vf_dev->pf_dev, (u8)vf_dev->pmf_index, NULL); - if (ret_code) - goto add_out; - vf_dev->pf_hmc_initialized = true; - } - - add_hmc_obj = (struct i40iw_virtchnl_hmc_obj_range *)vchnl_msg->iw_chnl_buf; - - memset(&info, 0, sizeof(info)); - info.hmc_info = hmc_info; - info.is_pf = false; - info.rsrc_type = (u32)add_hmc_obj->obj_type; - info.entry_type = (info.rsrc_type == I40IW_HMC_IW_PBLE) ? I40IW_SD_TYPE_PAGED : I40IW_SD_TYPE_DIRECT; - info.start_idx = add_hmc_obj->start_index; - info.count = add_hmc_obj->obj_count; - i40iw_debug(vf_dev->pf_dev, I40IW_DEBUG_VIRT, - "I40IW_VCHNL_OP_ADD_HMC_OBJ_RANGE. Add %u type %u objects\n", - info.count, info.rsrc_type); - ret_code = i40iw_sc_create_hmc_obj(vf_dev->pf_dev, &info); - if (!ret_code) - vf_dev->hmc_info.hmc_obj[add_hmc_obj->obj_type].cnt = add_hmc_obj->obj_count; -add_out: - vf_dev->msg_count--; - vchnl_pf_send_error_resp(vf_dev->pf_dev, vf_dev->vf_id, vchnl_msg, (u16)ret_code); -} - -/** - * pf_del_hmc_obj_callback - Callback for delete HMC Object - * @work_vf_dev: pointer to the VF Device - */ -static void pf_del_hmc_obj_callback(void *work_vf_dev) -{ - struct i40iw_vfdev *vf_dev = (struct i40iw_vfdev *)work_vf_dev; - struct i40iw_hmc_info *hmc_info = &vf_dev->hmc_info; - struct i40iw_virtchnl_op_buf *vchnl_msg = &vf_dev->vf_msg_buffer.vchnl_msg; - struct i40iw_hmc_del_obj_info info; - struct i40iw_virtchnl_hmc_obj_range *del_hmc_obj; - enum i40iw_status_code ret_code = I40IW_SUCCESS; - - if (!vf_dev->pf_hmc_initialized) - goto del_out; - - del_hmc_obj = (struct i40iw_virtchnl_hmc_obj_range *)vchnl_msg->iw_chnl_buf; - - memset(&info, 0, sizeof(info)); - info.hmc_info = hmc_info; - info.is_pf = false; - info.rsrc_type = (u32)del_hmc_obj->obj_type; - info.start_idx = del_hmc_obj->start_index; - info.count = del_hmc_obj->obj_count; - i40iw_debug(vf_dev->pf_dev, I40IW_DEBUG_VIRT, - "I40IW_VCHNL_OP_DEL_HMC_OBJ_RANGE. Delete %u type %u objects\n", - info.count, info.rsrc_type); - ret_code = i40iw_sc_del_hmc_obj(vf_dev->pf_dev, &info, false); -del_out: - vf_dev->msg_count--; - vchnl_pf_send_error_resp(vf_dev->pf_dev, vf_dev->vf_id, vchnl_msg, (u16)ret_code); -} - -/** - * i40iw_vf_init_pestat - Initialize stats for VF - * @dev: pointer to the VF Device - * @stats: Statistics structure pointer - * @index: Stats index - */ -static void i40iw_vf_init_pestat(struct i40iw_sc_dev *dev, struct i40iw_vsi_pestat *stats, u16 index) -{ - stats->hw = dev->hw; - i40iw_hw_stats_init(stats, (u8)index, false); - spin_lock_init(&stats->lock); -} - -/** - * i40iw_vchnl_recv_pf - Receive PF virtual channel messages - * @dev: IWARP device pointer - * @vf_id: Virtual function ID associated with the message - * @msg: Virtual channel message buffer pointer - * @len: Length of the virtual channels message - */ -enum i40iw_status_code i40iw_vchnl_recv_pf(struct i40iw_sc_dev *dev, - u32 vf_id, - u8 *msg, - u16 len) -{ - struct i40iw_virtchnl_op_buf *vchnl_msg = (struct i40iw_virtchnl_op_buf *)msg; - struct i40iw_vfdev *vf_dev = NULL; - struct i40iw_hmc_fcn_info hmc_fcn_info; - u16 iw_vf_idx; - u16 first_avail_iw_vf = I40IW_MAX_PE_ENABLED_VF_COUNT; - struct i40iw_virt_mem vf_dev_mem; - struct i40iw_virtchnl_work_info work_info; - struct i40iw_vsi_pestat *stats; - enum i40iw_status_code ret_code; - - if (!dev || !msg || !len) - return I40IW_ERR_PARAM; - - if (!dev->vchnl_up) - return I40IW_ERR_NOT_READY; - if (vchnl_msg->iw_op_code == I40IW_VCHNL_OP_GET_VER) { - vchnl_pf_send_get_ver_resp(dev, vf_id, vchnl_msg); - return I40IW_SUCCESS; - } - for (iw_vf_idx = 0; iw_vf_idx < I40IW_MAX_PE_ENABLED_VF_COUNT; iw_vf_idx++) { - if (!dev->vf_dev[iw_vf_idx]) { - if (first_avail_iw_vf == I40IW_MAX_PE_ENABLED_VF_COUNT) - first_avail_iw_vf = iw_vf_idx; - continue; - } - if (dev->vf_dev[iw_vf_idx]->vf_id == vf_id) { - vf_dev = dev->vf_dev[iw_vf_idx]; - break; - } - } - if (vf_dev) { - if (!vf_dev->msg_count) { - vf_dev->msg_count++; - } else { - i40iw_debug(dev, I40IW_DEBUG_VIRT, - "VF%u already has a channel message in progress.\n", - vf_id); - return I40IW_SUCCESS; - } - } - switch (vchnl_msg->iw_op_code) { - case I40IW_VCHNL_OP_GET_HMC_FCN: - if (!vf_dev && - (first_avail_iw_vf != I40IW_MAX_PE_ENABLED_VF_COUNT)) { - ret_code = i40iw_allocate_virt_mem(dev->hw, &vf_dev_mem, sizeof(struct i40iw_vfdev) + - (sizeof(struct i40iw_hmc_obj_info) * I40IW_HMC_IW_MAX)); - if (!ret_code) { - vf_dev = vf_dev_mem.va; - vf_dev->stats_initialized = false; - vf_dev->pf_dev = dev; - vf_dev->msg_count = 1; - vf_dev->vf_id = vf_id; - vf_dev->iw_vf_idx = first_avail_iw_vf; - vf_dev->pf_hmc_initialized = false; - vf_dev->hmc_info.hmc_obj = (struct i40iw_hmc_obj_info *)(&vf_dev[1]); - i40iw_debug(dev, I40IW_DEBUG_VIRT, - "vf_dev %p, hmc_info %p, hmc_obj %p\n", - vf_dev, &vf_dev->hmc_info, vf_dev->hmc_info.hmc_obj); - dev->vf_dev[first_avail_iw_vf] = vf_dev; - iw_vf_idx = first_avail_iw_vf; - } else { - i40iw_debug(dev, I40IW_DEBUG_VIRT, - "VF%u Unable to allocate a VF device structure.\n", - vf_id); - vchnl_pf_send_error_resp(dev, vf_id, vchnl_msg, (u16)I40IW_ERR_NO_MEMORY); - return I40IW_SUCCESS; - } - memcpy(&vf_dev->vf_msg_buffer.vchnl_msg, vchnl_msg, len); - hmc_fcn_info.callback_fcn = pf_cqp_get_hmc_fcn_callback; - hmc_fcn_info.vf_id = vf_id; - hmc_fcn_info.iw_vf_idx = vf_dev->iw_vf_idx; - hmc_fcn_info.cqp_callback_param = vf_dev; - hmc_fcn_info.free_fcn = false; - ret_code = i40iw_cqp_manage_hmc_fcn_cmd(dev, &hmc_fcn_info); - if (ret_code) - i40iw_debug(dev, I40IW_DEBUG_VIRT, - "VF%u error CQP HMC Function operation.\n", - vf_id); - i40iw_vf_init_pestat(dev, &vf_dev->pestat, vf_dev->pmf_index); - vf_dev->stats_initialized = true; - } else { - if (vf_dev) { - vf_dev->msg_count--; - vchnl_pf_send_get_hmc_fcn_resp(dev, vf_id, vchnl_msg, vf_dev->pmf_index); - } else { - vchnl_pf_send_error_resp(dev, vf_id, vchnl_msg, - (u16)I40IW_ERR_NO_MEMORY); - } - } - break; - case I40IW_VCHNL_OP_ADD_HMC_OBJ_RANGE: - if (!vf_dev) - return I40IW_ERR_BAD_PTR; - work_info.worker_vf_dev = vf_dev; - work_info.callback_fcn = pf_add_hmc_obj_callback; - memcpy(&vf_dev->vf_msg_buffer.vchnl_msg, vchnl_msg, len); - i40iw_cqp_spawn_worker(dev, &work_info, vf_dev->iw_vf_idx); - break; - case I40IW_VCHNL_OP_DEL_HMC_OBJ_RANGE: - if (!vf_dev) - return I40IW_ERR_BAD_PTR; - work_info.worker_vf_dev = vf_dev; - work_info.callback_fcn = pf_del_hmc_obj_callback; - memcpy(&vf_dev->vf_msg_buffer.vchnl_msg, vchnl_msg, len); - i40iw_cqp_spawn_worker(dev, &work_info, vf_dev->iw_vf_idx); - break; - case I40IW_VCHNL_OP_GET_STATS: - if (!vf_dev) - return I40IW_ERR_BAD_PTR; - stats = &vf_dev->pestat; - i40iw_hw_stats_read_all(stats, &stats->hw_stats); - vf_dev->msg_count--; - vchnl_pf_send_get_pe_stats_resp(dev, vf_id, vchnl_msg, &stats->hw_stats); - break; - default: - i40iw_debug(dev, I40IW_DEBUG_VIRT, - "40iw_vchnl_recv_pf: Invalid OpCode 0x%x\n", - vchnl_msg->iw_op_code); - vchnl_pf_send_error_resp(dev, vf_id, - vchnl_msg, (u16)I40IW_ERR_NOT_IMPLEMENTED); - } - return I40IW_SUCCESS; -} - -/** - * i40iw_vchnl_recv_vf - Receive VF virtual channel messages - * @dev: IWARP device pointer - * @vf_id: Virtual function ID associated with the message - * @msg: Virtual channel message buffer pointer - * @len: Length of the virtual channels message - */ -enum i40iw_status_code i40iw_vchnl_recv_vf(struct i40iw_sc_dev *dev, - u32 vf_id, - u8 *msg, - u16 len) -{ - struct i40iw_virtchnl_resp_buf *vchnl_msg_resp = (struct i40iw_virtchnl_resp_buf *)msg; - struct i40iw_virtchnl_req *vchnl_req; - - vchnl_req = (struct i40iw_virtchnl_req *)(uintptr_t)vchnl_msg_resp->iw_chnl_op_ctx; - vchnl_req->ret_code = (enum i40iw_status_code)vchnl_msg_resp->iw_op_ret_code; - if (len == (sizeof(*vchnl_msg_resp) + vchnl_req->parm_len - 1)) { - if (vchnl_req->parm_len && vchnl_req->parm) - memcpy(vchnl_req->parm, vchnl_msg_resp->iw_chnl_buf, vchnl_req->parm_len); - i40iw_debug(dev, I40IW_DEBUG_VIRT, - "%s: Got response, data size %u\n", __func__, - vchnl_req->parm_len); - } else { - i40iw_debug(dev, I40IW_DEBUG_VIRT, - "%s: error length on response, Got %u, expected %u\n", __func__, - len, (u32)(sizeof(*vchnl_msg_resp) + vchnl_req->parm_len - 1)); - } - - return I40IW_SUCCESS; -} - -/** - * i40iw_vchnl_vf_get_ver - Request Channel version - * @dev: IWARP device pointer - * @vchnl_ver: Virtual channel message version pointer - */ -enum i40iw_status_code i40iw_vchnl_vf_get_ver(struct i40iw_sc_dev *dev, - u32 *vchnl_ver) -{ - struct i40iw_virtchnl_req vchnl_req; - enum i40iw_status_code ret_code; - - if (!i40iw_vf_clear_to_send(dev)) - return I40IW_ERR_TIMEOUT; - memset(&vchnl_req, 0, sizeof(vchnl_req)); - vchnl_req.dev = dev; - vchnl_req.parm = vchnl_ver; - vchnl_req.parm_len = sizeof(*vchnl_ver); - vchnl_req.vchnl_msg = &dev->vchnl_vf_msg_buf.vchnl_msg; - - ret_code = vchnl_vf_send_get_ver_req(dev, &vchnl_req); - if (ret_code) { - i40iw_debug(dev, I40IW_DEBUG_VIRT, - "%s Send message failed 0x%0x\n", __func__, ret_code); - return ret_code; - } - ret_code = i40iw_vf_wait_vchnl_resp(dev); - if (ret_code) - return ret_code; - else - return vchnl_req.ret_code; -} - -/** - * i40iw_vchnl_vf_get_hmc_fcn - Request HMC Function - * @dev: IWARP device pointer - * @hmc_fcn: HMC function index pointer - */ -enum i40iw_status_code i40iw_vchnl_vf_get_hmc_fcn(struct i40iw_sc_dev *dev, - u16 *hmc_fcn) -{ - struct i40iw_virtchnl_req vchnl_req; - enum i40iw_status_code ret_code; - - if (!i40iw_vf_clear_to_send(dev)) - return I40IW_ERR_TIMEOUT; - memset(&vchnl_req, 0, sizeof(vchnl_req)); - vchnl_req.dev = dev; - vchnl_req.parm = hmc_fcn; - vchnl_req.parm_len = sizeof(*hmc_fcn); - vchnl_req.vchnl_msg = &dev->vchnl_vf_msg_buf.vchnl_msg; - - ret_code = vchnl_vf_send_get_hmc_fcn_req(dev, &vchnl_req); - if (ret_code) { - i40iw_debug(dev, I40IW_DEBUG_VIRT, - "%s Send message failed 0x%0x\n", __func__, ret_code); - return ret_code; - } - ret_code = i40iw_vf_wait_vchnl_resp(dev); - if (ret_code) - return ret_code; - else - return vchnl_req.ret_code; -} - -/** - * i40iw_vchnl_vf_add_hmc_objs - Add HMC Object - * @dev: IWARP device pointer - * @rsrc_type: HMC Resource type - * @start_index: Starting index of the objects to be added - * @rsrc_count: Number of resources to be added - */ -enum i40iw_status_code i40iw_vchnl_vf_add_hmc_objs(struct i40iw_sc_dev *dev, - enum i40iw_hmc_rsrc_type rsrc_type, - u32 start_index, - u32 rsrc_count) -{ - struct i40iw_virtchnl_req vchnl_req; - enum i40iw_status_code ret_code; - - if (!i40iw_vf_clear_to_send(dev)) - return I40IW_ERR_TIMEOUT; - memset(&vchnl_req, 0, sizeof(vchnl_req)); - vchnl_req.dev = dev; - vchnl_req.vchnl_msg = &dev->vchnl_vf_msg_buf.vchnl_msg; - - ret_code = vchnl_vf_send_add_hmc_objs_req(dev, - &vchnl_req, - rsrc_type, - start_index, - rsrc_count); - if (ret_code) { - i40iw_debug(dev, I40IW_DEBUG_VIRT, - "%s Send message failed 0x%0x\n", __func__, ret_code); - return ret_code; - } - ret_code = i40iw_vf_wait_vchnl_resp(dev); - if (ret_code) - return ret_code; - else - return vchnl_req.ret_code; -} - -/** - * i40iw_vchnl_vf_del_hmc_obj - del HMC obj - * @dev: IWARP device pointer - * @rsrc_type: HMC Resource type - * @start_index: Starting index of the object to delete - * @rsrc_count: Number of resources to be delete - */ -enum i40iw_status_code i40iw_vchnl_vf_del_hmc_obj(struct i40iw_sc_dev *dev, - enum i40iw_hmc_rsrc_type rsrc_type, - u32 start_index, - u32 rsrc_count) -{ - struct i40iw_virtchnl_req vchnl_req; - enum i40iw_status_code ret_code; - - if (!i40iw_vf_clear_to_send(dev)) - return I40IW_ERR_TIMEOUT; - memset(&vchnl_req, 0, sizeof(vchnl_req)); - vchnl_req.dev = dev; - vchnl_req.vchnl_msg = &dev->vchnl_vf_msg_buf.vchnl_msg; - - ret_code = vchnl_vf_send_del_hmc_objs_req(dev, - &vchnl_req, - rsrc_type, - start_index, - rsrc_count); - if (ret_code) { - i40iw_debug(dev, I40IW_DEBUG_VIRT, - "%s Send message failed 0x%0x\n", __func__, ret_code); - return ret_code; - } - ret_code = i40iw_vf_wait_vchnl_resp(dev); - if (ret_code) - return ret_code; - else - return vchnl_req.ret_code; -} - -/** - * i40iw_vchnl_vf_get_pe_stats - Get PE stats - * @dev: IWARP device pointer - * @hw_stats: HW stats struct - */ -enum i40iw_status_code i40iw_vchnl_vf_get_pe_stats(struct i40iw_sc_dev *dev, - struct i40iw_dev_hw_stats *hw_stats) -{ - struct i40iw_virtchnl_req vchnl_req; - enum i40iw_status_code ret_code; - - if (!i40iw_vf_clear_to_send(dev)) - return I40IW_ERR_TIMEOUT; - memset(&vchnl_req, 0, sizeof(vchnl_req)); - vchnl_req.dev = dev; - vchnl_req.parm = hw_stats; - vchnl_req.parm_len = sizeof(*hw_stats); - vchnl_req.vchnl_msg = &dev->vchnl_vf_msg_buf.vchnl_msg; - - ret_code = vchnl_vf_send_get_pe_stats_req(dev, &vchnl_req); - if (ret_code) { - i40iw_debug(dev, I40IW_DEBUG_VIRT, - "%s Send message failed 0x%0x\n", __func__, ret_code); - return ret_code; - } - ret_code = i40iw_vf_wait_vchnl_resp(dev); - if (ret_code) - return ret_code; - else - return vchnl_req.ret_code; -} diff --git a/drivers/infiniband/hw/i40iw/i40iw_virtchnl.h b/drivers/infiniband/hw/i40iw/i40iw_virtchnl.h deleted file mode 100644 index 24886ef08293..000000000000 --- a/drivers/infiniband/hw/i40iw/i40iw_virtchnl.h +++ /dev/null @@ -1,124 +0,0 @@ -/******************************************************************************* -* -* Copyright (c) 2015-2016 Intel Corporation. All rights reserved. -* -* This software is available to you under a choice of one of two -* licenses. You may choose to be licensed under the terms of the GNU -* General Public License (GPL) Version 2, available from the file -* COPYING in the main directory of this source tree, or the -* OpenFabrics.org BSD license below: -* -* Redistribution and use in source and binary forms, with or -* without modification, are permitted provided that the following -* conditions are met: -* -* - Redistributions of source code must retain the above -* copyright notice, this list of conditions and the following -* disclaimer. -* -* - Redistributions in binary form must reproduce the above -* copyright notice, this list of conditions and the following -* disclaimer in the documentation and/or other materials -* provided with the distribution. -* -* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS -* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN -* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -* SOFTWARE. -* -*******************************************************************************/ - -#ifndef I40IW_VIRTCHNL_H -#define I40IW_VIRTCHNL_H - -#include "i40iw_hmc.h" - -#pragma pack(push, 1) - -struct i40iw_virtchnl_op_buf { - u16 iw_op_code; - u16 iw_op_ver; - u16 iw_chnl_buf_len; - u16 rsvd; - u64 iw_chnl_op_ctx; - /* Member alignment MUST be maintained above this location */ - u8 iw_chnl_buf[1]; -}; - -struct i40iw_virtchnl_resp_buf { - u64 iw_chnl_op_ctx; - u16 iw_chnl_buf_len; - s16 iw_op_ret_code; - /* Member alignment MUST be maintained above this location */ - u16 rsvd[2]; - u8 iw_chnl_buf[1]; -}; - -enum i40iw_virtchnl_ops { - I40IW_VCHNL_OP_GET_VER = 0, - I40IW_VCHNL_OP_GET_HMC_FCN, - I40IW_VCHNL_OP_ADD_HMC_OBJ_RANGE, - I40IW_VCHNL_OP_DEL_HMC_OBJ_RANGE, - I40IW_VCHNL_OP_GET_STATS -}; - -#define I40IW_VCHNL_OP_GET_VER_V0 0 -#define I40IW_VCHNL_OP_GET_HMC_FCN_V0 0 -#define I40IW_VCHNL_OP_ADD_HMC_OBJ_RANGE_V0 0 -#define I40IW_VCHNL_OP_DEL_HMC_OBJ_RANGE_V0 0 -#define I40IW_VCHNL_OP_GET_STATS_V0 0 -#define I40IW_VCHNL_CHNL_VER_V0 0 - -struct i40iw_dev_hw_stats; - -struct i40iw_virtchnl_hmc_obj_range { - u16 obj_type; - u16 rsvd; - u32 start_index; - u32 obj_count; -}; - -enum i40iw_status_code i40iw_vchnl_recv_pf(struct i40iw_sc_dev *dev, - u32 vf_id, - u8 *msg, - u16 len); - -enum i40iw_status_code i40iw_vchnl_recv_vf(struct i40iw_sc_dev *dev, - u32 vf_id, - u8 *msg, - u16 len); - -struct i40iw_virtchnl_req { - struct i40iw_sc_dev *dev; - struct i40iw_virtchnl_op_buf *vchnl_msg; - void *parm; - u32 vf_id; - u16 parm_len; - s16 ret_code; -}; - -#pragma pack(pop) - -enum i40iw_status_code i40iw_vchnl_vf_get_ver(struct i40iw_sc_dev *dev, - u32 *vchnl_ver); - -enum i40iw_status_code i40iw_vchnl_vf_get_hmc_fcn(struct i40iw_sc_dev *dev, - u16 *hmc_fcn); - -enum i40iw_status_code i40iw_vchnl_vf_add_hmc_objs(struct i40iw_sc_dev *dev, - enum i40iw_hmc_rsrc_type rsrc_type, - u32 start_index, - u32 rsrc_count); - -enum i40iw_status_code i40iw_vchnl_vf_del_hmc_obj(struct i40iw_sc_dev *dev, - enum i40iw_hmc_rsrc_type rsrc_type, - u32 start_index, - u32 rsrc_count); - -enum i40iw_status_code i40iw_vchnl_vf_get_pe_stats(struct i40iw_sc_dev *dev, - struct i40iw_dev_hw_stats *hw_stats); -#endif diff --git a/drivers/infiniband/hw/irdma/Kconfig b/drivers/infiniband/hw/irdma/Kconfig new file mode 100644 index 000000000000..dab88286d549 --- /dev/null +++ b/drivers/infiniband/hw/irdma/Kconfig @@ -0,0 +1,12 @@ +# SPDX-License-Identifier: GPL-2.0-only +config INFINIBAND_IRDMA + tristate "Intel(R) Ethernet Protocol Driver for RDMA" + depends on INET + depends on IPV6 || !IPV6 + depends on PCI + depends on ICE && I40E + select GENERIC_ALLOCATOR + select CONFIG_AUXILIARY_BUS + help + This is an Intel(R) Ethernet Protocol Driver for RDMA driver + that support E810 (iWARP/RoCE) and X722 (iWARP) network devices. diff --git a/drivers/infiniband/hw/irdma/Makefile b/drivers/infiniband/hw/irdma/Makefile new file mode 100644 index 000000000000..48c3854235a0 --- /dev/null +++ b/drivers/infiniband/hw/irdma/Makefile @@ -0,0 +1,27 @@ +# SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB +# Copyright (c) 2019, Intel Corporation. + +# +# Makefile for the Intel(R) Ethernet Connection RDMA Linux Driver +# + +obj-$(CONFIG_INFINIBAND_IRDMA) += irdma.o + +irdma-objs := cm.o \ + ctrl.o \ + hmc.o \ + hw.o \ + i40iw_hw.o \ + i40iw_if.o \ + icrdma_hw.o \ + main.o \ + pble.o \ + puda.o \ + trace.o \ + uda.o \ + uk.o \ + utils.o \ + verbs.o \ + ws.o \ + +CFLAGS_trace.o = -I$(src) diff --git a/drivers/net/ethernet/intel/i40e/i40e_client.c b/drivers/net/ethernet/intel/i40e/i40e_client.c index e07ed065d3a4..ea2bb0140a6e 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_client.c +++ b/drivers/net/ethernet/intel/i40e/i40e_client.c @@ -8,8 +8,6 @@ #include "i40e.h" #include "i40e_prototype.h" -static const char i40e_client_interface_version_str[] = I40E_CLIENT_VERSION_STR; -static struct i40e_client *registered_client; static LIST_HEAD(i40e_devices); static DEFINE_MUTEX(i40e_device_mutex); DEFINE_IDA(i40e_client_ida); @@ -367,7 +365,6 @@ static void i40e_client_add_instance(struct i40e_pf *pf) else dev_err(&pf->pdev->dev, "MAC address list is empty!\n"); - cdev->client = registered_client; pf->cinst = cdev; cdev->lan_info.msix_count = pf->num_iwarp_msix; @@ -525,69 +522,6 @@ int i40e_lan_del_device(struct i40e_pf *pf) return ret; } -/** - * i40e_client_release - release client specific resources - * @client: pointer to the registered client - * - **/ -static void i40e_client_release(struct i40e_client *client) -{ - struct i40e_client_instance *cdev; - struct i40e_device *ldev; - struct i40e_pf *pf; - - mutex_lock(&i40e_device_mutex); - list_for_each_entry(ldev, &i40e_devices, list) { - pf = ldev->pf; - cdev = pf->cinst; - if (!cdev) - continue; - - while (test_and_set_bit(__I40E_SERVICE_SCHED, - pf->state)) - usleep_range(500, 1000); - - if (test_bit(__I40E_CLIENT_INSTANCE_OPENED, &cdev->state)) { - if (client->ops && client->ops->close) - client->ops->close(&cdev->lan_info, client, - false); - i40e_client_release_qvlist(&cdev->lan_info); - clear_bit(__I40E_CLIENT_INSTANCE_OPENED, &cdev->state); - - dev_warn(&pf->pdev->dev, - "Client %s instance for PF id %d closed\n", - client->name, pf->hw.pf_id); - } - /* delete the client instance */ - i40e_client_del_instance(pf); - dev_info(&pf->pdev->dev, "Deleted client instance of Client %s\n", - client->name); - clear_bit(__I40E_SERVICE_SCHED, pf->state); - } - mutex_unlock(&i40e_device_mutex); -} - -/** - * i40e_client_prepare - prepare client specific resources - * @client: pointer to the registered client - * - **/ -static void i40e_client_prepare(struct i40e_client *client) -{ - struct i40e_device *ldev; - struct i40e_pf *pf; - - mutex_lock(&i40e_device_mutex); - list_for_each_entry(ldev, &i40e_devices, list) { - pf = ldev->pf; - i40e_client_add_instance(pf); - /* Start the client subtask */ - set_bit(__I40E_CLIENT_SERVICE_REQUESTED, pf->state); - i40e_service_event_schedule(pf); - } - mutex_unlock(&i40e_device_mutex); -} - /** * i40e_client_virtchnl_send - TBD * @ldev: pointer to L2 context @@ -817,86 +751,3 @@ void i40e_client_device_unregister(struct i40e_info *ldev) clear_bit(__I40E_SERVICE_SCHED, pf->state); } EXPORT_SYMBOL_GPL(i40e_client_device_unregister); - -/* Retain these legacy global registration/unregistration calls till i40iw is - * removed from the kernel. The irdma unified driver does not use these - * exported symbols. - */ -/** - * i40e_register_client - Register a i40e client driver with the L2 driver - * @client: pointer to the i40e_client struct - * - * Returns 0 on success or non-0 on error - **/ -int i40e_register_client(struct i40e_client *client) -{ - int ret = 0; - - if (!client) { - ret = -EIO; - goto out; - } - - if (strlen(client->name) == 0) { - pr_info("i40e: Failed to register client with no name\n"); - ret = -EIO; - goto out; - } - - if (registered_client) { - pr_info("i40e: Client %s has already been registered!\n", - client->name); - ret = -EEXIST; - goto out; - } - - if ((client->version.major != I40E_CLIENT_VERSION_MAJOR) || - (client->version.minor != I40E_CLIENT_VERSION_MINOR)) { - pr_info("i40e: Failed to register client %s due to mismatched client interface version\n", - client->name); - pr_info("Client is using version: %02d.%02d.%02d while LAN driver supports %s\n", - client->version.major, client->version.minor, - client->version.build, - i40e_client_interface_version_str); - ret = -EIO; - goto out; - } - - registered_client = client; - - i40e_client_prepare(client); - - pr_info("i40e: Registered client %s\n", client->name); -out: - return ret; -} -EXPORT_SYMBOL(i40e_register_client); - -/** - * i40e_unregister_client - Unregister a i40e client driver with the L2 driver - * @client: pointer to the i40e_client struct - * - * Returns 0 on success or non-0 on error - **/ -int i40e_unregister_client(struct i40e_client *client) -{ - int ret = 0; - - if (registered_client != client) { - pr_info("i40e: Client %s has not been registered\n", - client->name); - ret = -ENODEV; - goto out; - } - registered_client = NULL; - /* When a unregister request comes through we would have to send - * a close for each of the client instances that were opened. - * client_release function is called to handle this. - */ - i40e_client_release(client); - - pr_info("i40e: Unregistered client %s\n", client->name); -out: - return ret; -} -EXPORT_SYMBOL(i40e_unregister_client); diff --git a/include/linux/net/intel/i40e_client.h b/include/linux/net/intel/i40e_client.h index 41f24b5241ab..6b3267b49755 100644 --- a/include/linux/net/intel/i40e_client.h +++ b/include/linux/net/intel/i40e_client.h @@ -197,8 +197,5 @@ static inline bool i40e_client_is_registered(struct i40e_client *client) void i40e_client_device_register(struct i40e_info *ldev, struct i40e_client *client); void i40e_client_device_unregister(struct i40e_info *ldev); -/* used by clients */ -int i40e_register_client(struct i40e_client *client); -int i40e_unregister_client(struct i40e_client *client); #endif /* _I40E_CLIENT_H_ */ diff --git a/include/uapi/rdma/i40iw-abi.h b/include/uapi/rdma/i40iw-abi.h deleted file mode 100644 index 79890baa6fdb..000000000000 --- a/include/uapi/rdma/i40iw-abi.h +++ /dev/null @@ -1,107 +0,0 @@ -/* - * Copyright (c) 2006 - 2016 Intel Corporation. All rights reserved. - * Copyright (c) 2005 Topspin Communications. All rights reserved. - * Copyright (c) 2005 Cisco Systems. All rights reserved. - * Copyright (c) 2005 Open Grid Computing, Inc. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - */ - -#ifndef I40IW_ABI_H -#define I40IW_ABI_H - -#include - -#define I40IW_ABI_VER 5 - -struct i40iw_alloc_ucontext_req { - __u32 reserved32; - __u8 userspace_ver; - __u8 reserved8[3]; -}; - -struct i40iw_alloc_ucontext_resp { - __u32 max_pds; /* maximum pds allowed for this user process */ - __u32 max_qps; /* maximum qps allowed for this user process */ - __u32 wq_size; /* size of the WQs (sq+rq) allocated to the mmaped area */ - __u8 kernel_ver; - __u8 reserved[3]; -}; - -struct i40iw_alloc_pd_resp { - __u32 pd_id; - __u8 reserved[4]; -}; - -struct i40iw_create_cq_req { - __aligned_u64 user_cq_buffer; - __aligned_u64 user_shadow_area; -}; - -struct i40iw_create_qp_req { - __aligned_u64 user_wqe_buffers; - __aligned_u64 user_compl_ctx; - - /* UDA QP PHB */ - __aligned_u64 user_sq_phb; /* place for VA of the sq phb buff */ - __aligned_u64 user_rq_phb; /* place for VA of the rq phb buff */ -}; - -enum i40iw_memreg_type { - IW_MEMREG_TYPE_MEM = 0x0000, - IW_MEMREG_TYPE_QP = 0x0001, - IW_MEMREG_TYPE_CQ = 0x0002, -}; - -struct i40iw_mem_reg_req { - __u16 reg_type; /* Memory, QP or CQ */ - __u16 cq_pages; - __u16 rq_pages; - __u16 sq_pages; -}; - -struct i40iw_create_cq_resp { - __u32 cq_id; - __u32 cq_size; - __u32 mmap_db_index; - __u32 reserved; -}; - -struct i40iw_create_qp_resp { - __u32 qp_id; - __u32 actual_sq_size; - __u32 actual_rq_size; - __u32 i40iw_drv_opt; - __u16 push_idx; - __u8 lsmm; - __u8 rsvd2; -}; - -#endif -- cgit v1.2.3 From a83d958504734f78f42b1e3392d93816297e790a Mon Sep 17 00:00:00 2001 From: Marcel Holtmann Date: Thu, 3 Jun 2021 21:20:26 +0200 Subject: Bluetooth: Fix VIRTIO_ID_BT assigned number It turned out that the VIRTIO_ID_* are not assigned in the virtio_ids.h file in the upstream kernel. Picking the next free one was wrong and there is a process that has been followed now. See https://github.com/oasis-tcs/virtio-spec/issues/108 for details. Fixes: afd2daa26c7a ("Bluetooth: Add support for virtio transport driver") Signed-off-by: Marcel Holtmann Signed-off-by: Luiz Augusto von Dentz --- include/uapi/linux/virtio_ids.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/uapi') diff --git a/include/uapi/linux/virtio_ids.h b/include/uapi/linux/virtio_ids.h index f0c35ce8628c..4fe842c3a3a9 100644 --- a/include/uapi/linux/virtio_ids.h +++ b/include/uapi/linux/virtio_ids.h @@ -54,7 +54,7 @@ #define VIRTIO_ID_SOUND 25 /* virtio sound */ #define VIRTIO_ID_FS 26 /* virtio filesystem */ #define VIRTIO_ID_PMEM 27 /* virtio pmem */ -#define VIRTIO_ID_BT 28 /* virtio bluetooth */ #define VIRTIO_ID_MAC80211_HWSIM 29 /* virtio mac80211-hwsim */ +#define VIRTIO_ID_BT 40 /* virtio bluetooth */ #endif /* _LINUX_VIRTIO_IDS_H */ -- cgit v1.2.3 From e32ea44c7ae476f4c90e35ab0a29dc8ff082bc11 Mon Sep 17 00:00:00 2001 From: Andreas Roeseler Date: Thu, 3 Jun 2021 16:22:11 -0500 Subject: icmp: fix lib conflict with trinity Including and in the dependencies breaks compilation of trinity due to multiple definitions. is only used in to provide the definition of the struct in_addr, but this can be substituted out by using the datatype __be32. Signed-off-by: Andreas Roeseler Signed-off-by: David S. Miller --- include/uapi/linux/icmp.h | 3 +-- net/ipv4/icmp.c | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) (limited to 'include/uapi') diff --git a/include/uapi/linux/icmp.h b/include/uapi/linux/icmp.h index c1da8244c5e1..163c0998aec9 100644 --- a/include/uapi/linux/icmp.h +++ b/include/uapi/linux/icmp.h @@ -20,7 +20,6 @@ #include #include -#include #include #include @@ -154,7 +153,7 @@ struct icmp_ext_echo_iio { struct { struct icmp_ext_echo_ctype3_hdr ctype3_hdr; union { - struct in_addr ipv4_addr; + __be32 ipv4_addr; struct in6_addr ipv6_addr; } ip_addr; } addr; diff --git a/net/ipv4/icmp.c b/net/ipv4/icmp.c index 7b6931a4d775..2e09d62d59e3 100644 --- a/net/ipv4/icmp.c +++ b/net/ipv4/icmp.c @@ -1059,7 +1059,7 @@ static bool icmp_echo(struct sk_buff *skb) if (ident_len != sizeof(iio->ident.addr.ctype3_hdr) + sizeof(struct in_addr)) goto send_mal_query; - dev = ip_dev_find(net, iio->ident.addr.ip_addr.ipv4_addr.s_addr); + dev = ip_dev_find(net, iio->ident.addr.ip_addr.ipv4_addr); break; #if IS_ENABLED(CONFIG_IPV6) case ICMP_AFI_IP6: -- cgit v1.2.3 From 819fbd3d8ef36c09576c2a0ffea503f5c46e9177 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Tue, 1 Jun 2021 11:31:30 +0200 Subject: media: dvb header files: move some headers to staging The audio, video and OSD APIs are used upstream only by the av7110 driver, which was moved to staging. So, move the corresponding header files to it. Signed-off-by: Mauro Carvalho Chehab --- drivers/staging/media/av7110/audio.h | 101 ++++++++++++++++ drivers/staging/media/av7110/av7110.h | 7 +- drivers/staging/media/av7110/osd.h | 181 ++++++++++++++++++++++++++++ drivers/staging/media/av7110/video.h | 220 ++++++++++++++++++++++++++++++++++ include/uapi/linux/dvb/audio.h | 101 ---------------- include/uapi/linux/dvb/osd.h | 181 ---------------------------- include/uapi/linux/dvb/video.h | 220 ---------------------------------- 7 files changed, 506 insertions(+), 505 deletions(-) create mode 100644 drivers/staging/media/av7110/audio.h create mode 100644 drivers/staging/media/av7110/osd.h create mode 100644 drivers/staging/media/av7110/video.h delete mode 100644 include/uapi/linux/dvb/audio.h delete mode 100644 include/uapi/linux/dvb/osd.h delete mode 100644 include/uapi/linux/dvb/video.h (limited to 'include/uapi') diff --git a/drivers/staging/media/av7110/audio.h b/drivers/staging/media/av7110/audio.h new file mode 100644 index 000000000000..2f869da69171 --- /dev/null +++ b/drivers/staging/media/av7110/audio.h @@ -0,0 +1,101 @@ +/* SPDX-License-Identifier: LGPL-2.1+ WITH Linux-syscall-note */ +/* + * audio.h - DEPRECATED MPEG-TS audio decoder API + * + * NOTE: should not be used on future drivers + * + * Copyright (C) 2000 Ralph Metzler + * & Marcus Metzler + * for convergence integrated media GmbH + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Lesser Public License + * as published by the Free Software Foundation; either version 2.1 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + * + */ + +#ifndef _DVBAUDIO_H_ +#define _DVBAUDIO_H_ + +#include + +typedef enum { + AUDIO_SOURCE_DEMUX, /* Select the demux as the main source */ + AUDIO_SOURCE_MEMORY /* Select internal memory as the main source */ +} audio_stream_source_t; + + +typedef enum { + AUDIO_STOPPED, /* Device is stopped */ + AUDIO_PLAYING, /* Device is currently playing */ + AUDIO_PAUSED /* Device is paused */ +} audio_play_state_t; + + +typedef enum { + AUDIO_STEREO, + AUDIO_MONO_LEFT, + AUDIO_MONO_RIGHT, + AUDIO_MONO, + AUDIO_STEREO_SWAPPED +} audio_channel_select_t; + + +typedef struct audio_mixer { + unsigned int volume_left; + unsigned int volume_right; + /* what else do we need? bass, pass-through, ... */ +} audio_mixer_t; + + +typedef struct audio_status { + int AV_sync_state; /* sync audio and video? */ + int mute_state; /* audio is muted */ + audio_play_state_t play_state; /* current playback state */ + audio_stream_source_t stream_source; /* current stream source */ + audio_channel_select_t channel_select; /* currently selected channel */ + int bypass_mode; /* pass on audio data to */ + audio_mixer_t mixer_state; /* current mixer state */ +} audio_status_t; /* separate decoder hardware */ + + +/* for GET_CAPABILITIES and SET_FORMAT, the latter should only set one bit */ +#define AUDIO_CAP_DTS 1 +#define AUDIO_CAP_LPCM 2 +#define AUDIO_CAP_MP1 4 +#define AUDIO_CAP_MP2 8 +#define AUDIO_CAP_MP3 16 +#define AUDIO_CAP_AAC 32 +#define AUDIO_CAP_OGG 64 +#define AUDIO_CAP_SDDS 128 +#define AUDIO_CAP_AC3 256 + +#define AUDIO_STOP _IO('o', 1) +#define AUDIO_PLAY _IO('o', 2) +#define AUDIO_PAUSE _IO('o', 3) +#define AUDIO_CONTINUE _IO('o', 4) +#define AUDIO_SELECT_SOURCE _IO('o', 5) +#define AUDIO_SET_MUTE _IO('o', 6) +#define AUDIO_SET_AV_SYNC _IO('o', 7) +#define AUDIO_SET_BYPASS_MODE _IO('o', 8) +#define AUDIO_CHANNEL_SELECT _IO('o', 9) +#define AUDIO_GET_STATUS _IOR('o', 10, audio_status_t) + +#define AUDIO_GET_CAPABILITIES _IOR('o', 11, unsigned int) +#define AUDIO_CLEAR_BUFFER _IO('o', 12) +#define AUDIO_SET_ID _IO('o', 13) +#define AUDIO_SET_MIXER _IOW('o', 14, audio_mixer_t) +#define AUDIO_SET_STREAMTYPE _IO('o', 15) +#define AUDIO_BILINGUAL_CHANNEL_SELECT _IO('o', 20) + +#endif /* _DVBAUDIO_H_ */ diff --git a/drivers/staging/media/av7110/av7110.h b/drivers/staging/media/av7110/av7110.h index 809d938ae166..b8e8fc8ddbe9 100644 --- a/drivers/staging/media/av7110/av7110.h +++ b/drivers/staging/media/av7110/av7110.h @@ -9,11 +9,12 @@ #include #include -#include -#include +#include "video.h" +#include "audio.h" +#include "osd.h" + #include #include -#include #include #include diff --git a/drivers/staging/media/av7110/osd.h b/drivers/staging/media/av7110/osd.h new file mode 100644 index 000000000000..858997c74043 --- /dev/null +++ b/drivers/staging/media/av7110/osd.h @@ -0,0 +1,181 @@ +/* SPDX-License-Identifier: LGPL-2.1+ WITH Linux-syscall-note */ +/* + * osd.h - DEPRECATED On Screen Display API + * + * NOTE: should not be used on future drivers + * + * Copyright (C) 2001 Ralph Metzler + * & Marcus Metzler + * for convergence integrated media GmbH + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Lesser Public License + * as published by the Free Software Foundation; either version 2.1 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + * + */ + +#ifndef _DVBOSD_H_ +#define _DVBOSD_H_ + +#include + +typedef enum { + /* All functions return -2 on "not open" */ + OSD_Close = 1, /* () */ + /* + * Disables OSD and releases the buffers + * returns 0 on success + */ + OSD_Open, /* (x0,y0,x1,y1,BitPerPixel[2/4/8](color&0x0F),mix[0..15](color&0xF0)) */ + /* + * Opens OSD with this size and bit depth + * returns 0 on success, -1 on DRAM allocation error, -2 on "already open" + */ + OSD_Show, /* () */ + /* + * enables OSD mode + * returns 0 on success + */ + OSD_Hide, /* () */ + /* + * disables OSD mode + * returns 0 on success + */ + OSD_Clear, /* () */ + /* + * Sets all pixel to color 0 + * returns 0 on success + */ + OSD_Fill, /* (color) */ + /* + * Sets all pixel to color + * returns 0 on success + */ + OSD_SetColor, /* (color,R{x0},G{y0},B{x1},opacity{y1}) */ + /* + * set palette entry to , and apply + * R,G,B: 0..255 + * R=Red, G=Green, B=Blue + * opacity=0: pixel opacity 0% (only video pixel shows) + * opacity=1..254: pixel opacity as specified in header + * opacity=255: pixel opacity 100% (only OSD pixel shows) + * returns 0 on success, -1 on error + */ + OSD_SetPalette, /* (firstcolor{color},lastcolor{x0},data) */ + /* + * Set a number of entries in the palette + * sets the entries "firstcolor" through "lastcolor" from the array "data" + * data has 4 byte for each color: + * R,G,B, and a opacity value: 0->transparent, 1..254->mix, 255->pixel + */ + OSD_SetTrans, /* (transparency{color}) */ + /* + * Sets transparency of mixed pixel (0..15) + * returns 0 on success + */ + OSD_SetPixel, /* (x0,y0,color) */ + /* + * sets pixel , to color number + * returns 0 on success, -1 on error + */ + OSD_GetPixel, /* (x0,y0) */ + /* returns color number of pixel ,, or -1 */ + OSD_SetRow, /* (x0,y0,x1,data) */ + /* + * fills pixels x0,y through x1,y with the content of data[] + * returns 0 on success, -1 on clipping all pixel (no pixel drawn) + */ + OSD_SetBlock, /* (x0,y0,x1,y1,increment{color},data) */ + /* + * fills pixels x0,y0 through x1,y1 with the content of data[] + * inc contains the width of one line in the data block, + * inc<=0 uses blockwidth as linewidth + * returns 0 on success, -1 on clipping all pixel + */ + OSD_FillRow, /* (x0,y0,x1,color) */ + /* + * fills pixels x0,y through x1,y with the color + * returns 0 on success, -1 on clipping all pixel + */ + OSD_FillBlock, /* (x0,y0,x1,y1,color) */ + /* + * fills pixels x0,y0 through x1,y1 with the color + * returns 0 on success, -1 on clipping all pixel + */ + OSD_Line, /* (x0,y0,x1,y1,color) */ + /* + * draw a line from x0,y0 to x1,y1 with the color + * returns 0 on success + */ + OSD_Query, /* (x0,y0,x1,y1,xasp{color}}), yasp=11 */ + /* + * fills parameters with the picture dimensions and the pixel aspect ratio + * returns 0 on success + */ + OSD_Test, /* () */ + /* + * draws a test picture. for debugging purposes only + * returns 0 on success + * TODO: remove "test" in final version + */ + OSD_Text, /* (x0,y0,size,color,text) */ + OSD_SetWindow, /* (x0) set window with number 0 + * & Ralph Metzler + * for convergence integrated media GmbH + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public License + * as published by the Free Software Foundation; either version 2.1 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + * + */ + +#ifndef _UAPI_DVBVIDEO_H_ +#define _UAPI_DVBVIDEO_H_ + +#include +#ifndef __KERNEL__ +#include +#endif + +typedef enum { + VIDEO_FORMAT_4_3, /* Select 4:3 format */ + VIDEO_FORMAT_16_9, /* Select 16:9 format. */ + VIDEO_FORMAT_221_1 /* 2.21:1 */ +} video_format_t; + + +typedef enum { + VIDEO_PAN_SCAN, /* use pan and scan format */ + VIDEO_LETTER_BOX, /* use letterbox format */ + VIDEO_CENTER_CUT_OUT /* use center cut out format */ +} video_displayformat_t; + +typedef struct { + int w; + int h; + video_format_t aspect_ratio; +} video_size_t; + +typedef enum { + VIDEO_SOURCE_DEMUX, /* Select the demux as the main source */ + VIDEO_SOURCE_MEMORY /* If this source is selected, the stream + comes from the user through the write + system call */ +} video_stream_source_t; + + +typedef enum { + VIDEO_STOPPED, /* Video is stopped */ + VIDEO_PLAYING, /* Video is currently playing */ + VIDEO_FREEZED /* Video is freezed */ +} video_play_state_t; + + +/* Decoder commands */ +#define VIDEO_CMD_PLAY (0) +#define VIDEO_CMD_STOP (1) +#define VIDEO_CMD_FREEZE (2) +#define VIDEO_CMD_CONTINUE (3) + +/* Flags for VIDEO_CMD_FREEZE */ +#define VIDEO_CMD_FREEZE_TO_BLACK (1 << 0) + +/* Flags for VIDEO_CMD_STOP */ +#define VIDEO_CMD_STOP_TO_BLACK (1 << 0) +#define VIDEO_CMD_STOP_IMMEDIATELY (1 << 1) + +/* Play input formats: */ +/* The decoder has no special format requirements */ +#define VIDEO_PLAY_FMT_NONE (0) +/* The decoder requires full GOPs */ +#define VIDEO_PLAY_FMT_GOP (1) + +/* The structure must be zeroed before use by the application + This ensures it can be extended safely in the future. */ +struct video_command { + __u32 cmd; + __u32 flags; + union { + struct { + __u64 pts; + } stop; + + struct { + /* 0 or 1000 specifies normal speed, + 1 specifies forward single stepping, + -1 specifies backward single stepping, + >1: playback at speed/1000 of the normal speed, + <-1: reverse playback at (-speed/1000) of the normal speed. */ + __s32 speed; + __u32 format; + } play; + + struct { + __u32 data[16]; + } raw; + }; +}; + +/* FIELD_UNKNOWN can be used if the hardware does not know whether + the Vsync is for an odd, even or progressive (i.e. non-interlaced) + field. */ +#define VIDEO_VSYNC_FIELD_UNKNOWN (0) +#define VIDEO_VSYNC_FIELD_ODD (1) +#define VIDEO_VSYNC_FIELD_EVEN (2) +#define VIDEO_VSYNC_FIELD_PROGRESSIVE (3) + +struct video_event { + __s32 type; +#define VIDEO_EVENT_SIZE_CHANGED 1 +#define VIDEO_EVENT_FRAME_RATE_CHANGED 2 +#define VIDEO_EVENT_DECODER_STOPPED 3 +#define VIDEO_EVENT_VSYNC 4 + /* unused, make sure to use atomic time for y2038 if it ever gets used */ + long timestamp; + union { + video_size_t size; + unsigned int frame_rate; /* in frames per 1000sec */ + unsigned char vsync_field; /* unknown/odd/even/progressive */ + } u; +}; + + +struct video_status { + int video_blank; /* blank video on freeze? */ + video_play_state_t play_state; /* current state of playback */ + video_stream_source_t stream_source; /* current source (demux/memory) */ + video_format_t video_format; /* current aspect ratio of stream*/ + video_displayformat_t display_format;/* selected cropping mode */ +}; + + +struct video_still_picture { + char __user *iFrame; /* pointer to a single iframe in memory */ + __s32 size; +}; + + +typedef __u16 video_attributes_t; +/* bits: descr. */ +/* 15-14 Video compression mode (0=MPEG-1, 1=MPEG-2) */ +/* 13-12 TV system (0=525/60, 1=625/50) */ +/* 11-10 Aspect ratio (0=4:3, 3=16:9) */ +/* 9- 8 permitted display mode on 4:3 monitor (0=both, 1=only pan-sca */ +/* 7 line 21-1 data present in GOP (1=yes, 0=no) */ +/* 6 line 21-2 data present in GOP (1=yes, 0=no) */ +/* 5- 3 source resolution (0=720x480/576, 1=704x480/576, 2=352x480/57 */ +/* 2 source letterboxed (1=yes, 0=no) */ +/* 0 film/camera mode (0= + *camera, 1=film (625/50 only)) */ + + +/* bit definitions for capabilities: */ +/* can the hardware decode MPEG1 and/or MPEG2? */ +#define VIDEO_CAP_MPEG1 1 +#define VIDEO_CAP_MPEG2 2 +/* can you send a system and/or program stream to video device? + (you still have to open the video and the audio device but only + send the stream to the video device) */ +#define VIDEO_CAP_SYS 4 +#define VIDEO_CAP_PROG 8 +/* can the driver also handle SPU, NAVI and CSS encoded data? + (CSS API is not present yet) */ +#define VIDEO_CAP_SPU 16 +#define VIDEO_CAP_NAVI 32 +#define VIDEO_CAP_CSS 64 + + +#define VIDEO_STOP _IO('o', 21) +#define VIDEO_PLAY _IO('o', 22) +#define VIDEO_FREEZE _IO('o', 23) +#define VIDEO_CONTINUE _IO('o', 24) +#define VIDEO_SELECT_SOURCE _IO('o', 25) +#define VIDEO_SET_BLANK _IO('o', 26) +#define VIDEO_GET_STATUS _IOR('o', 27, struct video_status) +#define VIDEO_GET_EVENT _IOR('o', 28, struct video_event) +#define VIDEO_SET_DISPLAY_FORMAT _IO('o', 29) +#define VIDEO_STILLPICTURE _IOW('o', 30, struct video_still_picture) +#define VIDEO_FAST_FORWARD _IO('o', 31) +#define VIDEO_SLOWMOTION _IO('o', 32) +#define VIDEO_GET_CAPABILITIES _IOR('o', 33, unsigned int) +#define VIDEO_CLEAR_BUFFER _IO('o', 34) +#define VIDEO_SET_STREAMTYPE _IO('o', 36) +#define VIDEO_SET_FORMAT _IO('o', 37) +#define VIDEO_GET_SIZE _IOR('o', 55, video_size_t) + +/** + * VIDEO_GET_PTS + * + * Read the 33 bit presentation time stamp as defined + * in ITU T-REC-H.222.0 / ISO/IEC 13818-1. + * + * The PTS should belong to the currently played + * frame if possible, but may also be a value close to it + * like the PTS of the last decoded frame or the last PTS + * extracted by the PES parser. + */ +#define VIDEO_GET_PTS _IOR('o', 57, __u64) + +/* Read the number of displayed frames since the decoder was started */ +#define VIDEO_GET_FRAME_COUNT _IOR('o', 58, __u64) + +#define VIDEO_COMMAND _IOWR('o', 59, struct video_command) +#define VIDEO_TRY_COMMAND _IOWR('o', 60, struct video_command) + +#endif /* _UAPI_DVBVIDEO_H_ */ diff --git a/include/uapi/linux/dvb/audio.h b/include/uapi/linux/dvb/audio.h deleted file mode 100644 index 2f869da69171..000000000000 --- a/include/uapi/linux/dvb/audio.h +++ /dev/null @@ -1,101 +0,0 @@ -/* SPDX-License-Identifier: LGPL-2.1+ WITH Linux-syscall-note */ -/* - * audio.h - DEPRECATED MPEG-TS audio decoder API - * - * NOTE: should not be used on future drivers - * - * Copyright (C) 2000 Ralph Metzler - * & Marcus Metzler - * for convergence integrated media GmbH - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Lesser Public License - * as published by the Free Software Foundation; either version 2.1 - * of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. - * - */ - -#ifndef _DVBAUDIO_H_ -#define _DVBAUDIO_H_ - -#include - -typedef enum { - AUDIO_SOURCE_DEMUX, /* Select the demux as the main source */ - AUDIO_SOURCE_MEMORY /* Select internal memory as the main source */ -} audio_stream_source_t; - - -typedef enum { - AUDIO_STOPPED, /* Device is stopped */ - AUDIO_PLAYING, /* Device is currently playing */ - AUDIO_PAUSED /* Device is paused */ -} audio_play_state_t; - - -typedef enum { - AUDIO_STEREO, - AUDIO_MONO_LEFT, - AUDIO_MONO_RIGHT, - AUDIO_MONO, - AUDIO_STEREO_SWAPPED -} audio_channel_select_t; - - -typedef struct audio_mixer { - unsigned int volume_left; - unsigned int volume_right; - /* what else do we need? bass, pass-through, ... */ -} audio_mixer_t; - - -typedef struct audio_status { - int AV_sync_state; /* sync audio and video? */ - int mute_state; /* audio is muted */ - audio_play_state_t play_state; /* current playback state */ - audio_stream_source_t stream_source; /* current stream source */ - audio_channel_select_t channel_select; /* currently selected channel */ - int bypass_mode; /* pass on audio data to */ - audio_mixer_t mixer_state; /* current mixer state */ -} audio_status_t; /* separate decoder hardware */ - - -/* for GET_CAPABILITIES and SET_FORMAT, the latter should only set one bit */ -#define AUDIO_CAP_DTS 1 -#define AUDIO_CAP_LPCM 2 -#define AUDIO_CAP_MP1 4 -#define AUDIO_CAP_MP2 8 -#define AUDIO_CAP_MP3 16 -#define AUDIO_CAP_AAC 32 -#define AUDIO_CAP_OGG 64 -#define AUDIO_CAP_SDDS 128 -#define AUDIO_CAP_AC3 256 - -#define AUDIO_STOP _IO('o', 1) -#define AUDIO_PLAY _IO('o', 2) -#define AUDIO_PAUSE _IO('o', 3) -#define AUDIO_CONTINUE _IO('o', 4) -#define AUDIO_SELECT_SOURCE _IO('o', 5) -#define AUDIO_SET_MUTE _IO('o', 6) -#define AUDIO_SET_AV_SYNC _IO('o', 7) -#define AUDIO_SET_BYPASS_MODE _IO('o', 8) -#define AUDIO_CHANNEL_SELECT _IO('o', 9) -#define AUDIO_GET_STATUS _IOR('o', 10, audio_status_t) - -#define AUDIO_GET_CAPABILITIES _IOR('o', 11, unsigned int) -#define AUDIO_CLEAR_BUFFER _IO('o', 12) -#define AUDIO_SET_ID _IO('o', 13) -#define AUDIO_SET_MIXER _IOW('o', 14, audio_mixer_t) -#define AUDIO_SET_STREAMTYPE _IO('o', 15) -#define AUDIO_BILINGUAL_CHANNEL_SELECT _IO('o', 20) - -#endif /* _DVBAUDIO_H_ */ diff --git a/include/uapi/linux/dvb/osd.h b/include/uapi/linux/dvb/osd.h deleted file mode 100644 index 858997c74043..000000000000 --- a/include/uapi/linux/dvb/osd.h +++ /dev/null @@ -1,181 +0,0 @@ -/* SPDX-License-Identifier: LGPL-2.1+ WITH Linux-syscall-note */ -/* - * osd.h - DEPRECATED On Screen Display API - * - * NOTE: should not be used on future drivers - * - * Copyright (C) 2001 Ralph Metzler - * & Marcus Metzler - * for convergence integrated media GmbH - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Lesser Public License - * as published by the Free Software Foundation; either version 2.1 - * of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. - * - */ - -#ifndef _DVBOSD_H_ -#define _DVBOSD_H_ - -#include - -typedef enum { - /* All functions return -2 on "not open" */ - OSD_Close = 1, /* () */ - /* - * Disables OSD and releases the buffers - * returns 0 on success - */ - OSD_Open, /* (x0,y0,x1,y1,BitPerPixel[2/4/8](color&0x0F),mix[0..15](color&0xF0)) */ - /* - * Opens OSD with this size and bit depth - * returns 0 on success, -1 on DRAM allocation error, -2 on "already open" - */ - OSD_Show, /* () */ - /* - * enables OSD mode - * returns 0 on success - */ - OSD_Hide, /* () */ - /* - * disables OSD mode - * returns 0 on success - */ - OSD_Clear, /* () */ - /* - * Sets all pixel to color 0 - * returns 0 on success - */ - OSD_Fill, /* (color) */ - /* - * Sets all pixel to color - * returns 0 on success - */ - OSD_SetColor, /* (color,R{x0},G{y0},B{x1},opacity{y1}) */ - /* - * set palette entry to , and apply - * R,G,B: 0..255 - * R=Red, G=Green, B=Blue - * opacity=0: pixel opacity 0% (only video pixel shows) - * opacity=1..254: pixel opacity as specified in header - * opacity=255: pixel opacity 100% (only OSD pixel shows) - * returns 0 on success, -1 on error - */ - OSD_SetPalette, /* (firstcolor{color},lastcolor{x0},data) */ - /* - * Set a number of entries in the palette - * sets the entries "firstcolor" through "lastcolor" from the array "data" - * data has 4 byte for each color: - * R,G,B, and a opacity value: 0->transparent, 1..254->mix, 255->pixel - */ - OSD_SetTrans, /* (transparency{color}) */ - /* - * Sets transparency of mixed pixel (0..15) - * returns 0 on success - */ - OSD_SetPixel, /* (x0,y0,color) */ - /* - * sets pixel , to color number - * returns 0 on success, -1 on error - */ - OSD_GetPixel, /* (x0,y0) */ - /* returns color number of pixel ,, or -1 */ - OSD_SetRow, /* (x0,y0,x1,data) */ - /* - * fills pixels x0,y through x1,y with the content of data[] - * returns 0 on success, -1 on clipping all pixel (no pixel drawn) - */ - OSD_SetBlock, /* (x0,y0,x1,y1,increment{color},data) */ - /* - * fills pixels x0,y0 through x1,y1 with the content of data[] - * inc contains the width of one line in the data block, - * inc<=0 uses blockwidth as linewidth - * returns 0 on success, -1 on clipping all pixel - */ - OSD_FillRow, /* (x0,y0,x1,color) */ - /* - * fills pixels x0,y through x1,y with the color - * returns 0 on success, -1 on clipping all pixel - */ - OSD_FillBlock, /* (x0,y0,x1,y1,color) */ - /* - * fills pixels x0,y0 through x1,y1 with the color - * returns 0 on success, -1 on clipping all pixel - */ - OSD_Line, /* (x0,y0,x1,y1,color) */ - /* - * draw a line from x0,y0 to x1,y1 with the color - * returns 0 on success - */ - OSD_Query, /* (x0,y0,x1,y1,xasp{color}}), yasp=11 */ - /* - * fills parameters with the picture dimensions and the pixel aspect ratio - * returns 0 on success - */ - OSD_Test, /* () */ - /* - * draws a test picture. for debugging purposes only - * returns 0 on success - * TODO: remove "test" in final version - */ - OSD_Text, /* (x0,y0,size,color,text) */ - OSD_SetWindow, /* (x0) set window with number 0 - * & Ralph Metzler - * for convergence integrated media GmbH - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public License - * as published by the Free Software Foundation; either version 2.1 - * of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. - * - */ - -#ifndef _UAPI_DVBVIDEO_H_ -#define _UAPI_DVBVIDEO_H_ - -#include -#ifndef __KERNEL__ -#include -#endif - -typedef enum { - VIDEO_FORMAT_4_3, /* Select 4:3 format */ - VIDEO_FORMAT_16_9, /* Select 16:9 format. */ - VIDEO_FORMAT_221_1 /* 2.21:1 */ -} video_format_t; - - -typedef enum { - VIDEO_PAN_SCAN, /* use pan and scan format */ - VIDEO_LETTER_BOX, /* use letterbox format */ - VIDEO_CENTER_CUT_OUT /* use center cut out format */ -} video_displayformat_t; - -typedef struct { - int w; - int h; - video_format_t aspect_ratio; -} video_size_t; - -typedef enum { - VIDEO_SOURCE_DEMUX, /* Select the demux as the main source */ - VIDEO_SOURCE_MEMORY /* If this source is selected, the stream - comes from the user through the write - system call */ -} video_stream_source_t; - - -typedef enum { - VIDEO_STOPPED, /* Video is stopped */ - VIDEO_PLAYING, /* Video is currently playing */ - VIDEO_FREEZED /* Video is freezed */ -} video_play_state_t; - - -/* Decoder commands */ -#define VIDEO_CMD_PLAY (0) -#define VIDEO_CMD_STOP (1) -#define VIDEO_CMD_FREEZE (2) -#define VIDEO_CMD_CONTINUE (3) - -/* Flags for VIDEO_CMD_FREEZE */ -#define VIDEO_CMD_FREEZE_TO_BLACK (1 << 0) - -/* Flags for VIDEO_CMD_STOP */ -#define VIDEO_CMD_STOP_TO_BLACK (1 << 0) -#define VIDEO_CMD_STOP_IMMEDIATELY (1 << 1) - -/* Play input formats: */ -/* The decoder has no special format requirements */ -#define VIDEO_PLAY_FMT_NONE (0) -/* The decoder requires full GOPs */ -#define VIDEO_PLAY_FMT_GOP (1) - -/* The structure must be zeroed before use by the application - This ensures it can be extended safely in the future. */ -struct video_command { - __u32 cmd; - __u32 flags; - union { - struct { - __u64 pts; - } stop; - - struct { - /* 0 or 1000 specifies normal speed, - 1 specifies forward single stepping, - -1 specifies backward single stepping, - >1: playback at speed/1000 of the normal speed, - <-1: reverse playback at (-speed/1000) of the normal speed. */ - __s32 speed; - __u32 format; - } play; - - struct { - __u32 data[16]; - } raw; - }; -}; - -/* FIELD_UNKNOWN can be used if the hardware does not know whether - the Vsync is for an odd, even or progressive (i.e. non-interlaced) - field. */ -#define VIDEO_VSYNC_FIELD_UNKNOWN (0) -#define VIDEO_VSYNC_FIELD_ODD (1) -#define VIDEO_VSYNC_FIELD_EVEN (2) -#define VIDEO_VSYNC_FIELD_PROGRESSIVE (3) - -struct video_event { - __s32 type; -#define VIDEO_EVENT_SIZE_CHANGED 1 -#define VIDEO_EVENT_FRAME_RATE_CHANGED 2 -#define VIDEO_EVENT_DECODER_STOPPED 3 -#define VIDEO_EVENT_VSYNC 4 - /* unused, make sure to use atomic time for y2038 if it ever gets used */ - long timestamp; - union { - video_size_t size; - unsigned int frame_rate; /* in frames per 1000sec */ - unsigned char vsync_field; /* unknown/odd/even/progressive */ - } u; -}; - - -struct video_status { - int video_blank; /* blank video on freeze? */ - video_play_state_t play_state; /* current state of playback */ - video_stream_source_t stream_source; /* current source (demux/memory) */ - video_format_t video_format; /* current aspect ratio of stream*/ - video_displayformat_t display_format;/* selected cropping mode */ -}; - - -struct video_still_picture { - char __user *iFrame; /* pointer to a single iframe in memory */ - __s32 size; -}; - - -typedef __u16 video_attributes_t; -/* bits: descr. */ -/* 15-14 Video compression mode (0=MPEG-1, 1=MPEG-2) */ -/* 13-12 TV system (0=525/60, 1=625/50) */ -/* 11-10 Aspect ratio (0=4:3, 3=16:9) */ -/* 9- 8 permitted display mode on 4:3 monitor (0=both, 1=only pan-sca */ -/* 7 line 21-1 data present in GOP (1=yes, 0=no) */ -/* 6 line 21-2 data present in GOP (1=yes, 0=no) */ -/* 5- 3 source resolution (0=720x480/576, 1=704x480/576, 2=352x480/57 */ -/* 2 source letterboxed (1=yes, 0=no) */ -/* 0 film/camera mode (0= - *camera, 1=film (625/50 only)) */ - - -/* bit definitions for capabilities: */ -/* can the hardware decode MPEG1 and/or MPEG2? */ -#define VIDEO_CAP_MPEG1 1 -#define VIDEO_CAP_MPEG2 2 -/* can you send a system and/or program stream to video device? - (you still have to open the video and the audio device but only - send the stream to the video device) */ -#define VIDEO_CAP_SYS 4 -#define VIDEO_CAP_PROG 8 -/* can the driver also handle SPU, NAVI and CSS encoded data? - (CSS API is not present yet) */ -#define VIDEO_CAP_SPU 16 -#define VIDEO_CAP_NAVI 32 -#define VIDEO_CAP_CSS 64 - - -#define VIDEO_STOP _IO('o', 21) -#define VIDEO_PLAY _IO('o', 22) -#define VIDEO_FREEZE _IO('o', 23) -#define VIDEO_CONTINUE _IO('o', 24) -#define VIDEO_SELECT_SOURCE _IO('o', 25) -#define VIDEO_SET_BLANK _IO('o', 26) -#define VIDEO_GET_STATUS _IOR('o', 27, struct video_status) -#define VIDEO_GET_EVENT _IOR('o', 28, struct video_event) -#define VIDEO_SET_DISPLAY_FORMAT _IO('o', 29) -#define VIDEO_STILLPICTURE _IOW('o', 30, struct video_still_picture) -#define VIDEO_FAST_FORWARD _IO('o', 31) -#define VIDEO_SLOWMOTION _IO('o', 32) -#define VIDEO_GET_CAPABILITIES _IOR('o', 33, unsigned int) -#define VIDEO_CLEAR_BUFFER _IO('o', 34) -#define VIDEO_SET_STREAMTYPE _IO('o', 36) -#define VIDEO_SET_FORMAT _IO('o', 37) -#define VIDEO_GET_SIZE _IOR('o', 55, video_size_t) - -/** - * VIDEO_GET_PTS - * - * Read the 33 bit presentation time stamp as defined - * in ITU T-REC-H.222.0 / ISO/IEC 13818-1. - * - * The PTS should belong to the currently played - * frame if possible, but may also be a value close to it - * like the PTS of the last decoded frame or the last PTS - * extracted by the PES parser. - */ -#define VIDEO_GET_PTS _IOR('o', 57, __u64) - -/* Read the number of displayed frames since the decoder was started */ -#define VIDEO_GET_FRAME_COUNT _IOR('o', 58, __u64) - -#define VIDEO_COMMAND _IOWR('o', 59, struct video_command) -#define VIDEO_TRY_COMMAND _IOWR('o', 60, struct video_command) - -#endif /* _UAPI_DVBVIDEO_H_ */ -- cgit v1.2.3 From 603e4922f1c81fc2ed3a87b4f91a8d3aafc7e093 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 31 May 2021 10:25:26 +0300 Subject: remove the raw driver The raw driver used to provide direct unbuffered access to block devices before O_DIRECT was invented. It has been obsolete for more than a decade. Acked-by: Greg Kroah-Hartman Acked-by: Arnd Bergmann Link: https://lore.kernel.org/lkml/Pine.LNX.4.64.0703180754060.6605@CPE00045a9c397f-CM001225dbafb6/ Signed-off-by: Christoph Hellwig Link: https://lore.kernel.org/r/20210531072526.97052-1-hch@lst.de Signed-off-by: Greg Kroah-Hartman --- drivers/char/Kconfig | 21 --- drivers/char/Makefile | 1 - drivers/char/mem.c | 1 - drivers/char/raw.c | 362 ----------------------------------------------- fs/block_dev.c | 6 +- include/linux/fs.h | 3 - include/uapi/linux/raw.h | 17 --- 7 files changed, 2 insertions(+), 409 deletions(-) delete mode 100644 drivers/char/raw.c delete mode 100644 include/uapi/linux/raw.h (limited to 'include/uapi') diff --git a/drivers/char/Kconfig b/drivers/char/Kconfig index b151e0fcdeb5..8e516aad632b 100644 --- a/drivers/char/Kconfig +++ b/drivers/char/Kconfig @@ -357,27 +357,6 @@ config NVRAM To compile this driver as a module, choose M here: the module will be called nvram. -config RAW_DRIVER - tristate "RAW driver (/dev/raw/rawN)" - depends on BLOCK - help - The raw driver permits block devices to be bound to /dev/raw/rawN. - Once bound, I/O against /dev/raw/rawN uses efficient zero-copy I/O. - See the raw(8) manpage for more details. - - Applications should preferably open the device (eg /dev/hda1) - with the O_DIRECT flag. - -config MAX_RAW_DEVS - int "Maximum number of RAW devices to support (1-65536)" - depends on RAW_DRIVER - range 1 65536 - default "256" - help - The maximum number of RAW devices that are supported. - Default is 256. Increase this number in case you need lots of - raw devices. - config DEVPORT bool "/dev/port character device" depends on ISA || PCI diff --git a/drivers/char/Makefile b/drivers/char/Makefile index c7e4fc733a37..264eb398fdd4 100644 --- a/drivers/char/Makefile +++ b/drivers/char/Makefile @@ -8,7 +8,6 @@ obj-$(CONFIG_TTY_PRINTK) += ttyprintk.o obj-y += misc.o obj-$(CONFIG_ATARI_DSP56K) += dsp56k.o obj-$(CONFIG_VIRTIO_CONSOLE) += virtio_console.o -obj-$(CONFIG_RAW_DRIVER) += raw.o obj-$(CONFIG_MSPEC) += mspec.o obj-$(CONFIG_UV_MMTIMER) += uv_mmtimer.o obj-$(CONFIG_IBM_BSR) += bsr.o diff --git a/drivers/char/mem.c b/drivers/char/mem.c index 15dc54fa1d47..1c596b5cdb27 100644 --- a/drivers/char/mem.c +++ b/drivers/char/mem.c @@ -16,7 +16,6 @@ #include #include #include -#include #include #include #include diff --git a/drivers/char/raw.c b/drivers/char/raw.c deleted file mode 100644 index 5d52a1f4738c..000000000000 --- a/drivers/char/raw.c +++ /dev/null @@ -1,362 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * linux/drivers/char/raw.c - * - * Front-end raw character devices. These can be bound to any block - * devices to provide genuine Unix raw character device semantics. - * - * We reserve minor number 0 for a control interface. ioctl()s on this - * device are used to bind the other minor numbers to block devices. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -struct raw_device_data { - dev_t binding; - struct block_device *bdev; - int inuse; -}; - -static struct class *raw_class; -static struct raw_device_data *raw_devices; -static DEFINE_MUTEX(raw_mutex); -static const struct file_operations raw_ctl_fops; /* forward declaration */ - -static int max_raw_minors = CONFIG_MAX_RAW_DEVS; - -module_param(max_raw_minors, int, 0); -MODULE_PARM_DESC(max_raw_minors, "Maximum number of raw devices (1-65536)"); - -/* - * Open/close code for raw IO. - * - * We just rewrite the i_mapping for the /dev/raw/rawN file descriptor to - * point at the blockdev's address_space and set the file handle to use - * O_DIRECT. - * - * Set the device's soft blocksize to the minimum possible. This gives the - * finest possible alignment and has no adverse impact on performance. - */ -static int raw_open(struct inode *inode, struct file *filp) -{ - const int minor = iminor(inode); - struct block_device *bdev; - int err; - - if (minor == 0) { /* It is the control device */ - filp->f_op = &raw_ctl_fops; - return 0; - } - - pr_warn_ratelimited( - "process %s (pid %d) is using the deprecated raw device\n" - "support will be removed in Linux 5.14.\n", - current->comm, current->pid); - - mutex_lock(&raw_mutex); - - /* - * All we need to do on open is check that the device is bound. - */ - err = -ENODEV; - if (!raw_devices[minor].binding) - goto out; - bdev = blkdev_get_by_dev(raw_devices[minor].binding, - filp->f_mode | FMODE_EXCL, raw_open); - if (IS_ERR(bdev)) { - err = PTR_ERR(bdev); - goto out; - } - err = set_blocksize(bdev, bdev_logical_block_size(bdev)); - if (err) - goto out1; - filp->f_flags |= O_DIRECT; - filp->f_mapping = bdev->bd_inode->i_mapping; - if (++raw_devices[minor].inuse == 1) - file_inode(filp)->i_mapping = - bdev->bd_inode->i_mapping; - filp->private_data = bdev; - raw_devices[minor].bdev = bdev; - mutex_unlock(&raw_mutex); - return 0; - -out1: - blkdev_put(bdev, filp->f_mode | FMODE_EXCL); -out: - mutex_unlock(&raw_mutex); - return err; -} - -/* - * When the final fd which refers to this character-special node is closed, we - * make its ->mapping point back at its own i_data. - */ -static int raw_release(struct inode *inode, struct file *filp) -{ - const int minor= iminor(inode); - struct block_device *bdev; - - mutex_lock(&raw_mutex); - bdev = raw_devices[minor].bdev; - if (--raw_devices[minor].inuse == 0) - /* Here inode->i_mapping == bdev->bd_inode->i_mapping */ - inode->i_mapping = &inode->i_data; - mutex_unlock(&raw_mutex); - - blkdev_put(bdev, filp->f_mode | FMODE_EXCL); - return 0; -} - -/* - * Forward ioctls to the underlying block device. - */ -static long -raw_ioctl(struct file *filp, unsigned int command, unsigned long arg) -{ - struct block_device *bdev = filp->private_data; - return blkdev_ioctl(bdev, 0, command, arg); -} - -static int bind_set(int number, u64 major, u64 minor) -{ - dev_t dev = MKDEV(major, minor); - dev_t raw = MKDEV(RAW_MAJOR, number); - struct raw_device_data *rawdev; - int err = 0; - - if (number <= 0 || number >= max_raw_minors) - return -EINVAL; - - if (MAJOR(dev) != major || MINOR(dev) != minor) - return -EINVAL; - - rawdev = &raw_devices[number]; - - /* - * This is like making block devices, so demand the - * same capability - */ - if (!capable(CAP_SYS_ADMIN)) - return -EPERM; - - /* - * For now, we don't need to check that the underlying - * block device is present or not: we can do that when - * the raw device is opened. Just check that the - * major/minor numbers make sense. - */ - - if (MAJOR(dev) == 0 && dev != 0) - return -EINVAL; - - mutex_lock(&raw_mutex); - if (rawdev->inuse) { - mutex_unlock(&raw_mutex); - return -EBUSY; - } - if (rawdev->binding) - module_put(THIS_MODULE); - - rawdev->binding = dev; - if (!dev) { - /* unbind */ - device_destroy(raw_class, raw); - } else { - __module_get(THIS_MODULE); - device_destroy(raw_class, raw); - device_create(raw_class, NULL, raw, NULL, "raw%d", number); - } - mutex_unlock(&raw_mutex); - return err; -} - -static int bind_get(int number, dev_t *dev) -{ - if (number <= 0 || number >= max_raw_minors) - return -EINVAL; - *dev = raw_devices[number].binding; - return 0; -} - -/* - * Deal with ioctls against the raw-device control interface, to bind - * and unbind other raw devices. - */ -static long raw_ctl_ioctl(struct file *filp, unsigned int command, - unsigned long arg) -{ - struct raw_config_request rq; - dev_t dev; - int err; - - switch (command) { - case RAW_SETBIND: - if (copy_from_user(&rq, (void __user *) arg, sizeof(rq))) - return -EFAULT; - - return bind_set(rq.raw_minor, rq.block_major, rq.block_minor); - - case RAW_GETBIND: - if (copy_from_user(&rq, (void __user *) arg, sizeof(rq))) - return -EFAULT; - - err = bind_get(rq.raw_minor, &dev); - if (err) - return err; - - rq.block_major = MAJOR(dev); - rq.block_minor = MINOR(dev); - - if (copy_to_user((void __user *)arg, &rq, sizeof(rq))) - return -EFAULT; - - return 0; - } - - return -EINVAL; -} - -#ifdef CONFIG_COMPAT -struct raw32_config_request { - compat_int_t raw_minor; - compat_u64 block_major; - compat_u64 block_minor; -}; - -static long raw_ctl_compat_ioctl(struct file *file, unsigned int cmd, - unsigned long arg) -{ - struct raw32_config_request __user *user_req = compat_ptr(arg); - struct raw32_config_request rq; - dev_t dev; - int err = 0; - - switch (cmd) { - case RAW_SETBIND: - if (copy_from_user(&rq, user_req, sizeof(rq))) - return -EFAULT; - - return bind_set(rq.raw_minor, rq.block_major, rq.block_minor); - - case RAW_GETBIND: - if (copy_from_user(&rq, user_req, sizeof(rq))) - return -EFAULT; - - err = bind_get(rq.raw_minor, &dev); - if (err) - return err; - - rq.block_major = MAJOR(dev); - rq.block_minor = MINOR(dev); - - if (copy_to_user(user_req, &rq, sizeof(rq))) - return -EFAULT; - - return 0; - } - - return -EINVAL; -} -#endif - -static const struct file_operations raw_fops = { - .read_iter = blkdev_read_iter, - .write_iter = blkdev_write_iter, - .fsync = blkdev_fsync, - .open = raw_open, - .release = raw_release, - .unlocked_ioctl = raw_ioctl, - .llseek = default_llseek, - .owner = THIS_MODULE, -}; - -static const struct file_operations raw_ctl_fops = { - .unlocked_ioctl = raw_ctl_ioctl, -#ifdef CONFIG_COMPAT - .compat_ioctl = raw_ctl_compat_ioctl, -#endif - .open = raw_open, - .owner = THIS_MODULE, - .llseek = noop_llseek, -}; - -static struct cdev raw_cdev; - -static char *raw_devnode(struct device *dev, umode_t *mode) -{ - return kasprintf(GFP_KERNEL, "raw/%s", dev_name(dev)); -} - -static int __init raw_init(void) -{ - dev_t dev = MKDEV(RAW_MAJOR, 0); - int ret; - - if (max_raw_minors < 1 || max_raw_minors > 65536) { - pr_warn("raw: invalid max_raw_minors (must be between 1 and 65536), using %d\n", - CONFIG_MAX_RAW_DEVS); - max_raw_minors = CONFIG_MAX_RAW_DEVS; - } - - raw_devices = vzalloc(array_size(max_raw_minors, - sizeof(struct raw_device_data))); - if (!raw_devices) { - printk(KERN_ERR "Not enough memory for raw device structures\n"); - ret = -ENOMEM; - goto error; - } - - ret = register_chrdev_region(dev, max_raw_minors, "raw"); - if (ret) - goto error; - - cdev_init(&raw_cdev, &raw_fops); - ret = cdev_add(&raw_cdev, dev, max_raw_minors); - if (ret) - goto error_region; - raw_class = class_create(THIS_MODULE, "raw"); - if (IS_ERR(raw_class)) { - printk(KERN_ERR "Error creating raw class.\n"); - cdev_del(&raw_cdev); - ret = PTR_ERR(raw_class); - goto error_region; - } - raw_class->devnode = raw_devnode; - device_create(raw_class, NULL, MKDEV(RAW_MAJOR, 0), NULL, "rawctl"); - - return 0; - -error_region: - unregister_chrdev_region(dev, max_raw_minors); -error: - vfree(raw_devices); - return ret; -} - -static void __exit raw_exit(void) -{ - device_destroy(raw_class, MKDEV(RAW_MAJOR, 0)); - class_destroy(raw_class); - cdev_del(&raw_cdev); - unregister_chrdev_region(MKDEV(RAW_MAJOR, 0), max_raw_minors); -} - -module_init(raw_init); -module_exit(raw_exit); -MODULE_LICENSE("GPL"); diff --git a/fs/block_dev.c b/fs/block_dev.c index 6cc4d4cfe0c2..15c448eb8226 100644 --- a/fs/block_dev.c +++ b/fs/block_dev.c @@ -1669,7 +1669,7 @@ static long block_ioctl(struct file *file, unsigned cmd, unsigned long arg) * Does not take i_mutex for the write and thus is not for general purpose * use. */ -ssize_t blkdev_write_iter(struct kiocb *iocb, struct iov_iter *from) +static ssize_t blkdev_write_iter(struct kiocb *iocb, struct iov_iter *from) { struct file *file = iocb->ki_filp; struct inode *bd_inode = bdev_file_inode(file); @@ -1707,9 +1707,8 @@ ssize_t blkdev_write_iter(struct kiocb *iocb, struct iov_iter *from) blk_finish_plug(&plug); return ret; } -EXPORT_SYMBOL_GPL(blkdev_write_iter); -ssize_t blkdev_read_iter(struct kiocb *iocb, struct iov_iter *to) +static ssize_t blkdev_read_iter(struct kiocb *iocb, struct iov_iter *to) { struct file *file = iocb->ki_filp; struct inode *bd_inode = bdev_file_inode(file); @@ -1731,7 +1730,6 @@ ssize_t blkdev_read_iter(struct kiocb *iocb, struct iov_iter *to) iov_iter_reexpand(to, iov_iter_count(to) + shorted); return ret; } -EXPORT_SYMBOL_GPL(blkdev_read_iter); /* * Try to release a page associated with block device when the system diff --git a/include/linux/fs.h b/include/linux/fs.h index c3c88fdb9b2a..8652ed7cdce8 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -3242,11 +3242,8 @@ ssize_t vfs_iocb_iter_write(struct file *file, struct kiocb *iocb, struct iov_iter *iter); /* fs/block_dev.c */ -extern ssize_t blkdev_read_iter(struct kiocb *iocb, struct iov_iter *to); -extern ssize_t blkdev_write_iter(struct kiocb *iocb, struct iov_iter *from); extern int blkdev_fsync(struct file *filp, loff_t start, loff_t end, int datasync); -extern void block_sync_page(struct page *page); /* fs/splice.c */ extern ssize_t generic_file_splice_read(struct file *, loff_t *, diff --git a/include/uapi/linux/raw.h b/include/uapi/linux/raw.h deleted file mode 100644 index 47874919d0b9..000000000000 --- a/include/uapi/linux/raw.h +++ /dev/null @@ -1,17 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -#ifndef __LINUX_RAW_H -#define __LINUX_RAW_H - -#include - -#define RAW_SETBIND _IO( 0xac, 0 ) -#define RAW_GETBIND _IO( 0xac, 1 ) - -struct raw_config_request -{ - int raw_minor; - __u64 block_major; - __u64 block_minor; -}; - -#endif /* __LINUX_RAW_H */ -- cgit v1.2.3 From 64c2c2c62f92339b176ea24403d8db16db36f9e6 Mon Sep 17 00:00:00 2001 From: Jan Kara Date: Tue, 25 May 2021 16:07:48 +0200 Subject: quota: Change quotactl_path() systcall to an fd-based one Some users have pointed out that path-based syscalls are problematic in some environments and at least directory fd argument and possibly also resolve flags are desirable for such syscalls. Rather than reimplementing all details of pathname lookup and following where it may eventually evolve, let's go for full file descriptor based syscall similar to how ioctl(2) works since the beginning. Managing of quotas isn't performance sensitive so the extra overhead of open does not matter and we are able to consume O_PATH descriptors as well which makes open cheap anyway. Also for frequent operations (such as retrieving usage information for all users) we can reuse single fd and in fact get even better performance as well as avoiding races with possible remounts etc. Tested-by: Sascha Hauer Acked-by: Christian Brauner Reviewed-by: Christoph Hellwig Signed-off-by: Jan Kara --- fs/quota/quota.c | 28 +++++++++++++--------------- include/linux/syscalls.h | 4 ++-- include/uapi/asm-generic/unistd.h | 4 ++-- kernel/sys_ni.c | 2 +- 4 files changed, 18 insertions(+), 20 deletions(-) (limited to 'include/uapi') diff --git a/fs/quota/quota.c b/fs/quota/quota.c index 05e4bd9ab6d6..2bcc9a6f1bfc 100644 --- a/fs/quota/quota.c +++ b/fs/quota/quota.c @@ -968,31 +968,30 @@ out: return ret; } -SYSCALL_DEFINE4(quotactl_path, unsigned int, cmd, const char __user *, - mountpoint, qid_t, id, void __user *, addr) +SYSCALL_DEFINE4(quotactl_fd, unsigned int, fd, unsigned int, cmd, + qid_t, id, void __user *, addr) { struct super_block *sb; - struct path mountpath; unsigned int cmds = cmd >> SUBCMDSHIFT; unsigned int type = cmd & SUBCMDMASK; + struct fd f; int ret; - if (type >= MAXQUOTAS) - return -EINVAL; + f = fdget_raw(fd); + if (!f.file) + return -EBADF; - ret = user_path_at(AT_FDCWD, mountpoint, - LOOKUP_FOLLOW | LOOKUP_AUTOMOUNT, &mountpath); - if (ret) - return ret; - - sb = mountpath.mnt->mnt_sb; + ret = -EINVAL; + if (type >= MAXQUOTAS) + goto out; if (quotactl_cmd_write(cmds)) { - ret = mnt_want_write(mountpath.mnt); + ret = mnt_want_write(f.file->f_path.mnt); if (ret) goto out; } + sb = f.file->f_path.mnt->mnt_sb; if (quotactl_cmd_onoff(cmds)) down_write(&sb->s_umount); else @@ -1006,9 +1005,8 @@ SYSCALL_DEFINE4(quotactl_path, unsigned int, cmd, const char __user *, up_read(&sb->s_umount); if (quotactl_cmd_write(cmds)) - mnt_drop_write(mountpath.mnt); + mnt_drop_write(f.file->f_path.mnt); out: - path_put(&mountpath); - + fdput(f); return ret; } diff --git a/include/linux/syscalls.h b/include/linux/syscalls.h index 050511e8f1f8..586128d5c3b8 100644 --- a/include/linux/syscalls.h +++ b/include/linux/syscalls.h @@ -485,8 +485,8 @@ asmlinkage long sys_pipe2(int __user *fildes, int flags); /* fs/quota.c */ asmlinkage long sys_quotactl(unsigned int cmd, const char __user *special, qid_t id, void __user *addr); -asmlinkage long sys_quotactl_path(unsigned int cmd, const char __user *mountpoint, - qid_t id, void __user *addr); +asmlinkage long sys_quotactl_fd(unsigned int fd, unsigned int cmd, qid_t id, + void __user *addr); /* fs/readdir.c */ asmlinkage long sys_getdents64(unsigned int fd, diff --git a/include/uapi/asm-generic/unistd.h b/include/uapi/asm-generic/unistd.h index 6de5a7fc066b..f211961ce1da 100644 --- a/include/uapi/asm-generic/unistd.h +++ b/include/uapi/asm-generic/unistd.h @@ -863,8 +863,8 @@ __SYSCALL(__NR_process_madvise, sys_process_madvise) __SC_COMP(__NR_epoll_pwait2, sys_epoll_pwait2, compat_sys_epoll_pwait2) #define __NR_mount_setattr 442 __SYSCALL(__NR_mount_setattr, sys_mount_setattr) -#define __NR_quotactl_path 443 -__SYSCALL(__NR_quotactl_path, sys_quotactl_path) +#define __NR_quotactl_fd 443 +__SYSCALL(__NR_quotactl_fd, sys_quotactl_fd) #define __NR_landlock_create_ruleset 444 __SYSCALL(__NR_landlock_create_ruleset, sys_landlock_create_ruleset) diff --git a/kernel/sys_ni.c b/kernel/sys_ni.c index 0ea8128468c3..dad4d994641e 100644 --- a/kernel/sys_ni.c +++ b/kernel/sys_ni.c @@ -99,7 +99,7 @@ COND_SYSCALL(flock); /* fs/quota.c */ COND_SYSCALL(quotactl); -COND_SYSCALL(quotactl_path); +COND_SYSCALL(quotactl_fd); /* fs/readdir.c */ -- cgit v1.2.3 From e2cf17d3774c323ef6dab6e9f7c0cfc5e742afd9 Mon Sep 17 00:00:00 2001 From: Florian Westphal Date: Fri, 4 Jun 2021 12:27:07 +0200 Subject: netfilter: add new hook nfnl subsystem This nfnl subsystem allows to dump the list of all active netfiler hooks, e.g. defrag, conntrack, nf/ip/arp/ip6tables and so on. This helps to see what kind of features are currently enabled in the network stack. Sample output from nft tool using this infra: $ nft list hook ip input family ip hook input { +0000000010 nft_do_chain_inet [nf_tables] # nft table firewalld INPUT +0000000100 nf_nat_ipv4_local_in [nf_nat] +2147483647 ipv4_confirm [nf_conntrack] } Signed-off-by: Florian Westphal Signed-off-by: Pablo Neira Ayuso --- include/uapi/linux/netfilter/nfnetlink.h | 3 +- include/uapi/linux/netfilter/nfnetlink_hook.h | 55 ++++ net/netfilter/Kconfig | 9 + net/netfilter/Makefile | 1 + net/netfilter/nfnetlink.c | 1 + net/netfilter/nfnetlink_hook.c | 375 ++++++++++++++++++++++++++ 6 files changed, 443 insertions(+), 1 deletion(-) create mode 100644 include/uapi/linux/netfilter/nfnetlink_hook.h create mode 100644 net/netfilter/nfnetlink_hook.c (limited to 'include/uapi') diff --git a/include/uapi/linux/netfilter/nfnetlink.h b/include/uapi/linux/netfilter/nfnetlink.h index 5bc960f220b3..6cd58cd2a6f0 100644 --- a/include/uapi/linux/netfilter/nfnetlink.h +++ b/include/uapi/linux/netfilter/nfnetlink.h @@ -60,7 +60,8 @@ struct nfgenmsg { #define NFNL_SUBSYS_CTHELPER 9 #define NFNL_SUBSYS_NFTABLES 10 #define NFNL_SUBSYS_NFT_COMPAT 11 -#define NFNL_SUBSYS_COUNT 12 +#define NFNL_SUBSYS_HOOK 12 +#define NFNL_SUBSYS_COUNT 13 /* Reserved control nfnetlink messages */ #define NFNL_MSG_BATCH_BEGIN NLMSG_MIN_TYPE diff --git a/include/uapi/linux/netfilter/nfnetlink_hook.h b/include/uapi/linux/netfilter/nfnetlink_hook.h new file mode 100644 index 000000000000..912ec60b26b0 --- /dev/null +++ b/include/uapi/linux/netfilter/nfnetlink_hook.h @@ -0,0 +1,55 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +#ifndef _NFNL_HOOK_H_ +#define _NFNL_HOOK_H_ + +enum nfnl_hook_msg_types { + NFNL_MSG_HOOK_GET, + NFNL_MSG_HOOK_MAX, +}; + +/** + * enum nfnl_hook_attributes - netfilter hook netlink attributes + * + * @NFNLA_HOOK_HOOKNUM: netfilter hook number (NLA_U32) + * @NFNLA_HOOK_PRIORITY: netfilter hook priority (NLA_U32) + * @NFNLA_HOOK_DEV: netdevice name (NLA_STRING) + * @NFNLA_HOOK_FUNCTION_NAME: hook function name (NLA_STRING) + * @NFNLA_HOOK_MODULE_NAME: kernel module that registered this hook (NLA_STRING) + * @NFNLA_HOOK_CHAIN_INFO: basechain hook metadata (NLA_NESTED) + */ +enum nfnl_hook_attributes { + NFNLA_HOOK_UNSPEC, + NFNLA_HOOK_HOOKNUM, + NFNLA_HOOK_PRIORITY, + NFNLA_HOOK_DEV, + NFNLA_HOOK_FUNCTION_NAME, + NFNLA_HOOK_MODULE_NAME, + NFNLA_HOOK_CHAIN_INFO, + __NFNLA_HOOK_MAX +}; +#define NFNLA_HOOK_MAX (__NFNLA_HOOK_MAX - 1) + +/** + * enum nfnl_hook_chain_info_attributes - chain description + * + * NFNLA_HOOK_INFO_DESC: nft chain and table name (enum nft_table_attributes) (NLA_NESTED) + * NFNLA_HOOK_INFO_TYPE: chain type (enum nfnl_hook_chaintype) (NLA_U32) + */ +enum nfnl_hook_chain_info_attributes { + NFNLA_HOOK_INFO_UNSPEC, + NFNLA_HOOK_INFO_DESC, + NFNLA_HOOK_INFO_TYPE, + __NFNLA_HOOK_INFO_MAX, +}; +#define NFNLA_HOOK_INFO_MAX (__NFNLA_HOOK_INFO_MAX - 1) + +/** + * enum nfnl_hook_chaintype - chain type + * + * @NFNL_HOOK_TYPE_NFTABLES nf_tables base chain + */ +enum nfnl_hook_chaintype { + NFNL_HOOK_TYPE_NFTABLES = 0x1, +}; + +#endif /* _NFNL_HOOK_H */ diff --git a/net/netfilter/Kconfig b/net/netfilter/Kconfig index 172d74560632..c81321372198 100644 --- a/net/netfilter/Kconfig +++ b/net/netfilter/Kconfig @@ -19,6 +19,15 @@ config NETFILTER_FAMILY_BRIDGE config NETFILTER_FAMILY_ARP bool +config NETFILTER_NETLINK_HOOK + tristate "Netfilter base hook dump support" + depends on NETFILTER_ADVANCED + select NETFILTER_NETLINK + help + If this option is enabled, the kernel will include support + to list the base netfilter hooks via NFNETLINK. + This is helpful for debugging. + config NETFILTER_NETLINK_ACCT tristate "Netfilter NFACCT over NFNETLINK interface" depends on NETFILTER_ADVANCED diff --git a/net/netfilter/Makefile b/net/netfilter/Makefile index e80e010354b1..87112dad1fd4 100644 --- a/net/netfilter/Makefile +++ b/net/netfilter/Makefile @@ -22,6 +22,7 @@ obj-$(CONFIG_NETFILTER_NETLINK_ACCT) += nfnetlink_acct.o obj-$(CONFIG_NETFILTER_NETLINK_QUEUE) += nfnetlink_queue.o obj-$(CONFIG_NETFILTER_NETLINK_LOG) += nfnetlink_log.o obj-$(CONFIG_NETFILTER_NETLINK_OSF) += nfnetlink_osf.o +obj-$(CONFIG_NETFILTER_NETLINK_HOOK) += nfnetlink_hook.o # connection tracking obj-$(CONFIG_NF_CONNTRACK) += nf_conntrack.o diff --git a/net/netfilter/nfnetlink.c b/net/netfilter/nfnetlink.c index 028a1f39318b..7e2c8dd01408 100644 --- a/net/netfilter/nfnetlink.c +++ b/net/netfilter/nfnetlink.c @@ -68,6 +68,7 @@ static const char *const nfnl_lockdep_names[NFNL_SUBSYS_COUNT] = { [NFNL_SUBSYS_CTHELPER] = "nfnl_subsys_cthelper", [NFNL_SUBSYS_NFTABLES] = "nfnl_subsys_nftables", [NFNL_SUBSYS_NFT_COMPAT] = "nfnl_subsys_nftcompat", + [NFNL_SUBSYS_HOOK] = "nfnl_subsys_hook", }; static const int nfnl_group2type[NFNLGRP_MAX+1] = { diff --git a/net/netfilter/nfnetlink_hook.c b/net/netfilter/nfnetlink_hook.c new file mode 100644 index 000000000000..04586dfa2acd --- /dev/null +++ b/net/netfilter/nfnetlink_hook.c @@ -0,0 +1,375 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (c) 2021 Red Hat GmbH + * + * Author: Florian Westphal + */ + +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +#include + +#include +#include + +static const struct nla_policy nfnl_hook_nla_policy[NFNLA_HOOK_MAX + 1] = { + [NFNLA_HOOK_HOOKNUM] = { .type = NLA_U32 }, + [NFNLA_HOOK_PRIORITY] = { .type = NLA_U32 }, + [NFNLA_HOOK_DEV] = { .type = NLA_STRING, + .len = IFNAMSIZ - 1 }, + [NFNLA_HOOK_FUNCTION_NAME] = { .type = NLA_NUL_STRING, + .len = KSYM_NAME_LEN, }, + [NFNLA_HOOK_MODULE_NAME] = { .type = NLA_NUL_STRING, + .len = MODULE_NAME_LEN, }, + [NFNLA_HOOK_CHAIN_INFO] = { .type = NLA_NESTED, }, +}; + +static int nf_netlink_dump_start_rcu(struct sock *nlsk, struct sk_buff *skb, + const struct nlmsghdr *nlh, + struct netlink_dump_control *c) +{ + int err; + + if (!try_module_get(THIS_MODULE)) + return -EINVAL; + + rcu_read_unlock(); + err = netlink_dump_start(nlsk, skb, nlh, c); + rcu_read_lock(); + module_put(THIS_MODULE); + + return err; +} + +struct nfnl_dump_hook_data { + char devname[IFNAMSIZ]; + unsigned long headv; + u8 hook; +}; + +static int nfnl_hook_put_nft_chain_info(struct sk_buff *nlskb, + const struct nfnl_dump_hook_data *ctx, + unsigned int seq, + const struct nf_hook_ops *ops) +{ + struct net *net = sock_net(nlskb->sk); + struct nlattr *nest, *nest2; + struct nft_chain *chain; + int ret = 0; + + if (ops->hook_ops_type != NF_HOOK_OP_NF_TABLES) + return 0; + + chain = ops->priv; + if (WARN_ON_ONCE(!chain)) + return 0; + + if (!nft_is_active(net, chain)) + return 0; + + nest = nla_nest_start(nlskb, NFNLA_HOOK_CHAIN_INFO); + if (!nest) + return -EMSGSIZE; + + ret = nla_put_be32(nlskb, NFNLA_HOOK_INFO_TYPE, + htonl(NFNL_HOOK_TYPE_NFTABLES)); + if (ret) + goto cancel_nest; + + nest2 = nla_nest_start(nlskb, NFNLA_HOOK_INFO_DESC); + if (!nest2) + goto cancel_nest; + + ret = nla_put_string(nlskb, NFTA_CHAIN_TABLE, chain->table->name); + if (ret) + goto cancel_nest; + + ret = nla_put_string(nlskb, NFTA_CHAIN_NAME, chain->name); + if (ret) + goto cancel_nest; + + nla_nest_end(nlskb, nest2); + nla_nest_end(nlskb, nest); + return ret; + +cancel_nest: + nla_nest_cancel(nlskb, nest); + return -EMSGSIZE; +} + +static int nfnl_hook_dump_one(struct sk_buff *nlskb, + const struct nfnl_dump_hook_data *ctx, + const struct nf_hook_ops *ops, + unsigned int seq) +{ + u16 event = nfnl_msg_type(NFNL_SUBSYS_HOOK, NFNL_MSG_HOOK_GET); + unsigned int portid = NETLINK_CB(nlskb).portid; + struct nlmsghdr *nlh; + int ret = -EMSGSIZE; +#ifdef CONFIG_KALLSYMS + char sym[KSYM_SYMBOL_LEN]; + char *module_name; +#endif + nlh = nfnl_msg_put(nlskb, portid, seq, event, + NLM_F_MULTI, ops->pf, NFNETLINK_V0, 0); + if (!nlh) + goto nla_put_failure; + +#ifdef CONFIG_KALLSYMS + ret = snprintf(sym, sizeof(sym), "%ps", ops->hook); + if (ret < 0 || ret > (int)sizeof(sym)) + goto nla_put_failure; + + module_name = strstr(sym, " ["); + if (module_name) { + char *end; + + module_name += 2; + end = strchr(module_name, ']'); + if (end) { + *end = 0; + + ret = nla_put_string(nlskb, NFNLA_HOOK_MODULE_NAME, module_name); + if (ret) + goto nla_put_failure; + } + } + + ret = nla_put_string(nlskb, NFNLA_HOOK_FUNCTION_NAME, sym); + if (ret) + goto nla_put_failure; +#endif + + ret = nla_put_be32(nlskb, NFNLA_HOOK_HOOKNUM, htonl(ops->hooknum)); + if (ret) + goto nla_put_failure; + + ret = nla_put_be32(nlskb, NFNLA_HOOK_PRIORITY, htonl(ops->priority)); + if (ret) + goto nla_put_failure; + + ret = nfnl_hook_put_nft_chain_info(nlskb, ctx, seq, ops); + if (ret) + goto nla_put_failure; + + nlmsg_end(nlskb, nlh); + return 0; +nla_put_failure: + nlmsg_trim(nlskb, nlh); + return ret; +} + +static const struct nf_hook_entries * +nfnl_hook_entries_head(u8 pf, unsigned int hook, struct net *net, const char *dev) +{ + const struct nf_hook_entries *hook_head = NULL; + struct net_device *netdev; + + switch (pf) { + case NFPROTO_IPV4: + if (hook >= ARRAY_SIZE(net->nf.hooks_ipv4)) + return ERR_PTR(-EINVAL); + hook_head = rcu_dereference(net->nf.hooks_ipv4[hook]); + break; + case NFPROTO_IPV6: + hook_head = rcu_dereference(net->nf.hooks_ipv6[hook]); + if (hook >= ARRAY_SIZE(net->nf.hooks_ipv6)) + return ERR_PTR(-EINVAL); + break; + case NFPROTO_ARP: +#ifdef CONFIG_NETFILTER_FAMILY_ARP + if (hook >= ARRAY_SIZE(net->nf.hooks_arp)) + return ERR_PTR(-EINVAL); + hook_head = rcu_dereference(net->nf.hooks_arp[hook]); +#endif + break; + case NFPROTO_BRIDGE: +#ifdef CONFIG_NETFILTER_FAMILY_BRIDGE + if (hook >= ARRAY_SIZE(net->nf.hooks_bridge)) + return ERR_PTR(-EINVAL); + hook_head = rcu_dereference(net->nf.hooks_bridge[hook]); +#endif + break; +#if IS_ENABLED(CONFIG_DECNET) + case NFPROTO_DECNET: + if (hook >= ARRAY_SIZE(net->nf.hooks_decnet)) + return ERR_PTR(-EINVAL); + hook_head = rcu_dereference(net->nf.hooks_decnet[hook]); + break; +#endif +#ifdef CONFIG_NETFILTER_INGRESS + case NFPROTO_NETDEV: + if (hook != NF_NETDEV_INGRESS) + return ERR_PTR(-EOPNOTSUPP); + + if (!dev) + return ERR_PTR(-ENODEV); + + netdev = dev_get_by_name_rcu(net, dev); + if (!netdev) + return ERR_PTR(-ENODEV); + + return rcu_dereference(netdev->nf_hooks_ingress); +#endif + default: + return ERR_PTR(-EPROTONOSUPPORT); + } + + return hook_head; +} + +static int nfnl_hook_dump(struct sk_buff *nlskb, + struct netlink_callback *cb) +{ + struct nfgenmsg *nfmsg = nlmsg_data(cb->nlh); + struct nfnl_dump_hook_data *ctx = cb->data; + int err, family = nfmsg->nfgen_family; + struct net *net = sock_net(nlskb->sk); + struct nf_hook_ops * const *ops; + const struct nf_hook_entries *e; + unsigned int i = cb->args[0]; + + rcu_read_lock(); + + e = nfnl_hook_entries_head(family, ctx->hook, net, ctx->devname); + if (!e) + goto done; + + if (IS_ERR(e)) { + cb->seq++; + goto done; + } + + if ((unsigned long)e != ctx->headv || i >= e->num_hook_entries) + cb->seq++; + + ops = nf_hook_entries_get_hook_ops(e); + + for (; i < e->num_hook_entries; i++) { + err = nfnl_hook_dump_one(nlskb, ctx, ops[i], cb->seq); + if (err) + break; + } + +done: + nl_dump_check_consistent(cb, nlmsg_hdr(nlskb)); + rcu_read_unlock(); + cb->args[0] = i; + return nlskb->len; +} + +static int nfnl_hook_dump_start(struct netlink_callback *cb) +{ + const struct nfgenmsg *nfmsg = nlmsg_data(cb->nlh); + const struct nlattr * const *nla = cb->data; + struct nfnl_dump_hook_data *ctx = NULL; + struct net *net = sock_net(cb->skb->sk); + u8 family = nfmsg->nfgen_family; + char name[IFNAMSIZ] = ""; + const void *head; + u32 hooknum; + + hooknum = ntohl(nla_get_be32(nla[NFNLA_HOOK_HOOKNUM])); + if (hooknum > 255) + return -EINVAL; + + if (family == NFPROTO_NETDEV) { + if (!nla[NFNLA_HOOK_DEV]) + return -EINVAL; + + nla_strscpy(name, nla[NFNLA_HOOK_DEV], sizeof(name)); + } + + rcu_read_lock(); + /* Not dereferenced; for consistency check only */ + head = nfnl_hook_entries_head(family, hooknum, net, name); + rcu_read_unlock(); + + if (head && IS_ERR(head)) + return PTR_ERR(head); + + ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); + if (!ctx) + return -ENOMEM; + + strscpy(ctx->devname, name, sizeof(ctx->devname)); + ctx->headv = (unsigned long)head; + ctx->hook = hooknum; + + cb->seq = 1; + cb->data = ctx; + + return 0; +} + +static int nfnl_hook_dump_stop(struct netlink_callback *cb) +{ + kfree(cb->data); + return 0; +} + +static int nfnl_hook_get(struct sk_buff *skb, + const struct nfnl_info *info, + const struct nlattr * const nla[]) +{ + if (!nla[NFNLA_HOOK_HOOKNUM]) + return -EINVAL; + + if (info->nlh->nlmsg_flags & NLM_F_DUMP) { + struct netlink_dump_control c = { + .start = nfnl_hook_dump_start, + .done = nfnl_hook_dump_stop, + .dump = nfnl_hook_dump, + .module = THIS_MODULE, + .data = (void *)nla, + }; + + return nf_netlink_dump_start_rcu(info->sk, skb, info->nlh, &c); + } + + return -EOPNOTSUPP; +} + +static const struct nfnl_callback nfnl_hook_cb[NFNL_MSG_HOOK_MAX] = { + [NFNL_MSG_HOOK_GET] = { + .call = nfnl_hook_get, + .type = NFNL_CB_RCU, + .attr_count = NFNLA_HOOK_MAX, + .policy = nfnl_hook_nla_policy + }, +}; + +static const struct nfnetlink_subsystem nfhook_subsys = { + .name = "nfhook", + .subsys_id = NFNL_SUBSYS_HOOK, + .cb_count = NFNL_MSG_HOOK_MAX, + .cb = nfnl_hook_cb, +}; + +MODULE_ALIAS_NFNL_SUBSYS(NFNL_SUBSYS_HOOK); + +static int __init nfnetlink_hook_init(void) +{ + return nfnetlink_subsys_register(&nfhook_subsys); +} + +static void __exit nfnetlink_hook_exit(void) +{ + nfnetlink_subsys_unregister(&nfhook_subsys); +} + +module_init(nfnetlink_hook_init); +module_exit(nfnetlink_hook_exit); + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Florian Westphal "); +MODULE_DESCRIPTION("nfnetlink_hook: list registered netfilter hooks"); -- cgit v1.2.3 From d409989b59ad0b8d108706db25e17c320a9664eb Mon Sep 17 00:00:00 2001 From: Chen Li Date: Mon, 7 Jun 2021 09:44:35 +0800 Subject: netlink: simplify NLMSG_DATA with NLMSG_HDRLEN The NLMSG_LENGTH(0) may confuse the API users, NLMSG_HDRLEN is much more clear. Besides, some code style problems are also fixed. Signed-off-by: Chen Li Signed-off-by: David S. Miller --- include/uapi/linux/netlink.h | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'include/uapi') diff --git a/include/uapi/linux/netlink.h b/include/uapi/linux/netlink.h index 3d94269bbfa8..4c0cde075c27 100644 --- a/include/uapi/linux/netlink.h +++ b/include/uapi/linux/netlink.h @@ -91,9 +91,10 @@ struct nlmsghdr { #define NLMSG_HDRLEN ((int) NLMSG_ALIGN(sizeof(struct nlmsghdr))) #define NLMSG_LENGTH(len) ((len) + NLMSG_HDRLEN) #define NLMSG_SPACE(len) NLMSG_ALIGN(NLMSG_LENGTH(len)) -#define NLMSG_DATA(nlh) ((void*)(((char*)nlh) + NLMSG_LENGTH(0))) +#define NLMSG_DATA(nlh) ((void *)(((char *)nlh) + NLMSG_HDRLEN)) #define NLMSG_NEXT(nlh,len) ((len) -= NLMSG_ALIGN((nlh)->nlmsg_len), \ - (struct nlmsghdr*)(((char*)(nlh)) + NLMSG_ALIGN((nlh)->nlmsg_len))) + (struct nlmsghdr *)(((char *)(nlh)) + \ + NLMSG_ALIGN((nlh)->nlmsg_len))) #define NLMSG_OK(nlh,len) ((len) >= (int)sizeof(struct nlmsghdr) && \ (nlh)->nlmsg_len >= sizeof(struct nlmsghdr) && \ (nlh)->nlmsg_len <= (len)) -- cgit v1.2.3 From 992da01aa932b432ef8dc3885fa76415b5dbe43f Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Thu, 10 Jun 2021 16:37:37 +0100 Subject: io_uring: change registration/upd/rsrc tagging ABI There are ABI moments about recently added rsrc registration/update and tagging that might become a nuisance in the future. First, IORING_REGISTER_RSRC[_UPD] hide different types of resources under it, so breaks fine control over them by restrictions. It works for now, but once those are wanted under restrictions it would require a rework. It was also inconvenient trying to fit a new resource not supporting all the features (e.g. dynamic update) into the interface, so better to return to IORING_REGISTER_* top level dispatching. Second, register/update were considered to accept a type of resource, however that's not a good idea because there might be several ways of registration of a single resource type, e.g. we may want to add non-contig buffers or anything more exquisite as dma mapped memory. So, remove IORING_RSRC_[FILE,BUFFER] out of the ABI, and place them internally for now to limit changes. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/9b554897a7c17ad6e3becc48dfed2f7af9f423d5.1623339162.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 39 +++++++++++++++++++++++++++------------ include/uapi/linux/io_uring.h | 18 +++++++++--------- 2 files changed, 36 insertions(+), 21 deletions(-) (limited to 'include/uapi') diff --git a/fs/io_uring.c b/fs/io_uring.c index 42380ed563c4..663fef3d56df 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -783,6 +783,11 @@ struct io_task_work { task_work_func_t func; }; +enum { + IORING_RSRC_FILE = 0, + IORING_RSRC_BUFFER = 1, +}; + /* * NOTE! Each of the iocb union members has the file pointer * as the first entry in their struct definition. So you can @@ -9911,7 +9916,7 @@ static int io_register_files_update(struct io_ring_ctx *ctx, void __user *arg, } static int io_register_rsrc_update(struct io_ring_ctx *ctx, void __user *arg, - unsigned size) + unsigned size, unsigned type) { struct io_uring_rsrc_update2 up; @@ -9919,13 +9924,13 @@ static int io_register_rsrc_update(struct io_ring_ctx *ctx, void __user *arg, return -EINVAL; if (copy_from_user(&up, arg, sizeof(up))) return -EFAULT; - if (!up.nr) + if (!up.nr || up.resv) return -EINVAL; - return __io_register_rsrc_update(ctx, up.type, &up, up.nr); + return __io_register_rsrc_update(ctx, type, &up, up.nr); } static int io_register_rsrc(struct io_ring_ctx *ctx, void __user *arg, - unsigned int size) + unsigned int size, unsigned int type) { struct io_uring_rsrc_register rr; @@ -9936,10 +9941,10 @@ static int io_register_rsrc(struct io_ring_ctx *ctx, void __user *arg, memset(&rr, 0, sizeof(rr)); if (copy_from_user(&rr, arg, size)) return -EFAULT; - if (!rr.nr) + if (!rr.nr || rr.resv || rr.resv2) return -EINVAL; - switch (rr.type) { + switch (type) { case IORING_RSRC_FILE: return io_sqe_files_register(ctx, u64_to_user_ptr(rr.data), rr.nr, u64_to_user_ptr(rr.tags)); @@ -9961,8 +9966,10 @@ static bool io_register_op_must_quiesce(int op) case IORING_REGISTER_PROBE: case IORING_REGISTER_PERSONALITY: case IORING_UNREGISTER_PERSONALITY: - case IORING_REGISTER_RSRC: - case IORING_REGISTER_RSRC_UPDATE: + case IORING_REGISTER_FILES2: + case IORING_REGISTER_FILES_UPDATE2: + case IORING_REGISTER_BUFFERS2: + case IORING_REGISTER_BUFFERS_UPDATE: return false; default: return true; @@ -10088,11 +10095,19 @@ static int __io_uring_register(struct io_ring_ctx *ctx, unsigned opcode, case IORING_REGISTER_RESTRICTIONS: ret = io_register_restrictions(ctx, arg, nr_args); break; - case IORING_REGISTER_RSRC: - ret = io_register_rsrc(ctx, arg, nr_args); + case IORING_REGISTER_FILES2: + ret = io_register_rsrc(ctx, arg, nr_args, IORING_RSRC_FILE); + break; + case IORING_REGISTER_FILES_UPDATE2: + ret = io_register_rsrc_update(ctx, arg, nr_args, + IORING_RSRC_FILE); + break; + case IORING_REGISTER_BUFFERS2: + ret = io_register_rsrc(ctx, arg, nr_args, IORING_RSRC_BUFFER); break; - case IORING_REGISTER_RSRC_UPDATE: - ret = io_register_rsrc_update(ctx, arg, nr_args); + case IORING_REGISTER_BUFFERS_UPDATE: + ret = io_register_rsrc_update(ctx, arg, nr_args, + IORING_RSRC_BUFFER); break; default: ret = -EINVAL; diff --git a/include/uapi/linux/io_uring.h b/include/uapi/linux/io_uring.h index e1ae46683301..48b4ddcd56ff 100644 --- a/include/uapi/linux/io_uring.h +++ b/include/uapi/linux/io_uring.h @@ -298,8 +298,12 @@ enum { IORING_UNREGISTER_PERSONALITY = 10, IORING_REGISTER_RESTRICTIONS = 11, IORING_REGISTER_ENABLE_RINGS = 12, - IORING_REGISTER_RSRC = 13, - IORING_REGISTER_RSRC_UPDATE = 14, + + /* extended with tagging */ + IORING_REGISTER_FILES2 = 13, + IORING_REGISTER_FILES_UPDATE2 = 14, + IORING_REGISTER_BUFFERS2 = 15, + IORING_REGISTER_BUFFERS_UPDATE = 16, /* this goes last */ IORING_REGISTER_LAST @@ -312,14 +316,10 @@ struct io_uring_files_update { __aligned_u64 /* __s32 * */ fds; }; -enum { - IORING_RSRC_FILE = 0, - IORING_RSRC_BUFFER = 1, -}; - struct io_uring_rsrc_register { - __u32 type; __u32 nr; + __u32 resv; + __u64 resv2; __aligned_u64 data; __aligned_u64 tags; }; @@ -335,8 +335,8 @@ struct io_uring_rsrc_update2 { __u32 resv; __aligned_u64 data; __aligned_u64 tags; - __u32 type; __u32 nr; + __u32 resv2; }; /* Skip updating fd indexes set to this value in the fd table */ -- cgit v1.2.3 From 9690557e22d63f13534fd167d293ac8ed8b104f9 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Thu, 10 Jun 2021 16:37:38 +0100 Subject: io_uring: add feature flag for rsrc tags Add IORING_FEAT_RSRC_TAGS indicating that io_uring supports a bunch of new IORING_REGISTER operations, in particular IORING_REGISTER_[FILES[,UPDATE]2,BUFFERS[2,UPDATE]] that support rsrc tagging, and also indicating implemented dynamic fixed buffer updates. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/9b995d4045b6c6b4ab7510ca124fd25ac2203af7.1623339162.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 3 ++- include/uapi/linux/io_uring.h | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) (limited to 'include/uapi') diff --git a/fs/io_uring.c b/fs/io_uring.c index 663fef3d56df..fa8794c61af7 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -9676,7 +9676,8 @@ static int io_uring_create(unsigned entries, struct io_uring_params *p, IORING_FEAT_SUBMIT_STABLE | IORING_FEAT_RW_CUR_POS | IORING_FEAT_CUR_PERSONALITY | IORING_FEAT_FAST_POLL | IORING_FEAT_POLL_32BITS | IORING_FEAT_SQPOLL_NONFIXED | - IORING_FEAT_EXT_ARG | IORING_FEAT_NATIVE_WORKERS; + IORING_FEAT_EXT_ARG | IORING_FEAT_NATIVE_WORKERS | + IORING_FEAT_RSRC_TAGS; if (copy_to_user(params, p, sizeof(*p))) { ret = -EFAULT; diff --git a/include/uapi/linux/io_uring.h b/include/uapi/linux/io_uring.h index 48b4ddcd56ff..162ff99ed2cb 100644 --- a/include/uapi/linux/io_uring.h +++ b/include/uapi/linux/io_uring.h @@ -280,6 +280,7 @@ struct io_uring_params { #define IORING_FEAT_SQPOLL_NONFIXED (1U << 7) #define IORING_FEAT_EXT_ARG (1U << 8) #define IORING_FEAT_NATIVE_WORKERS (1U << 9) +#define IORING_FEAT_RSRC_TAGS (1U << 10) /* * io_uring_register(2) opcodes and arguments -- cgit v1.2.3 From 6ddb5680085a3eefe0c6267e3514060045a13c95 Mon Sep 17 00:00:00 2001 From: Zhen Lei Date: Wed, 9 Jun 2021 10:27:01 +0800 Subject: audit: remove trailing spaces and tabs Run the following command to find and remove the trailing spaces and tabs: sed -r -i 's/[ \t]+$//' The files to be checked are as follows: kernel/audit* include/linux/audit.h include/uapi/linux/audit.h Signed-off-by: Zhen Lei Acked-by: Richard Guy Briggs Signed-off-by: Paul Moore --- include/uapi/linux/audit.h | 4 ++-- kernel/audit.h | 2 +- kernel/auditsc.c | 8 ++++---- 3 files changed, 7 insertions(+), 7 deletions(-) (limited to 'include/uapi') diff --git a/include/uapi/linux/audit.h b/include/uapi/linux/audit.h index cd2d8279a5e4..daa481729e9b 100644 --- a/include/uapi/linux/audit.h +++ b/include/uapi/linux/audit.h @@ -48,7 +48,7 @@ * 2500 - 2999 future user space (maybe integrity labels and related events) * * Messages from 1000-1199 are bi-directional. 1200-1299 & 2100 - 2999 are - * exclusively user space. 1300-2099 is kernel --> user space + * exclusively user space. 1300-2099 is kernel --> user space * communication. */ #define AUDIT_GET 1000 /* Get status */ @@ -78,7 +78,7 @@ #define AUDIT_LAST_USER_MSG 1199 #define AUDIT_FIRST_USER_MSG2 2100 /* More user space messages */ #define AUDIT_LAST_USER_MSG2 2999 - + #define AUDIT_DAEMON_START 1200 /* Daemon startup record */ #define AUDIT_DAEMON_END 1201 /* Daemon normal stop record */ #define AUDIT_DAEMON_ABORT 1202 /* Daemon error stop record */ diff --git a/kernel/audit.h b/kernel/audit.h index e518ad9374fc..b565ea16c0a5 100644 --- a/kernel/audit.h +++ b/kernel/audit.h @@ -1,5 +1,5 @@ /* SPDX-License-Identifier: GPL-2.0-or-later */ -/* audit -- definition of audit_context structure and supporting types +/* audit -- definition of audit_context structure and supporting types * * Copyright 2003-2004 Red Hat, Inc. * Copyright 2005 Hewlett-Packard Development Company, L.P. diff --git a/kernel/auditsc.c b/kernel/auditsc.c index 123f9dc12665..8dd73a64f921 100644 --- a/kernel/auditsc.c +++ b/kernel/auditsc.c @@ -343,13 +343,13 @@ static int audit_compare_uid(kuid_t uid, { struct audit_names *n; int rc; - + if (name) { rc = audit_uid_comparator(uid, f->op, name->uid); if (rc) return rc; } - + if (ctx) { list_for_each_entry(n, &ctx->names_list, list) { rc = audit_uid_comparator(uid, f->op, n->uid); @@ -367,13 +367,13 @@ static int audit_compare_gid(kgid_t gid, { struct audit_names *n; int rc; - + if (name) { rc = audit_gid_comparator(gid, f->op, name->gid); if (rc) return rc; } - + if (ctx) { list_for_each_entry(n, &ctx->names_list, list) { rc = audit_gid_comparator(gid, f->op, n->gid); -- cgit v1.2.3 From f07b2a5b04d4a50d931a0afe4e3e114ce09a2e4b Mon Sep 17 00:00:00 2001 From: Arseny Krasnov Date: Fri, 11 Jun 2021 14:12:22 +0300 Subject: virtio/vsock: defines and constants for SEQPACKET Add set of defines and constants for SOCK_SEQPACKET support in vsock. Signed-off-by: Arseny Krasnov Reviewed-by: Stefano Garzarella Signed-off-by: David S. Miller --- include/uapi/linux/virtio_vsock.h | 9 +++++++++ 1 file changed, 9 insertions(+) (limited to 'include/uapi') diff --git a/include/uapi/linux/virtio_vsock.h b/include/uapi/linux/virtio_vsock.h index 1d57ed3d84d2..3dd3555b2740 100644 --- a/include/uapi/linux/virtio_vsock.h +++ b/include/uapi/linux/virtio_vsock.h @@ -38,6 +38,9 @@ #include #include +/* The feature bitmap for virtio vsock */ +#define VIRTIO_VSOCK_F_SEQPACKET 1 /* SOCK_SEQPACKET supported */ + struct virtio_vsock_config { __le64 guest_cid; } __attribute__((packed)); @@ -65,6 +68,7 @@ struct virtio_vsock_hdr { enum virtio_vsock_type { VIRTIO_VSOCK_TYPE_STREAM = 1, + VIRTIO_VSOCK_TYPE_SEQPACKET = 2, }; enum virtio_vsock_op { @@ -91,4 +95,9 @@ enum virtio_vsock_shutdown { VIRTIO_VSOCK_SHUTDOWN_SEND = 2, }; +/* VIRTIO_VSOCK_OP_RW flags values */ +enum virtio_vsock_rw { + VIRTIO_VSOCK_SEQ_EOR = 1, +}; + #endif /* _UAPI_LINUX_VIRTIO_VSOCK_H */ -- cgit v1.2.3 From 00e77ed8e64d5f271c1f015c7153545980d48a76 Mon Sep 17 00:00:00 2001 From: Johannes Berg Date: Sat, 12 Jun 2021 10:20:55 +0200 Subject: rtnetlink: add IFLA_PARENT_[DEV|DEV_BUS]_NAME In some cases, for example in the upcoming WWAN framework changes, there's no natural "parent netdev", so sometimes dummy netdevs are created or similar. IFLA_PARENT_DEV_NAME is a new attribute intended to contain a device (sysfs, struct device) name that can be used instead when creating a new netdev, if the rtnetlink family implements it. As suggested by Parav Pandit, we also introduce IFLA_PARENT_DEV_BUS_NAME attribute in order to uniquely identify a device on the system (with bus/name pair). ip-link(8) support for the generic parent device attributes will help us avoid code duplication, so no other link type will require a custom code to handle the parent name attribute. E.g. the WWAN interface creation command will looks like this: $ ip link add wwan0-1 parent-dev wwan0 type wwan channel-id 1 So, some future subsystem (or driver) FOO will have an interface creation command that looks like this: $ ip link add foo1-3 parent-dev foo1 type foo bar-id 3 baz-type Y Below is an example of dumping link info of a random device with these new attributes: $ ip --details link show wlp0s20f3 4: wlp0s20f3: mtu 1500 qdisc noqueue state UP mode DORMANT group default qlen 1000 ... parent_bus pci parent_dev 0000:00:14.3 Co-developed-by: Sergey Ryazanov Signed-off-by: Sergey Ryazanov Co-developed-by: Loic Poulain Signed-off-by: Loic Poulain Suggested-by: Sergey Ryazanov Signed-off-by: Johannes Berg Signed-off-by: David S. Miller --- include/uapi/linux/if_link.h | 7 +++++++ net/core/rtnetlink.c | 10 ++++++++++ 2 files changed, 17 insertions(+) (limited to 'include/uapi') diff --git a/include/uapi/linux/if_link.h b/include/uapi/linux/if_link.h index a5a7f0e64865..4882e81514b6 100644 --- a/include/uapi/linux/if_link.h +++ b/include/uapi/linux/if_link.h @@ -341,6 +341,13 @@ enum { IFLA_ALT_IFNAME, /* Alternative ifname */ IFLA_PERM_ADDRESS, IFLA_PROTO_DOWN_REASON, + + /* device (sysfs) name as parent, used instead + * of IFLA_LINK where there's no parent netdev + */ + IFLA_PARENT_DEV_NAME, + IFLA_PARENT_DEV_BUS_NAME, + __IFLA_MAX }; diff --git a/net/core/rtnetlink.c b/net/core/rtnetlink.c index 92c3e43db812..170e97f3b3c6 100644 --- a/net/core/rtnetlink.c +++ b/net/core/rtnetlink.c @@ -1821,6 +1821,16 @@ static int rtnl_fill_ifinfo(struct sk_buff *skb, if (rtnl_fill_prop_list(skb, dev)) goto nla_put_failure; + if (dev->dev.parent && + nla_put_string(skb, IFLA_PARENT_DEV_NAME, + dev_name(dev->dev.parent))) + goto nla_put_failure; + + if (dev->dev.parent && dev->dev.parent->bus && + nla_put_string(skb, IFLA_PARENT_DEV_BUS_NAME, + dev->dev.parent->bus->name)) + goto nla_put_failure; + nlmsg_end(skb, nlh); return 0; -- cgit v1.2.3 From 88b710532e53de2466d1033fb1d5125aabf3215a Mon Sep 17 00:00:00 2001 From: Johannes Berg Date: Sat, 12 Jun 2021 10:20:56 +0200 Subject: wwan: add interface creation support Add support to create (and destroy) interfaces via a new rtnetlink kind "wwan". The responsible driver has to use the new wwan_register_ops() to make this possible. Signed-off-by: Johannes Berg Signed-off-by: Sergey Ryazanov Signed-off-by: Loic Poulain Signed-off-by: David S. Miller --- drivers/net/wwan/wwan_core.c | 245 +++++++++++++++++++++++++++++++++++++++++-- include/linux/wwan.h | 24 +++++ include/uapi/linux/wwan.h | 16 +++ net/core/rtnetlink.c | 1 + 4 files changed, 279 insertions(+), 7 deletions(-) create mode 100644 include/uapi/linux/wwan.h (limited to 'include/uapi') diff --git a/drivers/net/wwan/wwan_core.c b/drivers/net/wwan/wwan_core.c index 45a41aee8958..7e728042fc41 100644 --- a/drivers/net/wwan/wwan_core.c +++ b/drivers/net/wwan/wwan_core.c @@ -14,6 +14,8 @@ #include #include #include +#include +#include /* Maximum number of minors in use */ #define WWAN_MAX_MINORS (1 << MINORBITS) @@ -35,10 +37,16 @@ static int wwan_major; * * @id: WWAN device unique ID. * @dev: Underlying device. + * @port_id: Current available port ID to pick. + * @ops: wwan device ops + * @ops_ctxt: context to pass to ops */ struct wwan_device { unsigned int id; struct device dev; + atomic_t port_id; + const struct wwan_ops *ops; + void *ops_ctxt; }; /** @@ -102,7 +110,8 @@ static const struct device_type wwan_dev_type = { static int wwan_dev_parent_match(struct device *dev, const void *parent) { - return (dev->type == &wwan_dev_type && dev->parent == parent); + return (dev->type == &wwan_dev_type && + (dev->parent == parent || dev == parent)); } static struct wwan_device *wwan_dev_get_by_parent(struct device *parent) @@ -116,6 +125,23 @@ static struct wwan_device *wwan_dev_get_by_parent(struct device *parent) return to_wwan_dev(dev); } +static int wwan_dev_name_match(struct device *dev, const void *name) +{ + return dev->type == &wwan_dev_type && + strcmp(dev_name(dev), name) == 0; +} + +static struct wwan_device *wwan_dev_get_by_name(const char *name) +{ + struct device *dev; + + dev = class_find_device(wwan_class, NULL, name, wwan_dev_name_match); + if (!dev) + return ERR_PTR(-ENODEV); + + return to_wwan_dev(dev); +} + /* This function allocates and registers a new WWAN device OR if a WWAN device * already exist for the given parent, it gets a reference and return it. * This function is not exported (for now), it is called indirectly via @@ -180,9 +206,14 @@ static void wwan_remove_dev(struct wwan_device *wwandev) /* WWAN device is created and registered (get+add) along with its first * child port, and subsequent port registrations only grab a reference * (get). The WWAN device must then be unregistered (del+put) along with - * its latest port, and reference simply dropped (put) otherwise. + * its last port, and reference simply dropped (put) otherwise. In the + * same fashion, we must not unregister it when the ops are still there. */ - ret = device_for_each_child(&wwandev->dev, NULL, is_wwan_child); + if (wwandev->ops) + ret = 1; + else + ret = device_for_each_child(&wwandev->dev, NULL, is_wwan_child); + if (!ret) device_unregister(&wwandev->dev); else @@ -750,26 +781,226 @@ static const struct file_operations wwan_port_fops = { .llseek = noop_llseek, }; +/** + * wwan_register_ops - register WWAN device ops + * @parent: Device to use as parent and shared by all WWAN ports and + * created netdevs + * @ops: operations to register + * @ctxt: context to pass to operations + * + * Returns: 0 on success, a negative error code on failure + */ +int wwan_register_ops(struct device *parent, const struct wwan_ops *ops, + void *ctxt) +{ + struct wwan_device *wwandev; + + if (WARN_ON(!parent || !ops)) + return -EINVAL; + + wwandev = wwan_create_dev(parent); + if (!wwandev) + return -ENOMEM; + + if (WARN_ON(wwandev->ops)) { + wwan_remove_dev(wwandev); + return -EBUSY; + } + + if (!try_module_get(ops->owner)) { + wwan_remove_dev(wwandev); + return -ENODEV; + } + + wwandev->ops = ops; + wwandev->ops_ctxt = ctxt; + + return 0; +} +EXPORT_SYMBOL_GPL(wwan_register_ops); + +/** + * wwan_unregister_ops - remove WWAN device ops + * @parent: Device to use as parent and shared by all WWAN ports and + * created netdevs + */ +void wwan_unregister_ops(struct device *parent) +{ + struct wwan_device *wwandev = wwan_dev_get_by_parent(parent); + bool has_ops; + + if (WARN_ON(IS_ERR(wwandev))) + return; + + has_ops = wwandev->ops; + + /* put the reference obtained by wwan_dev_get_by_parent(), + * we should still have one (that the owner is giving back + * now) due to the ops being assigned, check that below + * and return if not. + */ + put_device(&wwandev->dev); + + if (WARN_ON(!has_ops)) + return; + + module_put(wwandev->ops->owner); + + wwandev->ops = NULL; + wwandev->ops_ctxt = NULL; + wwan_remove_dev(wwandev); +} +EXPORT_SYMBOL_GPL(wwan_unregister_ops); + +static int wwan_rtnl_validate(struct nlattr *tb[], struct nlattr *data[], + struct netlink_ext_ack *extack) +{ + if (!data) + return -EINVAL; + + if (!tb[IFLA_PARENT_DEV_NAME]) + return -EINVAL; + + if (!data[IFLA_WWAN_LINK_ID]) + return -EINVAL; + + return 0; +} + +static struct device_type wwan_type = { .name = "wwan" }; + +static struct net_device *wwan_rtnl_alloc(struct nlattr *tb[], + const char *ifname, + unsigned char name_assign_type, + unsigned int num_tx_queues, + unsigned int num_rx_queues) +{ + const char *devname = nla_data(tb[IFLA_PARENT_DEV_NAME]); + struct wwan_device *wwandev = wwan_dev_get_by_name(devname); + struct net_device *dev; + + if (IS_ERR(wwandev)) + return ERR_CAST(wwandev); + + /* only supported if ops were registered (not just ports) */ + if (!wwandev->ops) { + dev = ERR_PTR(-EOPNOTSUPP); + goto out; + } + + dev = alloc_netdev_mqs(wwandev->ops->priv_size, ifname, name_assign_type, + wwandev->ops->setup, num_tx_queues, num_rx_queues); + + if (dev) { + SET_NETDEV_DEV(dev, &wwandev->dev); + SET_NETDEV_DEVTYPE(dev, &wwan_type); + } + +out: + /* release the reference */ + put_device(&wwandev->dev); + return dev; +} + +static int wwan_rtnl_newlink(struct net *src_net, struct net_device *dev, + struct nlattr *tb[], struct nlattr *data[], + struct netlink_ext_ack *extack) +{ + struct wwan_device *wwandev = wwan_dev_get_by_parent(dev->dev.parent); + u32 link_id = nla_get_u32(data[IFLA_WWAN_LINK_ID]); + int ret; + + if (IS_ERR(wwandev)) + return PTR_ERR(wwandev); + + /* shouldn't have a netdev (left) with us as parent so WARN */ + if (WARN_ON(!wwandev->ops)) { + ret = -EOPNOTSUPP; + goto out; + } + + if (wwandev->ops->newlink) + ret = wwandev->ops->newlink(wwandev->ops_ctxt, dev, + link_id, extack); + else + ret = register_netdevice(dev); + +out: + /* release the reference */ + put_device(&wwandev->dev); + return ret; +} + +static void wwan_rtnl_dellink(struct net_device *dev, struct list_head *head) +{ + struct wwan_device *wwandev = wwan_dev_get_by_parent(dev->dev.parent); + + if (IS_ERR(wwandev)) + return; + + /* shouldn't have a netdev (left) with us as parent so WARN */ + if (WARN_ON(!wwandev->ops)) + goto out; + + if (wwandev->ops->dellink) + wwandev->ops->dellink(wwandev->ops_ctxt, dev, head); + else + unregister_netdevice(dev); + +out: + /* release the reference */ + put_device(&wwandev->dev); +} + +static const struct nla_policy wwan_rtnl_policy[IFLA_WWAN_MAX + 1] = { + [IFLA_WWAN_LINK_ID] = { .type = NLA_U32 }, +}; + +static struct rtnl_link_ops wwan_rtnl_link_ops __read_mostly = { + .kind = "wwan", + .maxtype = __IFLA_WWAN_MAX, + .alloc = wwan_rtnl_alloc, + .validate = wwan_rtnl_validate, + .newlink = wwan_rtnl_newlink, + .dellink = wwan_rtnl_dellink, + .policy = wwan_rtnl_policy, +}; + static int __init wwan_init(void) { + int err; + + err = rtnl_link_register(&wwan_rtnl_link_ops); + if (err) + return err; + wwan_class = class_create(THIS_MODULE, "wwan"); - if (IS_ERR(wwan_class)) - return PTR_ERR(wwan_class); + if (IS_ERR(wwan_class)) { + err = PTR_ERR(wwan_class); + goto unregister; + } /* chrdev used for wwan ports */ wwan_major = __register_chrdev(0, 0, WWAN_MAX_MINORS, "wwan_port", &wwan_port_fops); if (wwan_major < 0) { - class_destroy(wwan_class); - return wwan_major; + err = wwan_major; + goto destroy; } return 0; + +destroy: + class_destroy(wwan_class); +unregister: + rtnl_link_unregister(&wwan_rtnl_link_ops); + return err; } static void __exit wwan_exit(void) { __unregister_chrdev(wwan_major, 0, WWAN_MAX_MINORS, "wwan_port"); + rtnl_link_unregister(&wwan_rtnl_link_ops); class_destroy(wwan_class); } diff --git a/include/linux/wwan.h b/include/linux/wwan.h index fa33cc16d931..430a3a0817de 100644 --- a/include/linux/wwan.h +++ b/include/linux/wwan.h @@ -7,6 +7,7 @@ #include #include #include +#include /** * enum wwan_port_type - WWAN port types @@ -116,4 +117,27 @@ void wwan_port_txon(struct wwan_port *port); */ void *wwan_port_get_drvdata(struct wwan_port *port); +/** + * struct wwan_ops - WWAN device ops + * @owner: module owner of the WWAN ops + * @priv_size: size of private netdev data area + * @setup: set up a new netdev + * @newlink: register the new netdev + * @dellink: remove the given netdev + */ +struct wwan_ops { + struct module *owner; + unsigned int priv_size; + void (*setup)(struct net_device *dev); + int (*newlink)(void *ctxt, struct net_device *dev, + u32 if_id, struct netlink_ext_ack *extack); + void (*dellink)(void *ctxt, struct net_device *dev, + struct list_head *head); +}; + +int wwan_register_ops(struct device *parent, const struct wwan_ops *ops, + void *ctxt); + +void wwan_unregister_ops(struct device *parent); + #endif /* __WWAN_H */ diff --git a/include/uapi/linux/wwan.h b/include/uapi/linux/wwan.h new file mode 100644 index 000000000000..32a2720b4d11 --- /dev/null +++ b/include/uapi/linux/wwan.h @@ -0,0 +1,16 @@ +/* SPDX-License-Identifier: GPL-2.0-only WITH Linux-syscall-note */ +/* + * Copyright (C) 2021 Intel Corporation. + */ +#ifndef _UAPI_WWAN_H_ +#define _UAPI_WWAN_H_ + +enum { + IFLA_WWAN_UNSPEC, + IFLA_WWAN_LINK_ID, /* u32 */ + + __IFLA_WWAN_MAX +}; +#define IFLA_WWAN_MAX (__IFLA_WWAN_MAX - 1) + +#endif /* _UAPI_WWAN_H_ */ diff --git a/net/core/rtnetlink.c b/net/core/rtnetlink.c index 170e97f3b3c6..5baa86bca876 100644 --- a/net/core/rtnetlink.c +++ b/net/core/rtnetlink.c @@ -1890,6 +1890,7 @@ static const struct nla_policy ifla_policy[IFLA_MAX+1] = { [IFLA_PERM_ADDRESS] = { .type = NLA_REJECT }, [IFLA_PROTO_DOWN_REASON] = { .type = NLA_NESTED }, [IFLA_NEW_IFINDEX] = NLA_POLICY_MIN(NLA_S32, 1), + [IFLA_PARENT_DEV_NAME] = { .type = NLA_NUL_STRING }, }; static const struct nla_policy ifla_info_policy[IFLA_INFO_MAX+1] = { -- cgit v1.2.3 From 87815ee9d0060a91bdf18266e42837a9adb5972e Mon Sep 17 00:00:00 2001 From: Ben Widawsky Date: Tue, 13 Apr 2021 07:09:07 -0700 Subject: cxl/pci: Add media provisioning required commands Some of the commands have already been defined for the support of RAW commands (to be blocked). Unlike their usage in the RAW interface, when used through the supported interface, they will be coordinated and marshalled along with other commands being issued by userspace and the driver itself. That coordination will be added later. The list of commands was determined based on the learnings from libnvdimm and this list is provided directly from Dan. Recommended-by: Dan Williams Signed-off-by: Ben Widawsky Reviewed-by: Jonathan Cameron Link: https://lore.kernel.org/r/20210413140907.534404-1-ben.widawsky@intel.com Signed-off-by: Dan Williams --- drivers/cxl/pci.c | 19 +++++++++++++++++++ include/uapi/linux/cxl_mem.h | 12 ++++++++++++ 2 files changed, 31 insertions(+) (limited to 'include/uapi') diff --git a/drivers/cxl/pci.c b/drivers/cxl/pci.c index 5a1705b52278..a8f29ce35c2a 100644 --- a/drivers/cxl/pci.c +++ b/drivers/cxl/pci.c @@ -51,7 +51,14 @@ enum opcode { CXL_MBOX_OP_GET_LSA = 0x4102, CXL_MBOX_OP_SET_LSA = 0x4103, CXL_MBOX_OP_GET_HEALTH_INFO = 0x4200, + CXL_MBOX_OP_GET_ALERT_CONFIG = 0x4201, + CXL_MBOX_OP_SET_ALERT_CONFIG = 0x4202, + CXL_MBOX_OP_GET_SHUTDOWN_STATE = 0x4203, CXL_MBOX_OP_SET_SHUTDOWN_STATE = 0x4204, + CXL_MBOX_OP_GET_POISON = 0x4300, + CXL_MBOX_OP_INJECT_POISON = 0x4301, + CXL_MBOX_OP_CLEAR_POISON = 0x4302, + CXL_MBOX_OP_GET_SCAN_MEDIA_CAPS = 0x4303, CXL_MBOX_OP_SCAN_MEDIA = 0x4304, CXL_MBOX_OP_GET_SCAN_MEDIA = 0x4305, CXL_MBOX_OP_MAX = 0x10000 @@ -159,6 +166,18 @@ static struct cxl_mem_command mem_commands[CXL_MEM_COMMAND_ID_MAX] = { CXL_CMD(GET_LSA, 0x8, ~0, 0), CXL_CMD(GET_HEALTH_INFO, 0, 0x12, 0), CXL_CMD(GET_LOG, 0x18, ~0, CXL_CMD_FLAG_FORCE_ENABLE), + CXL_CMD(SET_PARTITION_INFO, 0x0a, 0, 0), + CXL_CMD(SET_LSA, ~0, 0, 0), + CXL_CMD(GET_ALERT_CONFIG, 0, 0x10, 0), + CXL_CMD(SET_ALERT_CONFIG, 0xc, 0, 0), + CXL_CMD(GET_SHUTDOWN_STATE, 0, 0x1, 0), + CXL_CMD(SET_SHUTDOWN_STATE, 0x1, 0, 0), + CXL_CMD(GET_POISON, 0x10, ~0, 0), + CXL_CMD(INJECT_POISON, 0x8, 0, 0), + CXL_CMD(CLEAR_POISON, 0x48, 0, 0), + CXL_CMD(GET_SCAN_MEDIA_CAPS, 0x10, 0x4, 0), + CXL_CMD(SCAN_MEDIA, 0x11, 0, 0), + CXL_CMD(GET_SCAN_MEDIA, 0, ~0, 0), }; /* diff --git a/include/uapi/linux/cxl_mem.h b/include/uapi/linux/cxl_mem.h index 3155382dfc9b..f6e8a005b113 100644 --- a/include/uapi/linux/cxl_mem.h +++ b/include/uapi/linux/cxl_mem.h @@ -29,6 +29,18 @@ ___C(GET_LSA, "Get Label Storage Area"), \ ___C(GET_HEALTH_INFO, "Get Health Info"), \ ___C(GET_LOG, "Get Log"), \ + ___C(SET_PARTITION_INFO, "Set Partition Information"), \ + ___C(SET_LSA, "Set Label Storage Area"), \ + ___C(GET_ALERT_CONFIG, "Get Alert Configuration"), \ + ___C(SET_ALERT_CONFIG, "Set Alert Configuration"), \ + ___C(GET_SHUTDOWN_STATE, "Get Shutdown State"), \ + ___C(SET_SHUTDOWN_STATE, "Set Shutdown State"), \ + ___C(GET_POISON, "Get Poison List"), \ + ___C(INJECT_POISON, "Inject Poison"), \ + ___C(CLEAR_POISON, "Clear Poison"), \ + ___C(GET_SCAN_MEDIA_CAPS, "Get Scan Media Capabilities"), \ + ___C(SCAN_MEDIA, "Scan Media"), \ + ___C(GET_SCAN_MEDIA, "Get Scan Media Results"), \ ___C(MAX, "invalid / last command") #define ___C(a, b) CXL_MEM_COMMAND_ID_##a -- cgit v1.2.3 From 8b1462b67f23da548f27b779a36b8ea75f5ef249 Mon Sep 17 00:00:00 2001 From: Marcin Juszkiewicz Date: Mon, 14 Jun 2021 17:37:12 +0200 Subject: quota: finish disable quotactl_path syscall In commit 5b9fedb31e47 ("quota: Disable quotactl_path syscall") Jan Kara disabled quotactl_path syscall on several architectures. This commit disables it on all architectures using unified list of system calls: - arm64 - arc - csky - h8300 - hexagon - nds32 - nios2 - openrisc - riscv (32/64) CC: Jan Kara CC: Christian Brauner CC: Sascha Hauer Link: https://lore.kernel.org/lkml/20210512153621.n5u43jsytbik4yze@wittgenstein Link: https://lore.kernel.org/r/20210614153712.313707-1-marcin@juszkiewicz.com.pl Fixes: 5b9fedb31e47 ("quota: Disable quotactl_path syscall") Acked-by: Christian Brauner Signed-off-by: Marcin Juszkiewicz Signed-off-by: Jan Kara --- include/uapi/asm-generic/unistd.h | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'include/uapi') diff --git a/include/uapi/asm-generic/unistd.h b/include/uapi/asm-generic/unistd.h index 6de5a7fc066b..d2a942086fcb 100644 --- a/include/uapi/asm-generic/unistd.h +++ b/include/uapi/asm-generic/unistd.h @@ -863,8 +863,7 @@ __SYSCALL(__NR_process_madvise, sys_process_madvise) __SC_COMP(__NR_epoll_pwait2, sys_epoll_pwait2, compat_sys_epoll_pwait2) #define __NR_mount_setattr 442 __SYSCALL(__NR_mount_setattr, sys_mount_setattr) -#define __NR_quotactl_path 443 -__SYSCALL(__NR_quotactl_path, sys_quotactl_path) +/* 443 is reserved for quotactl_path */ #define __NR_landlock_create_ruleset 444 __SYSCALL(__NR_landlock_create_ruleset, sys_landlock_create_ruleset) -- cgit v1.2.3 From e061047684af63f2d4f1338ec73140f6e29eb59f Mon Sep 17 00:00:00 2001 From: Kuniyuki Iwashima Date: Sat, 12 Jun 2021 21:32:21 +0900 Subject: bpf: Support BPF_FUNC_get_socket_cookie() for BPF_PROG_TYPE_SK_REUSEPORT. We will call sock_reuseport.prog for socket migration in the next commit, so the eBPF program has to know which listener is closing to select a new listener. We can currently get a unique ID of each listener in the userspace by calling bpf_map_lookup_elem() for BPF_MAP_TYPE_REUSEPORT_SOCKARRAY map. This patch makes the pointer of sk available in sk_reuseport_md so that we can get the ID by BPF_FUNC_get_socket_cookie() in the eBPF program. Suggested-by: Martin KaFai Lau Signed-off-by: Kuniyuki Iwashima Signed-off-by: Daniel Borkmann Reviewed-by: Eric Dumazet Acked-by: Martin KaFai Lau Link: https://lore.kernel.org/netdev/20201119001154.kapwihc2plp4f7zc@kafai-mbp.dhcp.thefacebook.com/ Link: https://lore.kernel.org/bpf/20210612123224.12525-9-kuniyu@amazon.co.jp --- include/uapi/linux/bpf.h | 1 + net/core/filter.c | 10 ++++++++++ tools/include/uapi/linux/bpf.h | 1 + 3 files changed, 12 insertions(+) (limited to 'include/uapi') diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h index 2c1ba70abbf1..f3b72588442b 100644 --- a/include/uapi/linux/bpf.h +++ b/include/uapi/linux/bpf.h @@ -5416,6 +5416,7 @@ struct sk_reuseport_md { __u32 ip_protocol; /* IP protocol. e.g. IPPROTO_TCP, IPPROTO_UDP */ __u32 bind_inany; /* Is sock bound to an INANY address? */ __u32 hash; /* A hash of the packet 4 tuples */ + __bpf_md_ptr(struct bpf_sock *, sk); }; #define BPF_TAG_SIZE 8 diff --git a/net/core/filter.c b/net/core/filter.c index caa88955562e..f753ab550525 100644 --- a/net/core/filter.c +++ b/net/core/filter.c @@ -10172,6 +10172,8 @@ sk_reuseport_func_proto(enum bpf_func_id func_id, return &sk_reuseport_load_bytes_proto; case BPF_FUNC_skb_load_bytes_relative: return &sk_reuseport_load_bytes_relative_proto; + case BPF_FUNC_get_socket_cookie: + return &bpf_get_socket_ptr_cookie_proto; default: return bpf_base_func_proto(func_id); } @@ -10201,6 +10203,10 @@ sk_reuseport_is_valid_access(int off, int size, case offsetof(struct sk_reuseport_md, hash): return size == size_default; + case offsetof(struct sk_reuseport_md, sk): + info->reg_type = PTR_TO_SOCKET; + return size == sizeof(__u64); + /* Fields that allow narrowing */ case bpf_ctx_range(struct sk_reuseport_md, eth_protocol): if (size < sizeof_field(struct sk_buff, protocol)) @@ -10273,6 +10279,10 @@ static u32 sk_reuseport_convert_ctx_access(enum bpf_access_type type, case offsetof(struct sk_reuseport_md, bind_inany): SK_REUSEPORT_LOAD_FIELD(bind_inany); break; + + case offsetof(struct sk_reuseport_md, sk): + SK_REUSEPORT_LOAD_FIELD(sk); + break; } return insn - insn_buf; diff --git a/tools/include/uapi/linux/bpf.h b/tools/include/uapi/linux/bpf.h index 2c1ba70abbf1..f3b72588442b 100644 --- a/tools/include/uapi/linux/bpf.h +++ b/tools/include/uapi/linux/bpf.h @@ -5416,6 +5416,7 @@ struct sk_reuseport_md { __u32 ip_protocol; /* IP protocol. e.g. IPPROTO_TCP, IPPROTO_UDP */ __u32 bind_inany; /* Is sock bound to an INANY address? */ __u32 hash; /* A hash of the packet 4 tuples */ + __bpf_md_ptr(struct bpf_sock *, sk); }; #define BPF_TAG_SIZE 8 -- cgit v1.2.3 From d5e4ddaeb6ab2c3c7fbb7b247a6d34bb0b18d87e Mon Sep 17 00:00:00 2001 From: Kuniyuki Iwashima Date: Sat, 12 Jun 2021 21:32:22 +0900 Subject: bpf: Support socket migration by eBPF. This patch introduces a new bpf_attach_type for BPF_PROG_TYPE_SK_REUSEPORT to check if the attached eBPF program is capable of migrating sockets. When the eBPF program is attached, we run it for socket migration if the expected_attach_type is BPF_SK_REUSEPORT_SELECT_OR_MIGRATE or net.ipv4.tcp_migrate_req is enabled. Currently, the expected_attach_type is not enforced for the BPF_PROG_TYPE_SK_REUSEPORT type of program. Thus, this commit follows the earlier idea in the commit aac3fc320d94 ("bpf: Post-hooks for sys_bind") to fix up the zero expected_attach_type in bpf_prog_load_fixup_attach_type(). Moreover, this patch adds a new field (migrating_sk) to sk_reuseport_md to select a new listener based on the child socket. migrating_sk varies depending on if it is migrating a request in the accept queue or during 3WHS. - accept_queue : sock (ESTABLISHED/SYN_RECV) - 3WHS : request_sock (NEW_SYN_RECV) In the eBPF program, we can select a new listener by BPF_FUNC_sk_select_reuseport(). Also, we can cancel migration by returning SK_DROP. This feature is useful when listeners have different settings at the socket API level or when we want to free resources as soon as possible. - SK_PASS with selected_sk, select it as a new listener - SK_PASS with selected_sk NULL, fallbacks to the random selection - SK_DROP, cancel the migration. There is a noteworthy point. We select a listening socket in three places, but we do not have struct skb at closing a listener or retransmitting a SYN+ACK. On the other hand, some helper functions do not expect skb is NULL (e.g. skb_header_pointer() in BPF_FUNC_skb_load_bytes(), skb_tail_pointer() in BPF_FUNC_skb_load_bytes_relative()). So we allocate an empty skb temporarily before running the eBPF program. Suggested-by: Martin KaFai Lau Signed-off-by: Kuniyuki Iwashima Signed-off-by: Daniel Borkmann Reviewed-by: Eric Dumazet Acked-by: Martin KaFai Lau Link: https://lore.kernel.org/netdev/20201123003828.xjpjdtk4ygl6tg6h@kafai-mbp.dhcp.thefacebook.com/ Link: https://lore.kernel.org/netdev/20201203042402.6cskdlit5f3mw4ru@kafai-mbp.dhcp.thefacebook.com/ Link: https://lore.kernel.org/netdev/20201209030903.hhow5r53l6fmozjn@kafai-mbp.dhcp.thefacebook.com/ Link: https://lore.kernel.org/bpf/20210612123224.12525-10-kuniyu@amazon.co.jp --- include/linux/bpf.h | 1 + include/linux/filter.h | 2 ++ include/uapi/linux/bpf.h | 15 +++++++++++++++ kernel/bpf/syscall.c | 13 +++++++++++++ net/core/filter.c | 13 ++++++++++++- net/core/sock_reuseport.c | 34 ++++++++++++++++++++++++++++++---- tools/include/uapi/linux/bpf.h | 15 +++++++++++++++ 7 files changed, 88 insertions(+), 5 deletions(-) (limited to 'include/uapi') diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 86dec5001ae2..f309fc1509f2 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -2048,6 +2048,7 @@ struct sk_reuseport_kern { struct sk_buff *skb; struct sock *sk; struct sock *selected_sk; + struct sock *migrating_sk; void *data_end; u32 hash; u32 reuseport_id; diff --git a/include/linux/filter.h b/include/linux/filter.h index c5ad7df029ed..688856e0b28a 100644 --- a/include/linux/filter.h +++ b/include/linux/filter.h @@ -996,11 +996,13 @@ void bpf_warn_invalid_xdp_action(u32 act); #ifdef CONFIG_INET struct sock *bpf_run_sk_reuseport(struct sock_reuseport *reuse, struct sock *sk, struct bpf_prog *prog, struct sk_buff *skb, + struct sock *migrating_sk, u32 hash); #else static inline struct sock * bpf_run_sk_reuseport(struct sock_reuseport *reuse, struct sock *sk, struct bpf_prog *prog, struct sk_buff *skb, + struct sock *migrating_sk, u32 hash) { return NULL; diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h index f3b72588442b..bf9252c7381e 100644 --- a/include/uapi/linux/bpf.h +++ b/include/uapi/linux/bpf.h @@ -994,6 +994,8 @@ enum bpf_attach_type { BPF_SK_LOOKUP, BPF_XDP, BPF_SK_SKB_VERDICT, + BPF_SK_REUSEPORT_SELECT, + BPF_SK_REUSEPORT_SELECT_OR_MIGRATE, __MAX_BPF_ATTACH_TYPE }; @@ -5416,7 +5418,20 @@ struct sk_reuseport_md { __u32 ip_protocol; /* IP protocol. e.g. IPPROTO_TCP, IPPROTO_UDP */ __u32 bind_inany; /* Is sock bound to an INANY address? */ __u32 hash; /* A hash of the packet 4 tuples */ + /* When reuse->migrating_sk is NULL, it is selecting a sk for the + * new incoming connection request (e.g. selecting a listen sk for + * the received SYN in the TCP case). reuse->sk is one of the sk + * in the reuseport group. The bpf prog can use reuse->sk to learn + * the local listening ip/port without looking into the skb. + * + * When reuse->migrating_sk is not NULL, reuse->sk is closed and + * reuse->migrating_sk is the socket that needs to be migrated + * to another listening socket. migrating_sk could be a fullsock + * sk that is fully established or a reqsk that is in-the-middle + * of 3-way handshake. + */ __bpf_md_ptr(struct bpf_sock *, sk); + __bpf_md_ptr(struct bpf_sock *, migrating_sk); }; #define BPF_TAG_SIZE 8 diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index 50457019da27..dbbc5342f221 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -1972,6 +1972,11 @@ static void bpf_prog_load_fixup_attach_type(union bpf_attr *attr) attr->expected_attach_type = BPF_CGROUP_INET_SOCK_CREATE; break; + case BPF_PROG_TYPE_SK_REUSEPORT: + if (!attr->expected_attach_type) + attr->expected_attach_type = + BPF_SK_REUSEPORT_SELECT; + break; } } @@ -2055,6 +2060,14 @@ bpf_prog_load_check_attach(enum bpf_prog_type prog_type, if (expected_attach_type == BPF_SK_LOOKUP) return 0; return -EINVAL; + case BPF_PROG_TYPE_SK_REUSEPORT: + switch (expected_attach_type) { + case BPF_SK_REUSEPORT_SELECT: + case BPF_SK_REUSEPORT_SELECT_OR_MIGRATE: + return 0; + default: + return -EINVAL; + } case BPF_PROG_TYPE_SYSCALL: case BPF_PROG_TYPE_EXT: if (expected_attach_type) diff --git a/net/core/filter.c b/net/core/filter.c index f753ab550525..5b86e47ef079 100644 --- a/net/core/filter.c +++ b/net/core/filter.c @@ -10044,11 +10044,13 @@ out: static void bpf_init_reuseport_kern(struct sk_reuseport_kern *reuse_kern, struct sock_reuseport *reuse, struct sock *sk, struct sk_buff *skb, + struct sock *migrating_sk, u32 hash) { reuse_kern->skb = skb; reuse_kern->sk = sk; reuse_kern->selected_sk = NULL; + reuse_kern->migrating_sk = migrating_sk; reuse_kern->data_end = skb->data + skb_headlen(skb); reuse_kern->hash = hash; reuse_kern->reuseport_id = reuse->reuseport_id; @@ -10057,12 +10059,13 @@ static void bpf_init_reuseport_kern(struct sk_reuseport_kern *reuse_kern, struct sock *bpf_run_sk_reuseport(struct sock_reuseport *reuse, struct sock *sk, struct bpf_prog *prog, struct sk_buff *skb, + struct sock *migrating_sk, u32 hash) { struct sk_reuseport_kern reuse_kern; enum sk_action action; - bpf_init_reuseport_kern(&reuse_kern, reuse, sk, skb, hash); + bpf_init_reuseport_kern(&reuse_kern, reuse, sk, skb, migrating_sk, hash); action = BPF_PROG_RUN(prog, &reuse_kern); if (action == SK_PASS) @@ -10207,6 +10210,10 @@ sk_reuseport_is_valid_access(int off, int size, info->reg_type = PTR_TO_SOCKET; return size == sizeof(__u64); + case offsetof(struct sk_reuseport_md, migrating_sk): + info->reg_type = PTR_TO_SOCK_COMMON_OR_NULL; + return size == sizeof(__u64); + /* Fields that allow narrowing */ case bpf_ctx_range(struct sk_reuseport_md, eth_protocol): if (size < sizeof_field(struct sk_buff, protocol)) @@ -10283,6 +10290,10 @@ static u32 sk_reuseport_convert_ctx_access(enum bpf_access_type type, case offsetof(struct sk_reuseport_md, sk): SK_REUSEPORT_LOAD_FIELD(sk); break; + + case offsetof(struct sk_reuseport_md, migrating_sk): + SK_REUSEPORT_LOAD_FIELD(migrating_sk); + break; } return insn - insn_buf; diff --git a/net/core/sock_reuseport.c b/net/core/sock_reuseport.c index b239f8cd9d39..de5ee3ae86d5 100644 --- a/net/core/sock_reuseport.c +++ b/net/core/sock_reuseport.c @@ -377,13 +377,17 @@ void reuseport_stop_listen_sock(struct sock *sk) { if (sk->sk_protocol == IPPROTO_TCP) { struct sock_reuseport *reuse; + struct bpf_prog *prog; spin_lock_bh(&reuseport_lock); reuse = rcu_dereference_protected(sk->sk_reuseport_cb, lockdep_is_held(&reuseport_lock)); + prog = rcu_dereference_protected(reuse->prog, + lockdep_is_held(&reuseport_lock)); - if (sock_net(sk)->ipv4.sysctl_tcp_migrate_req) { + if (sock_net(sk)->ipv4.sysctl_tcp_migrate_req || + (prog && prog->expected_attach_type == BPF_SK_REUSEPORT_SELECT_OR_MIGRATE)) { /* Migration capable, move sk from the listening section * to the closed section. */ @@ -488,7 +492,7 @@ struct sock *reuseport_select_sock(struct sock *sk, goto select_by_hash; if (prog->type == BPF_PROG_TYPE_SK_REUSEPORT) - sk2 = bpf_run_sk_reuseport(reuse, sk, prog, skb, hash); + sk2 = bpf_run_sk_reuseport(reuse, sk, prog, skb, NULL, hash); else sk2 = run_bpf_filter(reuse, socks, prog, skb, hdr_len); @@ -519,6 +523,8 @@ struct sock *reuseport_migrate_sock(struct sock *sk, { struct sock_reuseport *reuse; struct sock *nsk = NULL; + bool allocated = false; + struct bpf_prog *prog; u16 socks; u32 hash; @@ -536,10 +542,30 @@ struct sock *reuseport_migrate_sock(struct sock *sk, smp_rmb(); hash = migrating_sk->sk_hash; - if (sock_net(sk)->ipv4.sysctl_tcp_migrate_req) + prog = rcu_dereference(reuse->prog); + if (!prog || prog->expected_attach_type != BPF_SK_REUSEPORT_SELECT_OR_MIGRATE) { + if (sock_net(sk)->ipv4.sysctl_tcp_migrate_req) + goto select_by_hash; + goto out; + } + + if (!skb) { + skb = alloc_skb(0, GFP_ATOMIC); + if (!skb) + goto out; + allocated = true; + } + + nsk = bpf_run_sk_reuseport(reuse, sk, prog, skb, migrating_sk, hash); + + if (allocated) + kfree_skb(skb); + +select_by_hash: + if (!nsk) nsk = reuseport_select_sock_by_hash(reuse, hash, socks); - if (nsk && unlikely(!refcount_inc_not_zero(&nsk->sk_refcnt))) + if (IS_ERR_OR_NULL(nsk) || unlikely(!refcount_inc_not_zero(&nsk->sk_refcnt))) nsk = NULL; out: diff --git a/tools/include/uapi/linux/bpf.h b/tools/include/uapi/linux/bpf.h index f3b72588442b..bf9252c7381e 100644 --- a/tools/include/uapi/linux/bpf.h +++ b/tools/include/uapi/linux/bpf.h @@ -994,6 +994,8 @@ enum bpf_attach_type { BPF_SK_LOOKUP, BPF_XDP, BPF_SK_SKB_VERDICT, + BPF_SK_REUSEPORT_SELECT, + BPF_SK_REUSEPORT_SELECT_OR_MIGRATE, __MAX_BPF_ATTACH_TYPE }; @@ -5416,7 +5418,20 @@ struct sk_reuseport_md { __u32 ip_protocol; /* IP protocol. e.g. IPPROTO_TCP, IPPROTO_UDP */ __u32 bind_inany; /* Is sock bound to an INANY address? */ __u32 hash; /* A hash of the packet 4 tuples */ + /* When reuse->migrating_sk is NULL, it is selecting a sk for the + * new incoming connection request (e.g. selecting a listen sk for + * the received SYN in the TCP case). reuse->sk is one of the sk + * in the reuseport group. The bpf prog can use reuse->sk to learn + * the local listening ip/port without looking into the skb. + * + * When reuse->migrating_sk is not NULL, reuse->sk is closed and + * reuse->migrating_sk is the socket that needs to be migrated + * to another listening socket. migrating_sk could be a fullsock + * sk that is fully established or a reqsk that is in-the-middle + * of 3-way handshake. + */ __bpf_md_ptr(struct bpf_sock *, sk); + __bpf_md_ptr(struct bpf_sock *, migrating_sk); }; #define BPF_TAG_SIZE 8 -- cgit v1.2.3 From 776c53c6a448905d8b9b161805b67f82301bfe91 Mon Sep 17 00:00:00 2001 From: Maximilian Luz Date: Fri, 4 Jun 2021 15:47:52 +0200 Subject: platform/surface: aggregator_cdev: Add support for forwarding events to user-space Currently, debugging unknown events requires writing a custom driver. This is somewhat difficult, slow to adapt, and not entirely user-friendly for quickly trying to figure out things on devices of some third-party user. We can do better. We already have a user-space interface intended for debugging SAM EC requests, so let's add support for receiving events to that. This commit provides support for receiving events by reading from the controller file. It additionally introduces two new IOCTLs to control which event categories will be forwarded. Specifically, a user-space client can specify which target categories it wants to receive events from by registering the corresponding notifier(s) via the IOCTLs and after that, read the received events by reading from the controller device. Signed-off-by: Maximilian Luz Reviewed-by: Hans de Goede Link: https://lore.kernel.org/r/20210604134755.535590-5-luzmaximilian@gmail.com Signed-off-by: Hans de Goede --- Documentation/userspace-api/ioctl/ioctl-number.rst | 2 +- drivers/platform/surface/surface_aggregator_cdev.c | 460 +++++++++++++++++++-- include/uapi/linux/surface_aggregator/cdev.h | 41 +- 3 files changed, 477 insertions(+), 26 deletions(-) (limited to 'include/uapi') diff --git a/Documentation/userspace-api/ioctl/ioctl-number.rst b/Documentation/userspace-api/ioctl/ioctl-number.rst index 9bfc2b510c64..1409e40e6345 100644 --- a/Documentation/userspace-api/ioctl/ioctl-number.rst +++ b/Documentation/userspace-api/ioctl/ioctl-number.rst @@ -325,7 +325,7 @@ Code Seq# Include File Comments 0xA3 90-9F linux/dtlk.h 0xA4 00-1F uapi/linux/tee.h Generic TEE subsystem 0xA4 00-1F uapi/asm/sgx.h -0xA5 01 linux/surface_aggregator/cdev.h Microsoft Surface Platform System Aggregator +0xA5 01-05 linux/surface_aggregator/cdev.h Microsoft Surface Platform System Aggregator 0xA5 20-2F linux/surface_aggregator/dtx.h Microsoft Surface DTX driver diff --git a/drivers/platform/surface/surface_aggregator_cdev.c b/drivers/platform/surface/surface_aggregator_cdev.c index 79e28fab7e40..dcda377896b7 100644 --- a/drivers/platform/surface/surface_aggregator_cdev.c +++ b/drivers/platform/surface/surface_aggregator_cdev.c @@ -3,29 +3,69 @@ * Provides user-space access to the SSAM EC via the /dev/surface/aggregator * misc device. Intended for debugging and development. * - * Copyright (C) 2020 Maximilian Luz + * Copyright (C) 2020-2021 Maximilian Luz */ #include +#include #include +#include #include #include #include #include +#include #include #include #include +#include #include #include +#include #define SSAM_CDEV_DEVICE_NAME "surface_aggregator_cdev" + +/* -- Main structures. ------------------------------------------------------ */ + +enum ssam_cdev_device_state { + SSAM_CDEV_DEVICE_SHUTDOWN_BIT = BIT(0), +}; + struct ssam_cdev { struct kref kref; struct rw_semaphore lock; + + struct device *dev; struct ssam_controller *ctrl; struct miscdevice mdev; + unsigned long flags; + + struct rw_semaphore client_lock; /* Guards client list. */ + struct list_head client_list; +}; + +struct ssam_cdev_client; + +struct ssam_cdev_notifier { + struct ssam_cdev_client *client; + struct ssam_event_notifier nf; +}; + +struct ssam_cdev_client { + struct ssam_cdev *cdev; + struct list_head node; + + struct mutex notifier_lock; /* Guards notifier access for registration */ + struct ssam_cdev_notifier *notifier[SSH_NUM_EVENTS]; + + struct mutex read_lock; /* Guards FIFO buffer read access */ + struct mutex write_lock; /* Guards FIFO buffer write access */ + DECLARE_KFIFO(buffer, u8, 4096); + + wait_queue_head_t waitq; + struct fasync_struct *fasync; }; static void __ssam_cdev_release(struct kref *kref) @@ -47,24 +87,169 @@ static void ssam_cdev_put(struct ssam_cdev *cdev) kref_put(&cdev->kref, __ssam_cdev_release); } -static int ssam_cdev_device_open(struct inode *inode, struct file *filp) + +/* -- Notifier handling. ---------------------------------------------------- */ + +static u32 ssam_cdev_notifier(struct ssam_event_notifier *nf, const struct ssam_event *in) { - struct miscdevice *mdev = filp->private_data; - struct ssam_cdev *cdev = container_of(mdev, struct ssam_cdev, mdev); + struct ssam_cdev_notifier *cdev_nf = container_of(nf, struct ssam_cdev_notifier, nf); + struct ssam_cdev_client *client = cdev_nf->client; + struct ssam_cdev_event event; + size_t n = struct_size(&event, data, in->length); + + /* Translate event. */ + event.target_category = in->target_category; + event.target_id = in->target_id; + event.command_id = in->command_id; + event.instance_id = in->instance_id; + event.length = in->length; + + mutex_lock(&client->write_lock); + + /* Make sure we have enough space. */ + if (kfifo_avail(&client->buffer) < n) { + dev_warn(client->cdev->dev, + "buffer full, dropping event (tc: %#04x, tid: %#04x, cid: %#04x, iid: %#04x)\n", + in->target_category, in->target_id, in->command_id, in->instance_id); + mutex_unlock(&client->write_lock); + return 0; + } - filp->private_data = ssam_cdev_get(cdev); - return stream_open(inode, filp); + /* Copy event header and payload. */ + kfifo_in(&client->buffer, (const u8 *)&event, struct_size(&event, data, 0)); + kfifo_in(&client->buffer, &in->data[0], in->length); + + mutex_unlock(&client->write_lock); + + /* Notify waiting readers. */ + kill_fasync(&client->fasync, SIGIO, POLL_IN); + wake_up_interruptible(&client->waitq); + + /* + * Don't mark events as handled, this is the job of a proper driver and + * not the debugging interface. + */ + return 0; } -static int ssam_cdev_device_release(struct inode *inode, struct file *filp) +static int ssam_cdev_notifier_register(struct ssam_cdev_client *client, u8 tc, int priority) { - ssam_cdev_put(filp->private_data); - return 0; + const u16 rqid = ssh_tc_to_rqid(tc); + const u16 event = ssh_rqid_to_event(rqid); + struct ssam_cdev_notifier *nf; + int status; + + /* Validate notifier target category. */ + if (!ssh_rqid_is_event(rqid)) + return -EINVAL; + + mutex_lock(&client->notifier_lock); + + /* Check if the notifier has already been registered. */ + if (client->notifier[event]) { + mutex_unlock(&client->notifier_lock); + return -EEXIST; + } + + /* Allocate new notifier. */ + nf = kzalloc(sizeof(*nf), GFP_KERNEL); + if (!nf) { + mutex_unlock(&client->notifier_lock); + return -ENOMEM; + } + + /* + * Create a dummy notifier with the minimal required fields for + * observer registration. Note that we can skip fully specifying event + * and registry here as we do not need any matching and use silent + * registration, which does not enable the corresponding event. + */ + nf->client = client; + nf->nf.base.fn = ssam_cdev_notifier; + nf->nf.base.priority = priority; + nf->nf.event.id.target_category = tc; + nf->nf.event.mask = 0; /* Do not do any matching. */ + nf->nf.flags = SSAM_EVENT_NOTIFIER_OBSERVER; + + /* Register notifier. */ + status = ssam_notifier_register(client->cdev->ctrl, &nf->nf); + if (status) + kfree(nf); + else + client->notifier[event] = nf; + + mutex_unlock(&client->notifier_lock); + return status; } -static long ssam_cdev_request(struct ssam_cdev *cdev, unsigned long arg) +static int ssam_cdev_notifier_unregister(struct ssam_cdev_client *client, u8 tc) +{ + const u16 rqid = ssh_tc_to_rqid(tc); + const u16 event = ssh_rqid_to_event(rqid); + int status; + + /* Validate notifier target category. */ + if (!ssh_rqid_is_event(rqid)) + return -EINVAL; + + mutex_lock(&client->notifier_lock); + + /* Check if the notifier is currently registered. */ + if (!client->notifier[event]) { + mutex_unlock(&client->notifier_lock); + return -ENOENT; + } + + /* Unregister and free notifier. */ + status = ssam_notifier_unregister(client->cdev->ctrl, &client->notifier[event]->nf); + kfree(client->notifier[event]); + client->notifier[event] = NULL; + + mutex_unlock(&client->notifier_lock); + return status; +} + +static void ssam_cdev_notifier_unregister_all(struct ssam_cdev_client *client) +{ + int i; + + down_read(&client->cdev->lock); + + /* + * This function may be used during shutdown, thus we need to test for + * cdev->ctrl instead of the SSAM_CDEV_DEVICE_SHUTDOWN_BIT bit. + */ + if (client->cdev->ctrl) { + for (i = 0; i < SSH_NUM_EVENTS; i++) + ssam_cdev_notifier_unregister(client, i + 1); + + } else { + int count = 0; + + /* + * Device has been shut down. Any notifier remaining is a bug, + * so warn about that as this would otherwise hardly be + * noticeable. Nevertheless, free them as well. + */ + mutex_lock(&client->notifier_lock); + for (i = 0; i < SSH_NUM_EVENTS; i++) { + count += !!(client->notifier[i]); + kfree(client->notifier[i]); + client->notifier[i] = NULL; + } + mutex_unlock(&client->notifier_lock); + + WARN_ON(count > 0); + } + + up_read(&client->cdev->lock); +} + + +/* -- IOCTL functions. ------------------------------------------------------ */ + +static long ssam_cdev_request(struct ssam_cdev_client *client, struct ssam_cdev_request __user *r) { - struct ssam_cdev_request __user *r; struct ssam_cdev_request rqst; struct ssam_request spec = {}; struct ssam_response rsp = {}; @@ -72,7 +257,6 @@ static long ssam_cdev_request(struct ssam_cdev *cdev, unsigned long arg) void __user *rspdata; int status = 0, ret = 0, tmp; - r = (struct ssam_cdev_request __user *)arg; ret = copy_struct_from_user(&rqst, sizeof(rqst), r, sizeof(*r)); if (ret) goto out; @@ -152,7 +336,7 @@ static long ssam_cdev_request(struct ssam_cdev *cdev, unsigned long arg) } /* Perform request. */ - status = ssam_request_sync(cdev->ctrl, &spec, &rsp); + status = ssam_request_sync(client->cdev->ctrl, &spec, &rsp); if (status) goto out; @@ -177,48 +361,247 @@ out: return ret; } -static long __ssam_cdev_device_ioctl(struct ssam_cdev *cdev, unsigned int cmd, +static long ssam_cdev_notif_register(struct ssam_cdev_client *client, + const struct ssam_cdev_notifier_desc __user *d) +{ + struct ssam_cdev_notifier_desc desc; + long ret; + + ret = copy_struct_from_user(&desc, sizeof(desc), d, sizeof(*d)); + if (ret) + return ret; + + return ssam_cdev_notifier_register(client, desc.target_category, desc.priority); +} + +static long ssam_cdev_notif_unregister(struct ssam_cdev_client *client, + const struct ssam_cdev_notifier_desc __user *d) +{ + struct ssam_cdev_notifier_desc desc; + long ret; + + ret = copy_struct_from_user(&desc, sizeof(desc), d, sizeof(*d)); + if (ret) + return ret; + + return ssam_cdev_notifier_unregister(client, desc.target_category); +} + + +/* -- File operations. ------------------------------------------------------ */ + +static int ssam_cdev_device_open(struct inode *inode, struct file *filp) +{ + struct miscdevice *mdev = filp->private_data; + struct ssam_cdev_client *client; + struct ssam_cdev *cdev = container_of(mdev, struct ssam_cdev, mdev); + + /* Initialize client */ + client = vzalloc(sizeof(*client)); + if (!client) + return -ENOMEM; + + client->cdev = ssam_cdev_get(cdev); + + INIT_LIST_HEAD(&client->node); + + mutex_init(&client->notifier_lock); + + mutex_init(&client->read_lock); + mutex_init(&client->write_lock); + INIT_KFIFO(client->buffer); + init_waitqueue_head(&client->waitq); + + filp->private_data = client; + + /* Attach client. */ + down_write(&cdev->client_lock); + + if (test_bit(SSAM_CDEV_DEVICE_SHUTDOWN_BIT, &cdev->flags)) { + up_write(&cdev->client_lock); + mutex_destroy(&client->write_lock); + mutex_destroy(&client->read_lock); + mutex_destroy(&client->notifier_lock); + ssam_cdev_put(client->cdev); + vfree(client); + return -ENODEV; + } + list_add_tail(&client->node, &cdev->client_list); + + up_write(&cdev->client_lock); + + stream_open(inode, filp); + return 0; +} + +static int ssam_cdev_device_release(struct inode *inode, struct file *filp) +{ + struct ssam_cdev_client *client = filp->private_data; + + /* Force-unregister all remaining notifiers of this client. */ + ssam_cdev_notifier_unregister_all(client); + + /* Detach client. */ + down_write(&client->cdev->client_lock); + list_del(&client->node); + up_write(&client->cdev->client_lock); + + /* Free client. */ + mutex_destroy(&client->write_lock); + mutex_destroy(&client->read_lock); + + mutex_destroy(&client->notifier_lock); + + ssam_cdev_put(client->cdev); + vfree(client); + + return 0; +} + +static long __ssam_cdev_device_ioctl(struct ssam_cdev_client *client, unsigned int cmd, unsigned long arg) { switch (cmd) { case SSAM_CDEV_REQUEST: - return ssam_cdev_request(cdev, arg); + return ssam_cdev_request(client, (struct ssam_cdev_request __user *)arg); + + case SSAM_CDEV_NOTIF_REGISTER: + return ssam_cdev_notif_register(client, + (struct ssam_cdev_notifier_desc __user *)arg); + + case SSAM_CDEV_NOTIF_UNREGISTER: + return ssam_cdev_notif_unregister(client, + (struct ssam_cdev_notifier_desc __user *)arg); default: return -ENOTTY; } } -static long ssam_cdev_device_ioctl(struct file *file, unsigned int cmd, - unsigned long arg) +static long ssam_cdev_device_ioctl(struct file *file, unsigned int cmd, unsigned long arg) { - struct ssam_cdev *cdev = file->private_data; + struct ssam_cdev_client *client = file->private_data; long status; /* Ensure that controller is valid for as long as we need it. */ + if (down_read_killable(&client->cdev->lock)) + return -ERESTARTSYS; + + if (test_bit(SSAM_CDEV_DEVICE_SHUTDOWN_BIT, &client->cdev->flags)) { + up_read(&client->cdev->lock); + return -ENODEV; + } + + status = __ssam_cdev_device_ioctl(client, cmd, arg); + + up_read(&client->cdev->lock); + return status; +} + +static ssize_t ssam_cdev_read(struct file *file, char __user *buf, size_t count, loff_t *offs) +{ + struct ssam_cdev_client *client = file->private_data; + struct ssam_cdev *cdev = client->cdev; + unsigned int copied; + int status = 0; + if (down_read_killable(&cdev->lock)) return -ERESTARTSYS; - if (!cdev->ctrl) { + /* Make sure we're not shut down. */ + if (test_bit(SSAM_CDEV_DEVICE_SHUTDOWN_BIT, &cdev->flags)) { up_read(&cdev->lock); return -ENODEV; } - status = __ssam_cdev_device_ioctl(cdev, cmd, arg); + do { + /* Check availability, wait if necessary. */ + if (kfifo_is_empty(&client->buffer)) { + up_read(&cdev->lock); + + if (file->f_flags & O_NONBLOCK) + return -EAGAIN; + + status = wait_event_interruptible(client->waitq, + !kfifo_is_empty(&client->buffer) || + test_bit(SSAM_CDEV_DEVICE_SHUTDOWN_BIT, + &cdev->flags)); + if (status < 0) + return status; + + if (down_read_killable(&cdev->lock)) + return -ERESTARTSYS; + + /* Need to check that we're not shut down again. */ + if (test_bit(SSAM_CDEV_DEVICE_SHUTDOWN_BIT, &cdev->flags)) { + up_read(&cdev->lock); + return -ENODEV; + } + } + + /* Try to read from FIFO. */ + if (mutex_lock_interruptible(&client->read_lock)) { + up_read(&cdev->lock); + return -ERESTARTSYS; + } + + status = kfifo_to_user(&client->buffer, buf, count, &copied); + mutex_unlock(&client->read_lock); + + if (status < 0) { + up_read(&cdev->lock); + return status; + } + + /* We might not have gotten anything, check this here. */ + if (copied == 0 && (file->f_flags & O_NONBLOCK)) { + up_read(&cdev->lock); + return -EAGAIN; + } + } while (copied == 0); up_read(&cdev->lock); - return status; + return copied; +} + +static __poll_t ssam_cdev_poll(struct file *file, struct poll_table_struct *pt) +{ + struct ssam_cdev_client *client = file->private_data; + __poll_t events = 0; + + if (test_bit(SSAM_CDEV_DEVICE_SHUTDOWN_BIT, &client->cdev->flags)) + return EPOLLHUP | EPOLLERR; + + poll_wait(file, &client->waitq, pt); + + if (!kfifo_is_empty(&client->buffer)) + events |= EPOLLIN | EPOLLRDNORM; + + return events; +} + +static int ssam_cdev_fasync(int fd, struct file *file, int on) +{ + struct ssam_cdev_client *client = file->private_data; + + return fasync_helper(fd, file, on, &client->fasync); } static const struct file_operations ssam_controller_fops = { .owner = THIS_MODULE, .open = ssam_cdev_device_open, .release = ssam_cdev_device_release, + .read = ssam_cdev_read, + .poll = ssam_cdev_poll, + .fasync = ssam_cdev_fasync, .unlocked_ioctl = ssam_cdev_device_ioctl, .compat_ioctl = ssam_cdev_device_ioctl, - .llseek = noop_llseek, + .llseek = no_llseek, }; + +/* -- Device and driver setup ----------------------------------------------- */ + static int ssam_dbg_device_probe(struct platform_device *pdev) { struct ssam_controller *ctrl; @@ -236,6 +619,7 @@ static int ssam_dbg_device_probe(struct platform_device *pdev) kref_init(&cdev->kref); init_rwsem(&cdev->lock); cdev->ctrl = ctrl; + cdev->dev = &pdev->dev; cdev->mdev.parent = &pdev->dev; cdev->mdev.minor = MISC_DYNAMIC_MINOR; @@ -243,6 +627,9 @@ static int ssam_dbg_device_probe(struct platform_device *pdev) cdev->mdev.nodename = "surface/aggregator"; cdev->mdev.fops = &ssam_controller_fops; + init_rwsem(&cdev->client_lock); + INIT_LIST_HEAD(&cdev->client_list); + status = misc_register(&cdev->mdev); if (status) { kfree(cdev); @@ -256,8 +643,32 @@ static int ssam_dbg_device_probe(struct platform_device *pdev) static int ssam_dbg_device_remove(struct platform_device *pdev) { struct ssam_cdev *cdev = platform_get_drvdata(pdev); + struct ssam_cdev_client *client; - misc_deregister(&cdev->mdev); + /* + * Mark device as shut-down. Prevent new clients from being added and + * new operations from being executed. + */ + set_bit(SSAM_CDEV_DEVICE_SHUTDOWN_BIT, &cdev->flags); + + down_write(&cdev->client_lock); + + /* Remove all notifiers registered by us. */ + list_for_each_entry(client, &cdev->client_list, node) { + ssam_cdev_notifier_unregister_all(client); + } + + /* Wake up async clients. */ + list_for_each_entry(client, &cdev->client_list, node) { + kill_fasync(&client->fasync, SIGIO, POLL_HUP); + } + + /* Wake up blocking clients. */ + list_for_each_entry(client, &cdev->client_list, node) { + wake_up_interruptible(&client->waitq); + } + + up_write(&cdev->client_lock); /* * The controller is only guaranteed to be valid for as long as the @@ -266,8 +677,11 @@ static int ssam_dbg_device_remove(struct platform_device *pdev) */ down_write(&cdev->lock); cdev->ctrl = NULL; + cdev->dev = NULL; up_write(&cdev->lock); + misc_deregister(&cdev->mdev); + ssam_cdev_put(cdev); return 0; } diff --git a/include/uapi/linux/surface_aggregator/cdev.h b/include/uapi/linux/surface_aggregator/cdev.h index fbcce04abfe9..4f393fafc235 100644 --- a/include/uapi/linux/surface_aggregator/cdev.h +++ b/include/uapi/linux/surface_aggregator/cdev.h @@ -6,7 +6,7 @@ * device. This device provides direct user-space access to the SSAM EC. * Intended for debugging and development. * - * Copyright (C) 2020 Maximilian Luz + * Copyright (C) 2020-2021 Maximilian Luz */ #ifndef _UAPI_LINUX_SURFACE_AGGREGATOR_CDEV_H @@ -73,6 +73,43 @@ struct ssam_cdev_request { } response; } __attribute__((__packed__)); -#define SSAM_CDEV_REQUEST _IOWR(0xA5, 1, struct ssam_cdev_request) +/** + * struct ssam_cdev_notifier_desc - Notifier descriptor. + * @priority: Priority value determining the order in which notifier + * callbacks will be called. A higher value means higher + * priority, i.e. the associated callback will be executed + * earlier than other (lower priority) callbacks. + * @target_category: The event target category for which this notifier should + * receive events. + * + * Specifies the notifier that should be registered or unregistered, + * specifically with which priority and for which target category of events. + */ +struct ssam_cdev_notifier_desc { + __s32 priority; + __u8 target_category; +} __attribute__((__packed__)); + +/** + * struct ssam_cdev_event - SSAM event sent by the EC. + * @target_category: Target category of the event source. See &enum ssam_ssh_tc. + * @target_id: Target ID of the event source. + * @command_id: Command ID of the event. + * @instance_id: Instance ID of the event source. + * @length: Length of the event payload in bytes. + * @data: Event payload data. + */ +struct ssam_cdev_event { + __u8 target_category; + __u8 target_id; + __u8 command_id; + __u8 instance_id; + __u16 length; + __u8 data[]; +} __attribute__((__packed__)); + +#define SSAM_CDEV_REQUEST _IOWR(0xA5, 1, struct ssam_cdev_request) +#define SSAM_CDEV_NOTIF_REGISTER _IOW(0xA5, 2, struct ssam_cdev_notifier_desc) +#define SSAM_CDEV_NOTIF_UNREGISTER _IOW(0xA5, 3, struct ssam_cdev_notifier_desc) #endif /* _UAPI_LINUX_SURFACE_AGGREGATOR_CDEV_H */ -- cgit v1.2.3 From e8e298a653856b1f3a2bb7b1fe31d3faa93cc7dc Mon Sep 17 00:00:00 2001 From: Maximilian Luz Date: Fri, 4 Jun 2021 15:47:53 +0200 Subject: platform/surface: aggregator_cdev: Allow enabling of events from user-space While events can already be enabled and disabled via the generic request IOCTL, this bypasses the internal reference counting mechanism of the controller. Due to that, disabling an event will turn it off regardless of any other client having requested said event, which may break functionality of that client. To solve this, add IOCTLs wrapping the ssam_controller_event_enable() and ssam_controller_event_disable() functions, which have been previously introduced for this specific purpose. Signed-off-by: Maximilian Luz Reviewed-by: Hans de Goede Link: https://lore.kernel.org/r/20210604134755.535590-6-luzmaximilian@gmail.com Signed-off-by: Hans de Goede --- drivers/platform/surface/surface_aggregator_cdev.c | 58 ++++++++++++++++++++++ include/uapi/linux/surface_aggregator/cdev.h | 32 ++++++++++++ 2 files changed, 90 insertions(+) (limited to 'include/uapi') diff --git a/drivers/platform/surface/surface_aggregator_cdev.c b/drivers/platform/surface/surface_aggregator_cdev.c index dcda377896b7..7b86b36eaaa0 100644 --- a/drivers/platform/surface/surface_aggregator_cdev.c +++ b/drivers/platform/surface/surface_aggregator_cdev.c @@ -387,6 +387,58 @@ static long ssam_cdev_notif_unregister(struct ssam_cdev_client *client, return ssam_cdev_notifier_unregister(client, desc.target_category); } +static long ssam_cdev_event_enable(struct ssam_cdev_client *client, + const struct ssam_cdev_event_desc __user *d) +{ + struct ssam_cdev_event_desc desc; + struct ssam_event_registry reg; + struct ssam_event_id id; + long ret; + + /* Read descriptor from user-space. */ + ret = copy_struct_from_user(&desc, sizeof(desc), d, sizeof(*d)); + if (ret) + return ret; + + /* Translate descriptor. */ + reg.target_category = desc.reg.target_category; + reg.target_id = desc.reg.target_id; + reg.cid_enable = desc.reg.cid_enable; + reg.cid_disable = desc.reg.cid_disable; + + id.target_category = desc.id.target_category; + id.instance = desc.id.instance; + + /* Disable event. */ + return ssam_controller_event_enable(client->cdev->ctrl, reg, id, desc.flags); +} + +static long ssam_cdev_event_disable(struct ssam_cdev_client *client, + const struct ssam_cdev_event_desc __user *d) +{ + struct ssam_cdev_event_desc desc; + struct ssam_event_registry reg; + struct ssam_event_id id; + long ret; + + /* Read descriptor from user-space. */ + ret = copy_struct_from_user(&desc, sizeof(desc), d, sizeof(*d)); + if (ret) + return ret; + + /* Translate descriptor. */ + reg.target_category = desc.reg.target_category; + reg.target_id = desc.reg.target_id; + reg.cid_enable = desc.reg.cid_enable; + reg.cid_disable = desc.reg.cid_disable; + + id.target_category = desc.id.target_category; + id.instance = desc.id.instance; + + /* Disable event. */ + return ssam_controller_event_disable(client->cdev->ctrl, reg, id, desc.flags); +} + /* -- File operations. ------------------------------------------------------ */ @@ -473,6 +525,12 @@ static long __ssam_cdev_device_ioctl(struct ssam_cdev_client *client, unsigned i return ssam_cdev_notif_unregister(client, (struct ssam_cdev_notifier_desc __user *)arg); + case SSAM_CDEV_EVENT_ENABLE: + return ssam_cdev_event_enable(client, (struct ssam_cdev_event_desc __user *)arg); + + case SSAM_CDEV_EVENT_DISABLE: + return ssam_cdev_event_disable(client, (struct ssam_cdev_event_desc __user *)arg); + default: return -ENOTTY; } diff --git a/include/uapi/linux/surface_aggregator/cdev.h b/include/uapi/linux/surface_aggregator/cdev.h index 4f393fafc235..08f46b60b151 100644 --- a/include/uapi/linux/surface_aggregator/cdev.h +++ b/include/uapi/linux/surface_aggregator/cdev.h @@ -90,6 +90,36 @@ struct ssam_cdev_notifier_desc { __u8 target_category; } __attribute__((__packed__)); +/** + * struct ssam_cdev_event_desc - Event descriptor. + * @reg: Registry via which the event will be enabled/disabled. + * @reg.target_category: Target category for the event registry requests. + * @reg.target_id: Target ID for the event registry requests. + * @reg.cid_enable: Command ID for the event-enable request. + * @reg.cid_disable: Command ID for the event-disable request. + * @id: ID specifying the event. + * @id.target_category: Target category of the event source. + * @id.instance: Instance ID of the event source. + * @flags: Flags used for enabling the event. + * + * Specifies which event should be enabled/disabled and how to do that. + */ +struct ssam_cdev_event_desc { + struct { + __u8 target_category; + __u8 target_id; + __u8 cid_enable; + __u8 cid_disable; + } reg; + + struct { + __u8 target_category; + __u8 instance; + } id; + + __u8 flags; +} __attribute__((__packed__)); + /** * struct ssam_cdev_event - SSAM event sent by the EC. * @target_category: Target category of the event source. See &enum ssam_ssh_tc. @@ -111,5 +141,7 @@ struct ssam_cdev_event { #define SSAM_CDEV_REQUEST _IOWR(0xA5, 1, struct ssam_cdev_request) #define SSAM_CDEV_NOTIF_REGISTER _IOW(0xA5, 2, struct ssam_cdev_notifier_desc) #define SSAM_CDEV_NOTIF_UNREGISTER _IOW(0xA5, 3, struct ssam_cdev_notifier_desc) +#define SSAM_CDEV_EVENT_ENABLE _IOW(0xA5, 4, struct ssam_cdev_event_desc) +#define SSAM_CDEV_EVENT_DISABLE _IOW(0xA5, 5, struct ssam_cdev_event_desc) #endif /* _UAPI_LINUX_SURFACE_AGGREGATOR_CDEV_H */ -- cgit v1.2.3 From 8c40602b4be17571dfd75102f4f1e690311c5210 Mon Sep 17 00:00:00 2001 From: Guvenc Gulce Date: Wed, 16 Jun 2021 16:52:56 +0200 Subject: net/smc: Add netlink support for SMC statistics Add the netlink function which collects the statistics information and delivers it to the userspace. Signed-off-by: Guvenc Gulce Signed-off-by: Karsten Graul Signed-off-by: David S. Miller --- include/uapi/linux/smc.h | 69 ++++++++++++ net/smc/smc_netlink.c | 6 + net/smc/smc_stats.c | 279 +++++++++++++++++++++++++++++++++++++++++++++++ net/smc/smc_stats.h | 1 + 4 files changed, 355 insertions(+) (limited to 'include/uapi') diff --git a/include/uapi/linux/smc.h b/include/uapi/linux/smc.h index 3e68da07fba2..f32f11b30963 100644 --- a/include/uapi/linux/smc.h +++ b/include/uapi/linux/smc.h @@ -47,6 +47,7 @@ enum { SMC_NETLINK_GET_LGR_SMCD, SMC_NETLINK_GET_DEV_SMCD, SMC_NETLINK_GET_DEV_SMCR, + SMC_NETLINK_GET_STATS, }; /* SMC_GENL_FAMILY top level attributes */ @@ -58,6 +59,7 @@ enum { SMC_GEN_LGR_SMCD, /* nest */ SMC_GEN_DEV_SMCD, /* nest */ SMC_GEN_DEV_SMCR, /* nest */ + SMC_GEN_STATS, /* nest */ __SMC_GEN_MAX, SMC_GEN_MAX = __SMC_GEN_MAX - 1 }; @@ -159,4 +161,71 @@ enum { SMC_NLA_DEV_MAX = __SMC_NLA_DEV_MAX - 1 }; +/* SMC_NLA_STATS_T_TX(RX)_RMB_SIZE nested attributes */ +/* SMC_NLA_STATS_TX(RX)PLOAD_SIZE nested attributes */ +enum { + SMC_NLA_STATS_PLOAD_PAD, + SMC_NLA_STATS_PLOAD_8K, /* u64 */ + SMC_NLA_STATS_PLOAD_16K, /* u64 */ + SMC_NLA_STATS_PLOAD_32K, /* u64 */ + SMC_NLA_STATS_PLOAD_64K, /* u64 */ + SMC_NLA_STATS_PLOAD_128K, /* u64 */ + SMC_NLA_STATS_PLOAD_256K, /* u64 */ + SMC_NLA_STATS_PLOAD_512K, /* u64 */ + SMC_NLA_STATS_PLOAD_1024K, /* u64 */ + SMC_NLA_STATS_PLOAD_G_1024K, /* u64 */ + __SMC_NLA_STATS_PLOAD_MAX, + SMC_NLA_STATS_PLOAD_MAX = __SMC_NLA_STATS_PLOAD_MAX - 1 +}; + +/* SMC_NLA_STATS_T_TX(RX)_RMB_STATS nested attributes */ +enum { + SMC_NLA_STATS_RMB_PAD, + SMC_NLA_STATS_RMB_SIZE_SM_PEER_CNT, /* u64 */ + SMC_NLA_STATS_RMB_SIZE_SM_CNT, /* u64 */ + SMC_NLA_STATS_RMB_FULL_PEER_CNT, /* u64 */ + SMC_NLA_STATS_RMB_FULL_CNT, /* u64 */ + SMC_NLA_STATS_RMB_REUSE_CNT, /* u64 */ + SMC_NLA_STATS_RMB_ALLOC_CNT, /* u64 */ + SMC_NLA_STATS_RMB_DGRADE_CNT, /* u64 */ + __SMC_NLA_STATS_RMB_MAX, + SMC_NLA_STATS_RMB_MAX = __SMC_NLA_STATS_RMB_MAX - 1 +}; + +/* SMC_NLA_STATS_SMCD_TECH and _SMCR_TECH nested attributes */ +enum { + SMC_NLA_STATS_T_PAD, + SMC_NLA_STATS_T_TX_RMB_SIZE, /* nest */ + SMC_NLA_STATS_T_RX_RMB_SIZE, /* nest */ + SMC_NLA_STATS_T_TXPLOAD_SIZE, /* nest */ + SMC_NLA_STATS_T_RXPLOAD_SIZE, /* nest */ + SMC_NLA_STATS_T_TX_RMB_STATS, /* nest */ + SMC_NLA_STATS_T_RX_RMB_STATS, /* nest */ + SMC_NLA_STATS_T_CLNT_V1_SUCC, /* u64 */ + SMC_NLA_STATS_T_CLNT_V2_SUCC, /* u64 */ + SMC_NLA_STATS_T_SRV_V1_SUCC, /* u64 */ + SMC_NLA_STATS_T_SRV_V2_SUCC, /* u64 */ + SMC_NLA_STATS_T_SENDPAGE_CNT, /* u64 */ + SMC_NLA_STATS_T_SPLICE_CNT, /* u64 */ + SMC_NLA_STATS_T_CORK_CNT, /* u64 */ + SMC_NLA_STATS_T_NDLY_CNT, /* u64 */ + SMC_NLA_STATS_T_URG_DATA_CNT, /* u64 */ + SMC_NLA_STATS_T_RX_BYTES, /* u64 */ + SMC_NLA_STATS_T_TX_BYTES, /* u64 */ + SMC_NLA_STATS_T_RX_CNT, /* u64 */ + SMC_NLA_STATS_T_TX_CNT, /* u64 */ + __SMC_NLA_STATS_T_MAX, + SMC_NLA_STATS_T_MAX = __SMC_NLA_STATS_T_MAX - 1 +}; + +/* SMC_GEN_STATS attributes */ +enum { + SMC_NLA_STATS_PAD, + SMC_NLA_STATS_SMCD_TECH, /* nest */ + SMC_NLA_STATS_SMCR_TECH, /* nest */ + SMC_NLA_STATS_CLNT_HS_ERR_CNT, /* u64 */ + SMC_NLA_STATS_SRV_HS_ERR_CNT, /* u64 */ + __SMC_NLA_STATS_MAX, + SMC_NLA_STATS_MAX = __SMC_NLA_STATS_MAX - 1 +}; #endif /* _UAPI_LINUX_SMC_H */ diff --git a/net/smc/smc_netlink.c b/net/smc/smc_netlink.c index 140419a19dbf..30e30b23370f 100644 --- a/net/smc/smc_netlink.c +++ b/net/smc/smc_netlink.c @@ -19,6 +19,7 @@ #include "smc_core.h" #include "smc_ism.h" #include "smc_ib.h" +#include "smc_stats.h" #include "smc_netlink.h" #define SMC_CMD_MAX_ATTR 1 @@ -55,6 +56,11 @@ static const struct genl_ops smc_gen_nl_ops[] = { /* can be retrieved by unprivileged users */ .dumpit = smcr_nl_get_device, }, + { + .cmd = SMC_NETLINK_GET_STATS, + /* can be retrieved by unprivileged users */ + .dumpit = smc_nl_get_stats, + }, }; static const struct nla_policy smc_gen_nl_policy[2] = { diff --git a/net/smc/smc_stats.c b/net/smc/smc_stats.c index 76e938388520..72119d3d8558 100644 --- a/net/smc/smc_stats.c +++ b/net/smc/smc_stats.c @@ -12,6 +12,10 @@ #include #include #include +#include +#include +#include +#include "smc_netlink.h" #include "smc_stats.h" /* serialize fallback reason statistic gathering */ @@ -33,3 +37,278 @@ void smc_stats_exit(void) { free_percpu(smc_stats); } + +static int smc_nl_fill_stats_rmb_data(struct sk_buff *skb, + struct smc_stats *stats, int tech, + int type) +{ + struct smc_stats_rmbcnt *stats_rmb_cnt; + struct nlattr *attrs; + + if (type == SMC_NLA_STATS_T_TX_RMB_STATS) + stats_rmb_cnt = &stats->smc[tech].rmb_tx; + else + stats_rmb_cnt = &stats->smc[tech].rmb_rx; + + attrs = nla_nest_start(skb, type); + if (!attrs) + goto errout; + if (nla_put_u64_64bit(skb, SMC_NLA_STATS_RMB_REUSE_CNT, + stats_rmb_cnt->reuse_cnt, + SMC_NLA_STATS_RMB_PAD)) + goto errattr; + if (nla_put_u64_64bit(skb, SMC_NLA_STATS_RMB_SIZE_SM_PEER_CNT, + stats_rmb_cnt->buf_size_small_peer_cnt, + SMC_NLA_STATS_RMB_PAD)) + goto errattr; + if (nla_put_u64_64bit(skb, SMC_NLA_STATS_RMB_SIZE_SM_CNT, + stats_rmb_cnt->buf_size_small_cnt, + SMC_NLA_STATS_RMB_PAD)) + goto errattr; + if (nla_put_u64_64bit(skb, SMC_NLA_STATS_RMB_FULL_PEER_CNT, + stats_rmb_cnt->buf_full_peer_cnt, + SMC_NLA_STATS_RMB_PAD)) + goto errattr; + if (nla_put_u64_64bit(skb, SMC_NLA_STATS_RMB_FULL_CNT, + stats_rmb_cnt->buf_full_cnt, + SMC_NLA_STATS_RMB_PAD)) + goto errattr; + if (nla_put_u64_64bit(skb, SMC_NLA_STATS_RMB_ALLOC_CNT, + stats_rmb_cnt->alloc_cnt, + SMC_NLA_STATS_RMB_PAD)) + goto errattr; + if (nla_put_u64_64bit(skb, SMC_NLA_STATS_RMB_DGRADE_CNT, + stats_rmb_cnt->dgrade_cnt, + SMC_NLA_STATS_RMB_PAD)) + goto errattr; + + nla_nest_end(skb, attrs); + return 0; + +errattr: + nla_nest_cancel(skb, attrs); +errout: + return -EMSGSIZE; +} + +static int smc_nl_fill_stats_bufsize_data(struct sk_buff *skb, + struct smc_stats *stats, int tech, + int type) +{ + struct smc_stats_memsize *stats_pload; + struct nlattr *attrs; + + if (type == SMC_NLA_STATS_T_TXPLOAD_SIZE) + stats_pload = &stats->smc[tech].tx_pd; + else if (type == SMC_NLA_STATS_T_RXPLOAD_SIZE) + stats_pload = &stats->smc[tech].rx_pd; + else if (type == SMC_NLA_STATS_T_TX_RMB_SIZE) + stats_pload = &stats->smc[tech].tx_rmbsize; + else if (type == SMC_NLA_STATS_T_RX_RMB_SIZE) + stats_pload = &stats->smc[tech].rx_rmbsize; + else + goto errout; + + attrs = nla_nest_start(skb, type); + if (!attrs) + goto errout; + if (nla_put_u64_64bit(skb, SMC_NLA_STATS_PLOAD_8K, + stats_pload->buf[SMC_BUF_8K], + SMC_NLA_STATS_PLOAD_PAD)) + goto errattr; + if (nla_put_u64_64bit(skb, SMC_NLA_STATS_PLOAD_16K, + stats_pload->buf[SMC_BUF_16K], + SMC_NLA_STATS_PLOAD_PAD)) + goto errattr; + if (nla_put_u64_64bit(skb, SMC_NLA_STATS_PLOAD_32K, + stats_pload->buf[SMC_BUF_32K], + SMC_NLA_STATS_PLOAD_PAD)) + goto errattr; + if (nla_put_u64_64bit(skb, SMC_NLA_STATS_PLOAD_64K, + stats_pload->buf[SMC_BUF_64K], + SMC_NLA_STATS_PLOAD_PAD)) + goto errattr; + if (nla_put_u64_64bit(skb, SMC_NLA_STATS_PLOAD_128K, + stats_pload->buf[SMC_BUF_128K], + SMC_NLA_STATS_PLOAD_PAD)) + goto errattr; + if (nla_put_u64_64bit(skb, SMC_NLA_STATS_PLOAD_256K, + stats_pload->buf[SMC_BUF_256K], + SMC_NLA_STATS_PLOAD_PAD)) + goto errattr; + if (nla_put_u64_64bit(skb, SMC_NLA_STATS_PLOAD_512K, + stats_pload->buf[SMC_BUF_512K], + SMC_NLA_STATS_PLOAD_PAD)) + goto errattr; + if (nla_put_u64_64bit(skb, SMC_NLA_STATS_PLOAD_1024K, + stats_pload->buf[SMC_BUF_1024K], + SMC_NLA_STATS_PLOAD_PAD)) + goto errattr; + if (nla_put_u64_64bit(skb, SMC_NLA_STATS_PLOAD_G_1024K, + stats_pload->buf[SMC_BUF_G_1024K], + SMC_NLA_STATS_PLOAD_PAD)) + goto errattr; + + nla_nest_end(skb, attrs); + return 0; + +errattr: + nla_nest_cancel(skb, attrs); +errout: + return -EMSGSIZE; +} + +static int smc_nl_fill_stats_tech_data(struct sk_buff *skb, + struct smc_stats *stats, int tech) +{ + struct smc_stats_tech *smc_tech; + struct nlattr *attrs; + + smc_tech = &stats->smc[tech]; + if (tech == SMC_TYPE_D) + attrs = nla_nest_start(skb, SMC_NLA_STATS_SMCD_TECH); + else + attrs = nla_nest_start(skb, SMC_NLA_STATS_SMCR_TECH); + + if (!attrs) + goto errout; + if (smc_nl_fill_stats_rmb_data(skb, stats, tech, + SMC_NLA_STATS_T_TX_RMB_STATS)) + goto errattr; + if (smc_nl_fill_stats_rmb_data(skb, stats, tech, + SMC_NLA_STATS_T_RX_RMB_STATS)) + goto errattr; + if (smc_nl_fill_stats_bufsize_data(skb, stats, tech, + SMC_NLA_STATS_T_TXPLOAD_SIZE)) + goto errattr; + if (smc_nl_fill_stats_bufsize_data(skb, stats, tech, + SMC_NLA_STATS_T_RXPLOAD_SIZE)) + goto errattr; + if (smc_nl_fill_stats_bufsize_data(skb, stats, tech, + SMC_NLA_STATS_T_TX_RMB_SIZE)) + goto errattr; + if (smc_nl_fill_stats_bufsize_data(skb, stats, tech, + SMC_NLA_STATS_T_RX_RMB_SIZE)) + goto errattr; + if (nla_put_u64_64bit(skb, SMC_NLA_STATS_T_CLNT_V1_SUCC, + smc_tech->clnt_v1_succ_cnt, + SMC_NLA_STATS_PAD)) + goto errattr; + if (nla_put_u64_64bit(skb, SMC_NLA_STATS_T_CLNT_V2_SUCC, + smc_tech->clnt_v2_succ_cnt, + SMC_NLA_STATS_PAD)) + goto errattr; + if (nla_put_u64_64bit(skb, SMC_NLA_STATS_T_SRV_V1_SUCC, + smc_tech->srv_v1_succ_cnt, + SMC_NLA_STATS_PAD)) + goto errattr; + if (nla_put_u64_64bit(skb, SMC_NLA_STATS_T_SRV_V2_SUCC, + smc_tech->srv_v2_succ_cnt, + SMC_NLA_STATS_PAD)) + goto errattr; + if (nla_put_u64_64bit(skb, SMC_NLA_STATS_T_RX_BYTES, + smc_tech->rx_bytes, + SMC_NLA_STATS_PAD)) + goto errattr; + if (nla_put_u64_64bit(skb, SMC_NLA_STATS_T_TX_BYTES, + smc_tech->tx_bytes, + SMC_NLA_STATS_PAD)) + goto errattr; + if (nla_put_u64_64bit(skb, SMC_NLA_STATS_T_RX_CNT, + smc_tech->rx_cnt, + SMC_NLA_STATS_PAD)) + goto errattr; + if (nla_put_u64_64bit(skb, SMC_NLA_STATS_T_TX_CNT, + smc_tech->tx_cnt, + SMC_NLA_STATS_PAD)) + goto errattr; + if (nla_put_u64_64bit(skb, SMC_NLA_STATS_T_SENDPAGE_CNT, + smc_tech->sendpage_cnt, + SMC_NLA_STATS_PAD)) + goto errattr; + if (nla_put_u64_64bit(skb, SMC_NLA_STATS_T_CORK_CNT, + smc_tech->cork_cnt, + SMC_NLA_STATS_PAD)) + goto errattr; + if (nla_put_u64_64bit(skb, SMC_NLA_STATS_T_NDLY_CNT, + smc_tech->ndly_cnt, + SMC_NLA_STATS_PAD)) + goto errattr; + if (nla_put_u64_64bit(skb, SMC_NLA_STATS_T_SPLICE_CNT, + smc_tech->splice_cnt, + SMC_NLA_STATS_PAD)) + goto errattr; + if (nla_put_u64_64bit(skb, SMC_NLA_STATS_T_URG_DATA_CNT, + smc_tech->urg_data_cnt, + SMC_NLA_STATS_PAD)) + goto errattr; + + nla_nest_end(skb, attrs); + return 0; + +errattr: + nla_nest_cancel(skb, attrs); +errout: + return -EMSGSIZE; +} + +int smc_nl_get_stats(struct sk_buff *skb, + struct netlink_callback *cb) +{ + struct smc_nl_dmp_ctx *cb_ctx = smc_nl_dmp_ctx(cb); + struct smc_stats *stats; + struct nlattr *attrs; + int cpu, i, size; + void *nlh; + u64 *src; + u64 *sum; + + if (cb_ctx->pos[0]) + goto errmsg; + nlh = genlmsg_put(skb, NETLINK_CB(cb->skb).portid, cb->nlh->nlmsg_seq, + &smc_gen_nl_family, NLM_F_MULTI, + SMC_NETLINK_GET_STATS); + if (!nlh) + goto errmsg; + + attrs = nla_nest_start(skb, SMC_GEN_STATS); + if (!attrs) + goto errnest; + stats = kzalloc(sizeof(*stats), GFP_KERNEL); + if (!stats) + goto erralloc; + size = sizeof(*stats) / sizeof(u64); + for_each_possible_cpu(cpu) { + src = (u64 *)per_cpu_ptr(smc_stats, cpu); + sum = (u64 *)stats; + for (i = 0; i < size; i++) + *(sum++) += *(src++); + } + if (smc_nl_fill_stats_tech_data(skb, stats, SMC_TYPE_D)) + goto errattr; + if (smc_nl_fill_stats_tech_data(skb, stats, SMC_TYPE_R)) + goto errattr; + if (nla_put_u64_64bit(skb, SMC_NLA_STATS_CLNT_HS_ERR_CNT, + stats->clnt_hshake_err_cnt, + SMC_NLA_STATS_PAD)) + goto errattr; + if (nla_put_u64_64bit(skb, SMC_NLA_STATS_SRV_HS_ERR_CNT, + stats->srv_hshake_err_cnt, + SMC_NLA_STATS_PAD)) + goto errattr; + + nla_nest_end(skb, attrs); + genlmsg_end(skb, nlh); + cb_ctx->pos[0] = 1; + kfree(stats); + return skb->len; + +errattr: + kfree(stats); +erralloc: + nla_nest_cancel(skb, attrs); +errnest: + genlmsg_cancel(skb, nlh); +errmsg: + return skb->len; +} diff --git a/net/smc/smc_stats.h b/net/smc/smc_stats.h index 928372114cf1..84baaca59eaf 100644 --- a/net/smc/smc_stats.h +++ b/net/smc/smc_stats.h @@ -247,6 +247,7 @@ do { \ } \ while (0) +int smc_nl_get_stats(struct sk_buff *skb, struct netlink_callback *cb); int smc_stats_init(void) __init; void smc_stats_exit(void); -- cgit v1.2.3 From f0dd7bf5e33066e554442c509ef6351728b95b51 Mon Sep 17 00:00:00 2001 From: Guvenc Gulce Date: Wed, 16 Jun 2021 16:52:57 +0200 Subject: net/smc: Add netlink support for SMC fallback statistics Add support to collect more detailed SMC fallback reason statistics and provide these statistics to user space on the netlink interface. Signed-off-by: Guvenc Gulce Signed-off-by: Karsten Graul Signed-off-by: David S. Miller --- include/uapi/linux/smc.h | 14 ++++++++ net/smc/smc_netlink.c | 5 +++ net/smc/smc_netlink.h | 2 +- net/smc/smc_stats.c | 92 ++++++++++++++++++++++++++++++++++++++++++++++++ net/smc/smc_stats.h | 1 + 5 files changed, 113 insertions(+), 1 deletion(-) (limited to 'include/uapi') diff --git a/include/uapi/linux/smc.h b/include/uapi/linux/smc.h index f32f11b30963..0f7f87c70baf 100644 --- a/include/uapi/linux/smc.h +++ b/include/uapi/linux/smc.h @@ -48,6 +48,7 @@ enum { SMC_NETLINK_GET_DEV_SMCD, SMC_NETLINK_GET_DEV_SMCR, SMC_NETLINK_GET_STATS, + SMC_NETLINK_GET_FBACK_STATS, }; /* SMC_GENL_FAMILY top level attributes */ @@ -60,6 +61,7 @@ enum { SMC_GEN_DEV_SMCD, /* nest */ SMC_GEN_DEV_SMCR, /* nest */ SMC_GEN_STATS, /* nest */ + SMC_GEN_FBACK_STATS, /* nest */ __SMC_GEN_MAX, SMC_GEN_MAX = __SMC_GEN_MAX - 1 }; @@ -228,4 +230,16 @@ enum { __SMC_NLA_STATS_MAX, SMC_NLA_STATS_MAX = __SMC_NLA_STATS_MAX - 1 }; + +/* SMC_GEN_FBACK_STATS attributes */ +enum { + SMC_NLA_FBACK_STATS_PAD, + SMC_NLA_FBACK_STATS_TYPE, /* u8 */ + SMC_NLA_FBACK_STATS_SRV_CNT, /* u64 */ + SMC_NLA_FBACK_STATS_CLNT_CNT, /* u64 */ + SMC_NLA_FBACK_STATS_RSN_CODE, /* u32 */ + SMC_NLA_FBACK_STATS_RSN_CNT, /* u16 */ + __SMC_NLA_FBACK_STATS_MAX, + SMC_NLA_FBACK_STATS_MAX = __SMC_NLA_FBACK_STATS_MAX - 1 +}; #endif /* _UAPI_LINUX_SMC_H */ diff --git a/net/smc/smc_netlink.c b/net/smc/smc_netlink.c index 30e30b23370f..6fb6f96c1d17 100644 --- a/net/smc/smc_netlink.c +++ b/net/smc/smc_netlink.c @@ -61,6 +61,11 @@ static const struct genl_ops smc_gen_nl_ops[] = { /* can be retrieved by unprivileged users */ .dumpit = smc_nl_get_stats, }, + { + .cmd = SMC_NETLINK_GET_FBACK_STATS, + /* can be retrieved by unprivileged users */ + .dumpit = smc_nl_get_fback_stats, + }, }; static const struct nla_policy smc_gen_nl_policy[2] = { diff --git a/net/smc/smc_netlink.h b/net/smc/smc_netlink.h index 3477265cba6c..5ce2c0a89ccd 100644 --- a/net/smc/smc_netlink.h +++ b/net/smc/smc_netlink.h @@ -18,7 +18,7 @@ extern struct genl_family smc_gen_nl_family; struct smc_nl_dmp_ctx { - int pos[2]; + int pos[3]; }; static inline struct smc_nl_dmp_ctx *smc_nl_dmp_ctx(struct netlink_callback *c) diff --git a/net/smc/smc_stats.c b/net/smc/smc_stats.c index 72119d3d8558..b3d279d29c52 100644 --- a/net/smc/smc_stats.c +++ b/net/smc/smc_stats.c @@ -312,3 +312,95 @@ errnest: errmsg: return skb->len; } + +static int smc_nl_get_fback_details(struct sk_buff *skb, + struct netlink_callback *cb, int pos, + bool is_srv) +{ + struct smc_nl_dmp_ctx *cb_ctx = smc_nl_dmp_ctx(cb); + int cnt_reported = cb_ctx->pos[2]; + struct smc_stats_fback *trgt_arr; + struct nlattr *attrs; + int rc = 0; + void *nlh; + + if (is_srv) + trgt_arr = &fback_rsn.srv[0]; + else + trgt_arr = &fback_rsn.clnt[0]; + if (!trgt_arr[pos].fback_code) + return -ENODATA; + nlh = genlmsg_put(skb, NETLINK_CB(cb->skb).portid, cb->nlh->nlmsg_seq, + &smc_gen_nl_family, NLM_F_MULTI, + SMC_NETLINK_GET_FBACK_STATS); + if (!nlh) + goto errmsg; + attrs = nla_nest_start(skb, SMC_GEN_FBACK_STATS); + if (!attrs) + goto errout; + if (nla_put_u8(skb, SMC_NLA_FBACK_STATS_TYPE, is_srv)) + goto errattr; + if (!cnt_reported) { + if (nla_put_u64_64bit(skb, SMC_NLA_FBACK_STATS_SRV_CNT, + fback_rsn.srv_fback_cnt, + SMC_NLA_FBACK_STATS_PAD)) + goto errattr; + if (nla_put_u64_64bit(skb, SMC_NLA_FBACK_STATS_CLNT_CNT, + fback_rsn.clnt_fback_cnt, + SMC_NLA_FBACK_STATS_PAD)) + goto errattr; + cnt_reported = 1; + } + + if (nla_put_u32(skb, SMC_NLA_FBACK_STATS_RSN_CODE, + trgt_arr[pos].fback_code)) + goto errattr; + if (nla_put_u16(skb, SMC_NLA_FBACK_STATS_RSN_CNT, + trgt_arr[pos].count)) + goto errattr; + + cb_ctx->pos[2] = cnt_reported; + nla_nest_end(skb, attrs); + genlmsg_end(skb, nlh); + return rc; + +errattr: + nla_nest_cancel(skb, attrs); +errout: + genlmsg_cancel(skb, nlh); +errmsg: + return -EMSGSIZE; +} + +int smc_nl_get_fback_stats(struct sk_buff *skb, struct netlink_callback *cb) +{ + struct smc_nl_dmp_ctx *cb_ctx = smc_nl_dmp_ctx(cb); + int rc_srv = 0, rc_clnt = 0, k; + int skip_serv = cb_ctx->pos[1]; + int snum = cb_ctx->pos[0]; + bool is_srv = true; + + mutex_lock(&smc_stat_fback_rsn); + for (k = 0; k < SMC_MAX_FBACK_RSN_CNT; k++) { + if (k < snum) + continue; + if (!skip_serv) { + rc_srv = smc_nl_get_fback_details(skb, cb, k, is_srv); + if (rc_srv && rc_srv != ENODATA) + break; + } else { + skip_serv = 0; + } + rc_clnt = smc_nl_get_fback_details(skb, cb, k, !is_srv); + if (rc_clnt && rc_clnt != ENODATA) { + skip_serv = 1; + break; + } + if (rc_clnt == ENODATA && rc_srv == ENODATA) + break; + } + mutex_unlock(&smc_stat_fback_rsn); + cb_ctx->pos[1] = skip_serv; + cb_ctx->pos[0] = k; + return skb->len; +} diff --git a/net/smc/smc_stats.h b/net/smc/smc_stats.h index 84baaca59eaf..7c35b22d9e29 100644 --- a/net/smc/smc_stats.h +++ b/net/smc/smc_stats.h @@ -248,6 +248,7 @@ do { \ while (0) int smc_nl_get_stats(struct sk_buff *skb, struct netlink_callback *cb); +int smc_nl_get_fback_stats(struct sk_buff *skb, struct netlink_callback *cb); int smc_stats_init(void) __init; void smc_stats_exit(void); -- cgit v1.2.3 From 660a59369e1ed96dd0ff1d9d73bad5b48aa50884 Mon Sep 17 00:00:00 2001 From: Bob Pearson Date: Mon, 7 Jun 2021 23:25:44 -0500 Subject: RDMA/rxe: Add bind MW fields to rxe_send_wr Add fields to struct rxe_send_wr in rdma_user_rxe.h to support bind MW work requests Link: https://lore.kernel.org/r/20210608042552.33275-2-rpearsonhpe@gmail.com Signed-off-by: Bob Pearson Signed-off-by: Jason Gunthorpe --- include/uapi/rdma/rdma_user_rxe.h | 10 ++++++++++ 1 file changed, 10 insertions(+) (limited to 'include/uapi') diff --git a/include/uapi/rdma/rdma_user_rxe.h b/include/uapi/rdma/rdma_user_rxe.h index 068433e2229d..e283c2220aba 100644 --- a/include/uapi/rdma/rdma_user_rxe.h +++ b/include/uapi/rdma/rdma_user_rxe.h @@ -99,7 +99,16 @@ struct rxe_send_wr { __u32 remote_qkey; __u16 pkey_index; } ud; + struct { + __aligned_u64 addr; + __aligned_u64 length; + __u32 mr_lkey; + __u32 mw_rkey; + __u32 rkey; + __u32 access; + } mw; /* reg is only used by the kernel and is not part of the uapi */ +#ifdef __KERNEL__ struct { union { struct ib_mr *mr; @@ -108,6 +117,7 @@ struct rxe_send_wr { __u32 key; __u32 access; } reg; +#endif } wr; }; -- cgit v1.2.3 From 836382dc24717af203ce06703530528827086955 Mon Sep 17 00:00:00 2001 From: Pablo Neira Ayuso Date: Wed, 16 Jun 2021 22:25:05 +0200 Subject: netfilter: nf_tables: add last expression Add a new optional expression that tells you when last matching on a given rule / set element element has happened. Signed-off-by: Pablo Neira Ayuso --- include/net/netfilter/nf_tables_core.h | 1 + include/uapi/linux/netfilter/nf_tables.h | 15 ++++++ net/netfilter/Makefile | 2 +- net/netfilter/nf_tables_core.c | 1 + net/netfilter/nft_last.c | 87 ++++++++++++++++++++++++++++++++ 5 files changed, 105 insertions(+), 1 deletion(-) create mode 100644 net/netfilter/nft_last.c (limited to 'include/uapi') diff --git a/include/net/netfilter/nf_tables_core.h b/include/net/netfilter/nf_tables_core.h index 46c8d5bb5d8d..0fa5a6d98a00 100644 --- a/include/net/netfilter/nf_tables_core.h +++ b/include/net/netfilter/nf_tables_core.h @@ -16,6 +16,7 @@ extern struct nft_expr_type nft_range_type; extern struct nft_expr_type nft_meta_type; extern struct nft_expr_type nft_rt_type; extern struct nft_expr_type nft_exthdr_type; +extern struct nft_expr_type nft_last_type; #ifdef CONFIG_NETWORK_SECMARK extern struct nft_object_type nft_secmark_obj_type; diff --git a/include/uapi/linux/netfilter/nf_tables.h b/include/uapi/linux/netfilter/nf_tables.h index 19715e2679d1..e94d1fa554cb 100644 --- a/include/uapi/linux/netfilter/nf_tables.h +++ b/include/uapi/linux/netfilter/nf_tables.h @@ -1195,6 +1195,21 @@ enum nft_counter_attributes { }; #define NFTA_COUNTER_MAX (__NFTA_COUNTER_MAX - 1) +/** + * enum nft_last_attributes - nf_tables last expression netlink attributes + * + * @NFTA_LAST_SET: last update has been set, zero means never updated (NLA_U32) + * @NFTA_LAST_MSECS: milliseconds since last update (NLA_U64) + */ +enum nft_last_attributes { + NFTA_LAST_UNSPEC, + NFTA_LAST_SET, + NFTA_LAST_MSECS, + NFTA_LAST_PAD, + __NFTA_LAST_MAX +}; +#define NFTA_LAST_MAX (__NFTA_LAST_MAX - 1) + /** * enum nft_log_attributes - nf_tables log expression netlink attributes * diff --git a/net/netfilter/Makefile b/net/netfilter/Makefile index 87112dad1fd4..049890e00a3d 100644 --- a/net/netfilter/Makefile +++ b/net/netfilter/Makefile @@ -74,7 +74,7 @@ obj-$(CONFIG_NF_DUP_NETDEV) += nf_dup_netdev.o nf_tables-objs := nf_tables_core.o nf_tables_api.o nft_chain_filter.o \ nf_tables_trace.o nft_immediate.o nft_cmp.o nft_range.o \ nft_bitwise.o nft_byteorder.o nft_payload.o nft_lookup.o \ - nft_dynset.o nft_meta.o nft_rt.o nft_exthdr.o \ + nft_dynset.o nft_meta.o nft_rt.o nft_exthdr.o nft_last.o \ nft_chain_route.o nf_tables_offload.o \ nft_set_hash.o nft_set_bitmap.o nft_set_rbtree.o \ nft_set_pipapo.o diff --git a/net/netfilter/nf_tables_core.c b/net/netfilter/nf_tables_core.c index 7780342e2f2d..866cfba04d6c 100644 --- a/net/netfilter/nf_tables_core.c +++ b/net/netfilter/nf_tables_core.c @@ -268,6 +268,7 @@ static struct nft_expr_type *nft_basic_types[] = { &nft_meta_type, &nft_rt_type, &nft_exthdr_type, + &nft_last_type, }; static struct nft_object_type *nft_basic_objects[] = { diff --git a/net/netfilter/nft_last.c b/net/netfilter/nft_last.c new file mode 100644 index 000000000000..913ac45167f2 --- /dev/null +++ b/net/netfilter/nft_last.c @@ -0,0 +1,87 @@ +// SPDX-License-Identifier: GPL-2.0-only +#include +#include +#include +#include +#include +#include +#include +#include + +struct nft_last_priv { + unsigned long last_jiffies; + unsigned int last_set; +}; + +static const struct nla_policy nft_last_policy[NFTA_LAST_MAX + 1] = { + [NFTA_LAST_SET] = { .type = NLA_U32 }, + [NFTA_LAST_MSECS] = { .type = NLA_U64 }, +}; + +static int nft_last_init(const struct nft_ctx *ctx, const struct nft_expr *expr, + const struct nlattr * const tb[]) +{ + struct nft_last_priv *priv = nft_expr_priv(expr); + u64 last_jiffies; + int err; + + if (tb[NFTA_LAST_MSECS]) { + err = nf_msecs_to_jiffies64(tb[NFTA_LAST_MSECS], &last_jiffies); + if (err < 0) + return err; + + priv->last_jiffies = jiffies + (unsigned long)last_jiffies; + priv->last_set = 1; + } + + return 0; +} + +static void nft_last_eval(const struct nft_expr *expr, + struct nft_regs *regs, const struct nft_pktinfo *pkt) +{ + struct nft_last_priv *priv = nft_expr_priv(expr); + + priv->last_jiffies = jiffies; + priv->last_set = 1; +} + +static int nft_last_dump(struct sk_buff *skb, const struct nft_expr *expr) +{ + struct nft_last_priv *priv = nft_expr_priv(expr); + __be64 msecs; + + if (time_before(jiffies, priv->last_jiffies)) + priv->last_set = 0; + + if (priv->last_set) + msecs = nf_jiffies64_to_msecs(jiffies - priv->last_jiffies); + else + msecs = 0; + + if (nla_put_be32(skb, NFTA_LAST_SET, htonl(priv->last_set)) || + nla_put_be64(skb, NFTA_LAST_MSECS, msecs, NFTA_LAST_PAD)) + goto nla_put_failure; + + return 0; + +nla_put_failure: + return -1; +} + +static const struct nft_expr_ops nft_last_ops = { + .type = &nft_last_type, + .size = NFT_EXPR_SIZE(sizeof(struct nft_last_priv)), + .eval = nft_last_eval, + .init = nft_last_init, + .dump = nft_last_dump, +}; + +struct nft_expr_type nft_last_type __read_mostly = { + .name = "last", + .ops = &nft_last_ops, + .policy = nft_last_policy, + .maxattr = NFTA_LAST_MAX, + .flags = NFT_EXPR_STATEFUL, + .owner = THIS_MODULE, +}; -- cgit v1.2.3 From fe76421d1da1dcdb3a2cd8428ac40106bff28bc0 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Thu, 17 Jun 2021 10:19:54 -0600 Subject: io_uring: allow user configurable IO thread CPU affinity io-wq defaults to per-node masks for IO workers. This works fine by default, but isn't particularly handy for workloads that prefer more specific affinities, for either performance or isolation reasons. This adds IORING_REGISTER_IOWQ_AFF that allows the user to pass in a CPU mask that is then applied to IO thread workers, and an IORING_UNREGISTER_IOWQ_AFF that simply resets the masks back to the default of per-node. Note that no care is given to existing IO threads, they will need to go through a reschedule before the affinity is correct if they are already running or sleeping. Signed-off-by: Jens Axboe --- fs/io-wq.c | 17 +++++++++++++++ fs/io-wq.h | 2 ++ fs/io_uring.c | 51 +++++++++++++++++++++++++++++++++++++++++++ include/uapi/linux/io_uring.h | 4 ++++ 4 files changed, 74 insertions(+) (limited to 'include/uapi') diff --git a/fs/io-wq.c b/fs/io-wq.c index 2af8e1df4646..bb4d3ee9592e 100644 --- a/fs/io-wq.c +++ b/fs/io-wq.c @@ -1087,6 +1087,23 @@ static int io_wq_cpu_offline(unsigned int cpu, struct hlist_node *node) return __io_wq_cpu_online(wq, cpu, false); } +int io_wq_cpu_affinity(struct io_wq *wq, cpumask_var_t mask) +{ + int i; + + rcu_read_lock(); + for_each_node(i) { + struct io_wqe *wqe = wq->wqes[i]; + + if (mask) + cpumask_copy(wqe->cpu_mask, mask); + else + cpumask_copy(wqe->cpu_mask, cpumask_of_node(i)); + } + rcu_read_unlock(); + return 0; +} + static __init int io_wq_init(void) { int ret; diff --git a/fs/io-wq.h b/fs/io-wq.h index af2df0680ee2..02299cdcf55c 100644 --- a/fs/io-wq.h +++ b/fs/io-wq.h @@ -128,6 +128,8 @@ void io_wq_put_and_exit(struct io_wq *wq); void io_wq_enqueue(struct io_wq *wq, struct io_wq_work *work); void io_wq_hash_work(struct io_wq_work *work, void *val); +int io_wq_cpu_affinity(struct io_wq *wq, cpumask_var_t mask); + static inline bool io_wq_is_hashed(struct io_wq_work *work) { return work->flags & IO_WQ_WORK_HASHED; diff --git a/fs/io_uring.c b/fs/io_uring.c index d916eb2cef09..46a25a7cb70a 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -9983,6 +9983,43 @@ static int io_register_rsrc(struct io_ring_ctx *ctx, void __user *arg, return -EINVAL; } +static int io_register_iowq_aff(struct io_ring_ctx *ctx, void __user *arg, + unsigned len) +{ + struct io_uring_task *tctx = current->io_uring; + cpumask_var_t new_mask; + int ret; + + if (!tctx || !tctx->io_wq) + return -EINVAL; + + if (!alloc_cpumask_var(&new_mask, GFP_KERNEL)) + return -ENOMEM; + + cpumask_clear(new_mask); + if (len > cpumask_size()) + len = cpumask_size(); + + if (copy_from_user(new_mask, arg, len)) { + free_cpumask_var(new_mask); + return -EFAULT; + } + + ret = io_wq_cpu_affinity(tctx->io_wq, new_mask); + free_cpumask_var(new_mask); + return ret; +} + +static int io_unregister_iowq_aff(struct io_ring_ctx *ctx) +{ + struct io_uring_task *tctx = current->io_uring; + + if (!tctx || !tctx->io_wq) + return -EINVAL; + + return io_wq_cpu_affinity(tctx->io_wq, NULL); +} + static bool io_register_op_must_quiesce(int op) { switch (op) { @@ -9998,6 +10035,8 @@ static bool io_register_op_must_quiesce(int op) case IORING_REGISTER_FILES_UPDATE2: case IORING_REGISTER_BUFFERS2: case IORING_REGISTER_BUFFERS_UPDATE: + case IORING_REGISTER_IOWQ_AFF: + case IORING_UNREGISTER_IOWQ_AFF: return false; default: return true; @@ -10137,6 +10176,18 @@ static int __io_uring_register(struct io_ring_ctx *ctx, unsigned opcode, ret = io_register_rsrc_update(ctx, arg, nr_args, IORING_RSRC_BUFFER); break; + case IORING_REGISTER_IOWQ_AFF: + ret = -EINVAL; + if (!arg || !nr_args) + break; + ret = io_register_iowq_aff(ctx, arg, nr_args); + break; + case IORING_UNREGISTER_IOWQ_AFF: + ret = -EINVAL; + if (arg || nr_args) + break; + ret = io_unregister_iowq_aff(ctx); + break; default: ret = -EINVAL; break; diff --git a/include/uapi/linux/io_uring.h b/include/uapi/linux/io_uring.h index 162ff99ed2cb..f1f9ac114b51 100644 --- a/include/uapi/linux/io_uring.h +++ b/include/uapi/linux/io_uring.h @@ -306,6 +306,10 @@ enum { IORING_REGISTER_BUFFERS2 = 15, IORING_REGISTER_BUFFERS_UPDATE = 16, + /* set/clear io-wq thread affinities */ + IORING_REGISTER_IOWQ_AFF = 17, + IORING_UNREGISTER_IOWQ_AFF = 18, + /* this goes last */ IORING_REGISTER_LAST }; -- cgit v1.2.3 From 644f706719f0297bc5f65c8891de1c32f042eae5 Mon Sep 17 00:00:00 2001 From: Vitaly Kuznetsov Date: Fri, 21 May 2021 11:51:36 +0200 Subject: KVM: x86: hyper-v: Introduce KVM_CAP_HYPERV_ENFORCE_CPUID Modeled after KVM_CAP_ENFORCE_PV_FEATURE_CPUID, the new capability allows for limiting Hyper-V features to those exposed to the guest in Hyper-V CPUIDs (0x40000003, 0x40000004, ...). Signed-off-by: Vitaly Kuznetsov Signed-off-by: Paolo Bonzini Message-Id: <20210521095204.2161214-3-vkuznets@redhat.com> Signed-off-by: Paolo Bonzini --- Documentation/virt/kvm/api.rst | 11 +++++++++++ arch/x86/include/asm/kvm_host.h | 1 + arch/x86/kvm/hyperv.c | 21 +++++++++++++++++++++ arch/x86/kvm/hyperv.h | 1 + arch/x86/kvm/x86.c | 4 ++++ include/uapi/linux/kvm.h | 1 + 6 files changed, 39 insertions(+) (limited to 'include/uapi') diff --git a/Documentation/virt/kvm/api.rst b/Documentation/virt/kvm/api.rst index 7fcb2fd38f42..80154d5d98a1 100644 --- a/Documentation/virt/kvm/api.rst +++ b/Documentation/virt/kvm/api.rst @@ -6891,3 +6891,14 @@ This capability is always enabled. This capability indicates that the KVM virtual PTP service is supported in the host. A VMM can check whether the service is available to the guest on migration. + +8.33 KVM_CAP_HYPERV_ENFORCE_CPUID +----------------------------- + +Architectures: x86 + +When enabled, KVM will disable emulated Hyper-V features provided to the +guest according to the bits Hyper-V CPUID feature leaves. Otherwise, all +currently implmented Hyper-V features are provided unconditionally when +Hyper-V identification is set in the HYPERV_CPUID_INTERFACE (0x40000001) +leaf. diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index 1fdb212127c4..556a8ec89451 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -543,6 +543,7 @@ struct kvm_vcpu_hv { struct kvm_vcpu_hv_stimer stimer[HV_SYNIC_STIMER_COUNT]; DECLARE_BITMAP(stimer_pending_bitmap, HV_SYNIC_STIMER_COUNT); cpumask_t tlb_flush; + bool enforce_cpuid; }; /* Xen HVM per vcpu emulation context */ diff --git a/arch/x86/kvm/hyperv.c b/arch/x86/kvm/hyperv.c index dbd3152b1379..02b0ee189f82 100644 --- a/arch/x86/kvm/hyperv.c +++ b/arch/x86/kvm/hyperv.c @@ -1853,6 +1853,27 @@ void kvm_hv_set_cpuid(struct kvm_vcpu *vcpu) vcpu->arch.hyperv_enabled = false; } +int kvm_hv_set_enforce_cpuid(struct kvm_vcpu *vcpu, bool enforce) +{ + struct kvm_vcpu_hv *hv_vcpu; + int ret = 0; + + if (!to_hv_vcpu(vcpu)) { + if (enforce) { + ret = kvm_hv_vcpu_init(vcpu); + if (ret) + return ret; + } else { + return 0; + } + } + + hv_vcpu = to_hv_vcpu(vcpu); + hv_vcpu->enforce_cpuid = enforce; + + return ret; +} + bool kvm_hv_hypercall_enabled(struct kvm_vcpu *vcpu) { return vcpu->arch.hyperv_enabled && to_kvm_hv(vcpu->kvm)->hv_guest_os_id; diff --git a/arch/x86/kvm/hyperv.h b/arch/x86/kvm/hyperv.h index 60547d5cb6d7..730da8537d05 100644 --- a/arch/x86/kvm/hyperv.h +++ b/arch/x86/kvm/hyperv.h @@ -138,6 +138,7 @@ void kvm_hv_invalidate_tsc_page(struct kvm *kvm); void kvm_hv_init_vm(struct kvm *kvm); void kvm_hv_destroy_vm(struct kvm *kvm); void kvm_hv_set_cpuid(struct kvm_vcpu *vcpu); +int kvm_hv_set_enforce_cpuid(struct kvm_vcpu *vcpu, bool enforce); int kvm_vm_ioctl_hv_eventfd(struct kvm *kvm, struct kvm_hyperv_eventfd *args); int kvm_get_hv_cpuid(struct kvm_vcpu *vcpu, struct kvm_cpuid2 *cpuid, struct kvm_cpuid_entry2 __user *entries); diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 63e48738764e..475376a97419 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -3955,6 +3955,7 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) case KVM_CAP_HYPERV_TLBFLUSH: case KVM_CAP_HYPERV_SEND_IPI: case KVM_CAP_HYPERV_CPUID: + case KVM_CAP_HYPERV_ENFORCE_CPUID: case KVM_CAP_SYS_HYPERV_CPUID: case KVM_CAP_PCI_SEGMENT: case KVM_CAP_DEBUGREGS: @@ -4878,6 +4879,9 @@ static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu, return static_call(kvm_x86_enable_direct_tlbflush)(vcpu); + case KVM_CAP_HYPERV_ENFORCE_CPUID: + return kvm_hv_set_enforce_cpuid(vcpu, cap->args[0]); + case KVM_CAP_ENFORCE_PV_FEATURE_CPUID: vcpu->arch.pv_cpuid.enforce = cap->args[0]; if (vcpu->arch.pv_cpuid.enforce) diff --git a/include/uapi/linux/kvm.h b/include/uapi/linux/kvm.h index 79d9c44d1ad7..792816144092 100644 --- a/include/uapi/linux/kvm.h +++ b/include/uapi/linux/kvm.h @@ -1083,6 +1083,7 @@ struct kvm_ppc_resize_hpt { #define KVM_CAP_SGX_ATTRIBUTE 196 #define KVM_CAP_VM_COPY_ENC_CONTEXT_FROM 197 #define KVM_CAP_PTP_KVM 198 +#define KVM_CAP_HYPERV_ENFORCE_CPUID 199 #ifdef KVM_CAP_IRQ_ROUTING -- cgit v1.2.3 From 6dba940352038b56db9b591b172fb2ec76a5fd5e Mon Sep 17 00:00:00 2001 From: Maxim Levitsky Date: Mon, 7 Jun 2021 12:02:02 +0300 Subject: KVM: x86: Introduce KVM_GET_SREGS2 / KVM_SET_SREGS2 This is a new version of KVM_GET_SREGS / KVM_SET_SREGS. It has the following changes: * Has flags for future extensions * Has vcpu's PDPTRs, allowing to save/restore them on migration. * Lacks obsolete interrupt bitmap (done now via KVM_SET_VCPU_EVENTS) New capability, KVM_CAP_SREGS2 is added to signal the userspace of this ioctl. Signed-off-by: Maxim Levitsky Message-Id: <20210607090203.133058-8-mlevitsk@redhat.com> Signed-off-by: Paolo Bonzini --- Documentation/virt/kvm/api.rst | 48 ++++++++++++++ arch/x86/include/uapi/asm/kvm.h | 13 ++++ arch/x86/kvm/kvm_cache_regs.h | 5 ++ arch/x86/kvm/x86.c | 142 ++++++++++++++++++++++++++++++++-------- include/uapi/linux/kvm.h | 4 ++ 5 files changed, 185 insertions(+), 27 deletions(-) (limited to 'include/uapi') diff --git a/Documentation/virt/kvm/api.rst b/Documentation/virt/kvm/api.rst index 80154d5d98a1..cded99561adf 100644 --- a/Documentation/virt/kvm/api.rst +++ b/Documentation/virt/kvm/api.rst @@ -5034,6 +5034,54 @@ see KVM_XEN_VCPU_SET_ATTR above. The KVM_XEN_VCPU_ATTR_TYPE_RUNSTATE_ADJUST type may not be used with the KVM_XEN_VCPU_GET_ATTR ioctl. + +4.131 KVM_GET_SREGS2 +------------------ + +:Capability: KVM_CAP_SREGS2 +:Architectures: x86 +:Type: vcpu ioctl +:Parameters: struct kvm_sregs2 (out) +:Returns: 0 on success, -1 on error + +Reads special registers from the vcpu. +This ioctl (when supported) replaces the KVM_GET_SREGS. + +:: + +struct kvm_sregs2 { + /* out (KVM_GET_SREGS2) / in (KVM_SET_SREGS2) */ + struct kvm_segment cs, ds, es, fs, gs, ss; + struct kvm_segment tr, ldt; + struct kvm_dtable gdt, idt; + __u64 cr0, cr2, cr3, cr4, cr8; + __u64 efer; + __u64 apic_base; + __u64 flags; + __u64 pdptrs[4]; +}; + +flags values for ``kvm_sregs2``: + +``KVM_SREGS2_FLAGS_PDPTRS_VALID`` + + Indicates thats the struct contain valid PDPTR values. + + +4.132 KVM_SET_SREGS2 +------------------ + +:Capability: KVM_CAP_SREGS2 +:Architectures: x86 +:Type: vcpu ioctl +:Parameters: struct kvm_sregs2 (in) +:Returns: 0 on success, -1 on error + +Writes special registers into the vcpu. +See KVM_GET_SREGS2 for the data structures. +This ioctl (when supported) replaces the KVM_SET_SREGS. + + 5. The kvm_run structure ======================== diff --git a/arch/x86/include/uapi/asm/kvm.h b/arch/x86/include/uapi/asm/kvm.h index 0662f644aad9..a6c327f8ad9e 100644 --- a/arch/x86/include/uapi/asm/kvm.h +++ b/arch/x86/include/uapi/asm/kvm.h @@ -159,6 +159,19 @@ struct kvm_sregs { __u64 interrupt_bitmap[(KVM_NR_INTERRUPTS + 63) / 64]; }; +struct kvm_sregs2 { + /* out (KVM_GET_SREGS2) / in (KVM_SET_SREGS2) */ + struct kvm_segment cs, ds, es, fs, gs, ss; + struct kvm_segment tr, ldt; + struct kvm_dtable gdt, idt; + __u64 cr0, cr2, cr3, cr4, cr8; + __u64 efer; + __u64 apic_base; + __u64 flags; + __u64 pdptrs[4]; +}; +#define KVM_SREGS2_FLAGS_PDPTRS_VALID 1 + /* for KVM_GET_FPU and KVM_SET_FPU */ struct kvm_fpu { __u8 fpr[8][16]; diff --git a/arch/x86/kvm/kvm_cache_regs.h b/arch/x86/kvm/kvm_cache_regs.h index 296d67f689ef..90e1ffdc05b7 100644 --- a/arch/x86/kvm/kvm_cache_regs.h +++ b/arch/x86/kvm/kvm_cache_regs.h @@ -125,6 +125,11 @@ static inline u64 kvm_pdptr_read(struct kvm_vcpu *vcpu, int index) return vcpu->arch.walk_mmu->pdptrs[index]; } +static inline void kvm_pdptr_write(struct kvm_vcpu *vcpu, int index, u64 value) +{ + vcpu->arch.walk_mmu->pdptrs[index] = value; +} + static inline ulong kvm_read_cr0_bits(struct kvm_vcpu *vcpu, ulong mask) { ulong tmask = mask & KVM_POSSIBLE_CR0_GUEST_BITS; diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 188c180d9f6e..8085ab830c80 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -114,6 +114,9 @@ static void __kvm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags); static void store_regs(struct kvm_vcpu *vcpu); static int sync_regs(struct kvm_vcpu *vcpu); +static int __set_sregs2(struct kvm_vcpu *vcpu, struct kvm_sregs2 *sregs2); +static void __get_sregs2(struct kvm_vcpu *vcpu, struct kvm_sregs2 *sregs2); + struct kvm_x86_ops kvm_x86_ops __read_mostly; EXPORT_SYMBOL_GPL(kvm_x86_ops); @@ -817,7 +820,6 @@ int load_pdptrs(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu, unsigned long cr3) memcpy(mmu->pdptrs, pdpte, sizeof(mmu->pdptrs)); kvm_register_mark_dirty(vcpu, VCPU_EXREG_PDPTR); - out: return ret; @@ -3956,6 +3958,7 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) case KVM_CAP_SGX_ATTRIBUTE: #endif case KVM_CAP_VM_COPY_ENC_CONTEXT_FROM: + case KVM_CAP_SREGS2: r = 1; break; case KVM_CAP_SET_GUEST_DEBUG2: @@ -4870,6 +4873,7 @@ long kvm_arch_vcpu_ioctl(struct file *filp, void __user *argp = (void __user *)arg; int r; union { + struct kvm_sregs2 *sregs2; struct kvm_lapic_state *lapic; struct kvm_xsave *xsave; struct kvm_xcrs *xcrs; @@ -5242,6 +5246,28 @@ long kvm_arch_vcpu_ioctl(struct file *filp, break; } #endif + case KVM_GET_SREGS2: { + u.sregs2 = kzalloc(sizeof(struct kvm_sregs2), GFP_KERNEL); + r = -ENOMEM; + if (!u.sregs2) + goto out; + __get_sregs2(vcpu, u.sregs2); + r = -EFAULT; + if (copy_to_user(argp, u.sregs2, sizeof(struct kvm_sregs2))) + goto out; + r = 0; + break; + } + case KVM_SET_SREGS2: { + u.sregs2 = memdup_user(argp, sizeof(struct kvm_sregs2)); + if (IS_ERR(u.sregs2)) { + r = PTR_ERR(u.sregs2); + u.sregs2 = NULL; + goto out; + } + r = __set_sregs2(vcpu, u.sregs2); + break; + } default: r = -EINVAL; } @@ -9937,7 +9963,7 @@ void kvm_get_cs_db_l_bits(struct kvm_vcpu *vcpu, int *db, int *l) } EXPORT_SYMBOL_GPL(kvm_get_cs_db_l_bits); -static void __get_sregs(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs) +static void __get_sregs_common(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs) { struct desc_ptr dt; @@ -9970,14 +9996,36 @@ skip_protected_regs: sregs->cr8 = kvm_get_cr8(vcpu); sregs->efer = vcpu->arch.efer; sregs->apic_base = kvm_get_apic_base(vcpu); +} - memset(sregs->interrupt_bitmap, 0, sizeof(sregs->interrupt_bitmap)); +static void __get_sregs(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs) +{ + __get_sregs_common(vcpu, sregs); + + if (vcpu->arch.guest_state_protected) + return; if (vcpu->arch.interrupt.injected && !vcpu->arch.interrupt.soft) set_bit(vcpu->arch.interrupt.nr, (unsigned long *)sregs->interrupt_bitmap); } +static void __get_sregs2(struct kvm_vcpu *vcpu, struct kvm_sregs2 *sregs2) +{ + int i; + + __get_sregs_common(vcpu, (struct kvm_sregs *)sregs2); + + if (vcpu->arch.guest_state_protected) + return; + + if (is_pae_paging(vcpu)) { + for (i = 0 ; i < 4 ; i++) + sregs2->pdptrs[i] = kvm_pdptr_read(vcpu, i); + sregs2->flags |= KVM_SREGS2_FLAGS_PDPTRS_VALID; + } +} + int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs) { @@ -10096,24 +10144,23 @@ static bool kvm_is_valid_sregs(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs) return kvm_is_valid_cr4(vcpu, sregs->cr4); } -static int __set_sregs(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs) +static int __set_sregs_common(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs, + int *mmu_reset_needed, bool update_pdptrs) { struct msr_data apic_base_msr; - int mmu_reset_needed = 0; - int pending_vec, max_bits, idx; + int idx; struct desc_ptr dt; - int ret = -EINVAL; if (!kvm_is_valid_sregs(vcpu, sregs)) - goto out; + return -EINVAL; apic_base_msr.data = sregs->apic_base; apic_base_msr.host_initiated = true; if (kvm_set_apic_base(vcpu, &apic_base_msr)) - goto out; + return -EINVAL; if (vcpu->arch.guest_state_protected) - goto skip_protected_regs; + return 0; dt.size = sregs->idt.limit; dt.address = sregs->idt.base; @@ -10123,31 +10170,30 @@ static int __set_sregs(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs) static_call(kvm_x86_set_gdt)(vcpu, &dt); vcpu->arch.cr2 = sregs->cr2; - mmu_reset_needed |= kvm_read_cr3(vcpu) != sregs->cr3; + *mmu_reset_needed |= kvm_read_cr3(vcpu) != sregs->cr3; vcpu->arch.cr3 = sregs->cr3; kvm_register_mark_available(vcpu, VCPU_EXREG_CR3); kvm_set_cr8(vcpu, sregs->cr8); - mmu_reset_needed |= vcpu->arch.efer != sregs->efer; + *mmu_reset_needed |= vcpu->arch.efer != sregs->efer; static_call(kvm_x86_set_efer)(vcpu, sregs->efer); - mmu_reset_needed |= kvm_read_cr0(vcpu) != sregs->cr0; + *mmu_reset_needed |= kvm_read_cr0(vcpu) != sregs->cr0; static_call(kvm_x86_set_cr0)(vcpu, sregs->cr0); vcpu->arch.cr0 = sregs->cr0; - mmu_reset_needed |= kvm_read_cr4(vcpu) != sregs->cr4; + *mmu_reset_needed |= kvm_read_cr4(vcpu) != sregs->cr4; static_call(kvm_x86_set_cr4)(vcpu, sregs->cr4); - idx = srcu_read_lock(&vcpu->kvm->srcu); - if (is_pae_paging(vcpu)) { - load_pdptrs(vcpu, vcpu->arch.walk_mmu, kvm_read_cr3(vcpu)); - mmu_reset_needed = 1; + if (update_pdptrs) { + idx = srcu_read_lock(&vcpu->kvm->srcu); + if (is_pae_paging(vcpu)) { + load_pdptrs(vcpu, vcpu->arch.walk_mmu, kvm_read_cr3(vcpu)); + *mmu_reset_needed = 1; + } + srcu_read_unlock(&vcpu->kvm->srcu, idx); } - srcu_read_unlock(&vcpu->kvm->srcu, idx); - - if (mmu_reset_needed) - kvm_mmu_reset_context(vcpu); kvm_set_segment(vcpu, &sregs->cs, VCPU_SREG_CS); kvm_set_segment(vcpu, &sregs->ds, VCPU_SREG_DS); @@ -10167,20 +10213,62 @@ static int __set_sregs(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs) !is_protmode(vcpu)) vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE; -skip_protected_regs: + return 0; +} + +static int __set_sregs(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs) +{ + int pending_vec, max_bits; + int mmu_reset_needed = 0; + int ret = __set_sregs_common(vcpu, sregs, &mmu_reset_needed, true); + + if (ret) + return ret; + + if (mmu_reset_needed) + kvm_mmu_reset_context(vcpu); + max_bits = KVM_NR_INTERRUPTS; pending_vec = find_first_bit( (const unsigned long *)sregs->interrupt_bitmap, max_bits); + if (pending_vec < max_bits) { kvm_queue_interrupt(vcpu, pending_vec, false); pr_debug("Set back pending irq %d\n", pending_vec); + kvm_make_request(KVM_REQ_EVENT, vcpu); } + return 0; +} - kvm_make_request(KVM_REQ_EVENT, vcpu); +static int __set_sregs2(struct kvm_vcpu *vcpu, struct kvm_sregs2 *sregs2) +{ + int mmu_reset_needed = 0; + bool valid_pdptrs = sregs2->flags & KVM_SREGS2_FLAGS_PDPTRS_VALID; + bool pae = (sregs2->cr0 & X86_CR0_PG) && (sregs2->cr4 & X86_CR4_PAE) && + !(sregs2->efer & EFER_LMA); + int i, ret; - ret = 0; -out: - return ret; + if (sregs2->flags & ~KVM_SREGS2_FLAGS_PDPTRS_VALID) + return -EINVAL; + + if (valid_pdptrs && (!pae || vcpu->arch.guest_state_protected)) + return -EINVAL; + + ret = __set_sregs_common(vcpu, (struct kvm_sregs *)sregs2, + &mmu_reset_needed, !valid_pdptrs); + if (ret) + return ret; + + if (valid_pdptrs) { + for (i = 0; i < 4 ; i++) + kvm_pdptr_write(vcpu, i, sregs2->pdptrs[i]); + + kvm_register_mark_dirty(vcpu, VCPU_EXREG_PDPTR); + mmu_reset_needed = 1; + } + if (mmu_reset_needed) + kvm_mmu_reset_context(vcpu); + return 0; } int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu, diff --git a/include/uapi/linux/kvm.h b/include/uapi/linux/kvm.h index 792816144092..90d44138dbfb 100644 --- a/include/uapi/linux/kvm.h +++ b/include/uapi/linux/kvm.h @@ -1084,6 +1084,7 @@ struct kvm_ppc_resize_hpt { #define KVM_CAP_VM_COPY_ENC_CONTEXT_FROM 197 #define KVM_CAP_PTP_KVM 198 #define KVM_CAP_HYPERV_ENFORCE_CPUID 199 +#define KVM_CAP_SREGS2 200 #ifdef KVM_CAP_IRQ_ROUTING @@ -1622,6 +1623,9 @@ struct kvm_xen_hvm_attr { #define KVM_XEN_VCPU_GET_ATTR _IOWR(KVMIO, 0xca, struct kvm_xen_vcpu_attr) #define KVM_XEN_VCPU_SET_ATTR _IOW(KVMIO, 0xcb, struct kvm_xen_vcpu_attr) +#define KVM_GET_SREGS2 _IOR(KVMIO, 0xcc, struct kvm_sregs2) +#define KVM_SET_SREGS2 _IOW(KVMIO, 0xcd, struct kvm_sregs2) + struct kvm_xen_vcpu_attr { __u16 type; __u16 pad[3]; -- cgit v1.2.3 From 0dbb11230437895f7cd6fc55da61cef011e997d8 Mon Sep 17 00:00:00 2001 From: Ashish Kalra Date: Tue, 8 Jun 2021 18:05:43 +0000 Subject: KVM: X86: Introduce KVM_HC_MAP_GPA_RANGE hypercall This hypercall is used by the SEV guest to notify a change in the page encryption status to the hypervisor. The hypercall should be invoked only when the encryption attribute is changed from encrypted -> decrypted and vice versa. By default all guest pages are considered encrypted. The hypercall exits to userspace to manage the guest shared regions and integrate with the userspace VMM's migration code. Cc: Thomas Gleixner Cc: Ingo Molnar Cc: "H. Peter Anvin" Cc: Paolo Bonzini Cc: Joerg Roedel Cc: Borislav Petkov Cc: Tom Lendacky Cc: x86@kernel.org Cc: kvm@vger.kernel.org Cc: linux-kernel@vger.kernel.org Reviewed-by: Steve Rutherford Signed-off-by: Brijesh Singh Signed-off-by: Ashish Kalra Co-developed-by: Sean Christopherson Signed-off-by: Sean Christopherson Co-developed-by: Paolo Bonzini Signed-off-by: Paolo Bonzini Message-Id: <90778988e1ee01926ff9cac447aacb745f954c8c.1623174621.git.ashish.kalra@amd.com> Signed-off-by: Paolo Bonzini --- Documentation/virt/kvm/api.rst | 19 +++++++++++++++ Documentation/virt/kvm/cpuid.rst | 7 ++++++ Documentation/virt/kvm/hypercalls.rst | 21 ++++++++++++++++ Documentation/virt/kvm/msr.rst | 13 ++++++++++ arch/x86/include/asm/kvm_host.h | 2 ++ arch/x86/include/uapi/asm/kvm_para.h | 13 ++++++++++ arch/x86/kvm/x86.c | 46 +++++++++++++++++++++++++++++++++++ include/uapi/linux/kvm.h | 1 + include/uapi/linux/kvm_para.h | 1 + 9 files changed, 123 insertions(+) (limited to 'include/uapi') diff --git a/Documentation/virt/kvm/api.rst b/Documentation/virt/kvm/api.rst index cded99561adf..e328caa35d6c 100644 --- a/Documentation/virt/kvm/api.rst +++ b/Documentation/virt/kvm/api.rst @@ -6950,3 +6950,22 @@ guest according to the bits Hyper-V CPUID feature leaves. Otherwise, all currently implmented Hyper-V features are provided unconditionally when Hyper-V identification is set in the HYPERV_CPUID_INTERFACE (0x40000001) leaf. + +8.34 KVM_CAP_EXIT_HYPERCALL +--------------------------- + +:Capability: KVM_CAP_EXIT_HYPERCALL +:Architectures: x86 +:Type: vm + +This capability, if enabled, will cause KVM to exit to userspace +with KVM_EXIT_HYPERCALL exit reason to process some hypercalls. + +Calling KVM_CHECK_EXTENSION for this capability will return a bitmask +of hypercalls that can be configured to exit to userspace. +Right now, the only such hypercall is KVM_HC_MAP_GPA_RANGE. + +The argument to KVM_ENABLE_CAP is also a bitmask, and must be a subset +of the result of KVM_CHECK_EXTENSION. KVM will forward to userspace +the hypercalls whose corresponding bit is in the argument, and return +ENOSYS for the others. diff --git a/Documentation/virt/kvm/cpuid.rst b/Documentation/virt/kvm/cpuid.rst index cf62162d4be2..bda3e3e737d7 100644 --- a/Documentation/virt/kvm/cpuid.rst +++ b/Documentation/virt/kvm/cpuid.rst @@ -96,6 +96,13 @@ KVM_FEATURE_MSI_EXT_DEST_ID 15 guest checks this feature bit before using extended destination ID bits in MSI address bits 11-5. +KVM_FEATURE_HC_MAP_GPA_RANGE 16 guest checks this feature bit before + using the map gpa range hypercall + to notify the page state change + +KVM_FEATURE_MIGRATION_CONTROL 17 guest checks this feature bit before + using MSR_KVM_MIGRATION_CONTROL + KVM_FEATURE_CLOCKSOURCE_STABLE_BIT 24 host will warn if no guest-side per-cpu warps are expected in kvmclock diff --git a/Documentation/virt/kvm/hypercalls.rst b/Documentation/virt/kvm/hypercalls.rst index ed4fddd364ea..e56fa8b9cfca 100644 --- a/Documentation/virt/kvm/hypercalls.rst +++ b/Documentation/virt/kvm/hypercalls.rst @@ -169,3 +169,24 @@ a0: destination APIC ID :Usage example: When sending a call-function IPI-many to vCPUs, yield if any of the IPI target vCPUs was preempted. + +8. KVM_HC_MAP_GPA_RANGE +------------------------- +:Architecture: x86 +:Status: active +:Purpose: Request KVM to map a GPA range with the specified attributes. + +a0: the guest physical address of the start page +a1: the number of (4kb) pages (must be contiguous in GPA space) +a2: attributes + + Where 'attributes' : + * bits 3:0 - preferred page size encoding 0 = 4kb, 1 = 2mb, 2 = 1gb, etc... + * bit 4 - plaintext = 0, encrypted = 1 + * bits 63:5 - reserved (must be zero) + +**Implementation note**: this hypercall is implemented in userspace via +the KVM_CAP_EXIT_HYPERCALL capability. Userspace must enable that capability +before advertising KVM_FEATURE_HC_MAP_GPA_RANGE in the guest CPUID. In +addition, if the guest supports KVM_FEATURE_MIGRATION_CONTROL, userspace +must also set up an MSR filter to process writes to MSR_KVM_MIGRATION_CONTROL. diff --git a/Documentation/virt/kvm/msr.rst b/Documentation/virt/kvm/msr.rst index e37a14c323d2..9315fc385fb0 100644 --- a/Documentation/virt/kvm/msr.rst +++ b/Documentation/virt/kvm/msr.rst @@ -376,3 +376,16 @@ data: write '1' to bit 0 of the MSR, this causes the host to re-scan its queue and check if there are more notifications pending. The MSR is available if KVM_FEATURE_ASYNC_PF_INT is present in CPUID. + +MSR_KVM_MIGRATION_CONTROL: + 0x4b564d08 + +data: + This MSR is available if KVM_FEATURE_MIGRATION_CONTROL is present in + CPUID. Bit 0 represents whether live migration of the guest is allowed. + + When a guest is started, bit 0 will be 0 if the guest has encrypted + memory and 1 if the guest does not have encrypted memory. If the + guest is communicating page encryption status to the host using the + ``KVM_HC_MAP_GPA_RANGE`` hypercall, it can set bit 0 in this MSR to + allow live migration of the guest. diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index a0c29e29dd48..e11d64aa0bcd 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -1087,6 +1087,8 @@ struct kvm_arch { u32 user_space_msr_mask; struct kvm_x86_msr_filter __rcu *msr_filter; + u32 hypercall_exit_enabled; + /* Guest can access the SGX PROVISIONKEY. */ bool sgx_provisioning_allowed; diff --git a/arch/x86/include/uapi/asm/kvm_para.h b/arch/x86/include/uapi/asm/kvm_para.h index 950afebfba88..5146bbab84d4 100644 --- a/arch/x86/include/uapi/asm/kvm_para.h +++ b/arch/x86/include/uapi/asm/kvm_para.h @@ -33,6 +33,8 @@ #define KVM_FEATURE_PV_SCHED_YIELD 13 #define KVM_FEATURE_ASYNC_PF_INT 14 #define KVM_FEATURE_MSI_EXT_DEST_ID 15 +#define KVM_FEATURE_HC_MAP_GPA_RANGE 16 +#define KVM_FEATURE_MIGRATION_CONTROL 17 #define KVM_HINTS_REALTIME 0 @@ -54,6 +56,7 @@ #define MSR_KVM_POLL_CONTROL 0x4b564d05 #define MSR_KVM_ASYNC_PF_INT 0x4b564d06 #define MSR_KVM_ASYNC_PF_ACK 0x4b564d07 +#define MSR_KVM_MIGRATION_CONTROL 0x4b564d08 struct kvm_steal_time { __u64 steal; @@ -90,6 +93,16 @@ struct kvm_clock_pairing { /* MSR_KVM_ASYNC_PF_INT */ #define KVM_ASYNC_PF_VEC_MASK GENMASK(7, 0) +/* MSR_KVM_MIGRATION_CONTROL */ +#define KVM_MIGRATION_READY (1 << 0) + +/* KVM_HC_MAP_GPA_RANGE */ +#define KVM_MAP_GPA_RANGE_PAGE_SZ_4K 0 +#define KVM_MAP_GPA_RANGE_PAGE_SZ_2M (1 << 0) +#define KVM_MAP_GPA_RANGE_PAGE_SZ_1G (1 << 1) +#define KVM_MAP_GPA_RANGE_ENC_STAT(n) (n << 4) +#define KVM_MAP_GPA_RANGE_ENCRYPTED KVM_MAP_GPA_RANGE_ENC_STAT(1) +#define KVM_MAP_GPA_RANGE_DECRYPTED KVM_MAP_GPA_RANGE_ENC_STAT(0) /* Operations for KVM_HC_MMU_OP */ #define KVM_MMU_OP_WRITE_PTE 1 diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index ceb60f64085c..8b898ec8d349 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -103,6 +103,8 @@ static u64 __read_mostly efer_reserved_bits = ~((u64)EFER_SCE); static u64 __read_mostly cr4_reserved_bits = CR4_RESERVED_BITS; +#define KVM_EXIT_HYPERCALL_VALID_MASK (1 << KVM_HC_MAP_GPA_RANGE) + #define KVM_X2APIC_API_VALID_FLAGS (KVM_X2APIC_API_USE_32BIT_IDS | \ KVM_X2APIC_API_DISABLE_BROADCAST_QUIRK) @@ -3996,6 +3998,9 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) case KVM_CAP_SREGS2: r = 1; break; + case KVM_CAP_EXIT_HYPERCALL: + r = KVM_EXIT_HYPERCALL_VALID_MASK; + break; case KVM_CAP_SET_GUEST_DEBUG2: return KVM_GUESTDBG_VALID_MASK; #ifdef CONFIG_KVM_XEN @@ -5622,6 +5627,14 @@ split_irqchip_unlock: if (kvm_x86_ops.vm_copy_enc_context_from) r = kvm_x86_ops.vm_copy_enc_context_from(kvm, cap->args[0]); return r; + case KVM_CAP_EXIT_HYPERCALL: + if (cap->args[0] & ~KVM_EXIT_HYPERCALL_VALID_MASK) { + r = -EINVAL; + break; + } + kvm->arch.hypercall_exit_enabled = cap->args[0]; + r = 0; + break; default: r = -EINVAL; break; @@ -8548,6 +8561,17 @@ no_yield: return; } +static int complete_hypercall_exit(struct kvm_vcpu *vcpu) +{ + u64 ret = vcpu->run->hypercall.ret; + + if (!is_64_bit_mode(vcpu)) + ret = (u32)ret; + kvm_rax_write(vcpu, ret); + ++vcpu->stat.hypercalls; + return kvm_skip_emulated_instruction(vcpu); +} + int kvm_emulate_hypercall(struct kvm_vcpu *vcpu) { unsigned long nr, a0, a1, a2, a3, ret; @@ -8613,6 +8637,28 @@ int kvm_emulate_hypercall(struct kvm_vcpu *vcpu) kvm_sched_yield(vcpu, a0); ret = 0; break; + case KVM_HC_MAP_GPA_RANGE: { + u64 gpa = a0, npages = a1, attrs = a2; + + ret = -KVM_ENOSYS; + if (!(vcpu->kvm->arch.hypercall_exit_enabled & (1 << KVM_HC_MAP_GPA_RANGE))) + break; + + if (!PAGE_ALIGNED(gpa) || !npages || + gpa_to_gfn(gpa) + npages <= gpa_to_gfn(gpa)) { + ret = -KVM_EINVAL; + break; + } + + vcpu->run->exit_reason = KVM_EXIT_HYPERCALL; + vcpu->run->hypercall.nr = KVM_HC_MAP_GPA_RANGE; + vcpu->run->hypercall.args[0] = gpa; + vcpu->run->hypercall.args[1] = npages; + vcpu->run->hypercall.args[2] = attrs; + vcpu->run->hypercall.longmode = op_64_bit; + vcpu->arch.complete_userspace_io = complete_hypercall_exit; + return 0; + } default: ret = -KVM_ENOSYS; break; diff --git a/include/uapi/linux/kvm.h b/include/uapi/linux/kvm.h index 90d44138dbfb..9febe1412f7a 100644 --- a/include/uapi/linux/kvm.h +++ b/include/uapi/linux/kvm.h @@ -1085,6 +1085,7 @@ struct kvm_ppc_resize_hpt { #define KVM_CAP_PTP_KVM 198 #define KVM_CAP_HYPERV_ENFORCE_CPUID 199 #define KVM_CAP_SREGS2 200 +#define KVM_CAP_EXIT_HYPERCALL 201 #ifdef KVM_CAP_IRQ_ROUTING diff --git a/include/uapi/linux/kvm_para.h b/include/uapi/linux/kvm_para.h index 8b86609849b9..960c7e93d1a9 100644 --- a/include/uapi/linux/kvm_para.h +++ b/include/uapi/linux/kvm_para.h @@ -29,6 +29,7 @@ #define KVM_HC_CLOCK_PAIRING 9 #define KVM_HC_SEND_IPI 10 #define KVM_HC_SCHED_YIELD 11 +#define KVM_HC_MAP_GPA_RANGE 12 /* * hypercalls use architecture specific -- cgit v1.2.3 From 2d8ea148e553e1dd4e80a87741abdfb229e2b323 Mon Sep 17 00:00:00 2001 From: Jian Shen Date: Thu, 17 Jun 2021 11:37:11 +0800 Subject: net: fix mistake path for netdev_features_strings Th_strings arrays netdev_features_strings, tunable_strings, and phy_tunable_strings has been moved to file net/ethtool/common.c. So fixes the comment. Signed-off-by: Jian Shen Signed-off-by: David S. Miller --- include/linux/netdev_features.h | 2 +- include/uapi/linux/ethtool.h | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) (limited to 'include/uapi') diff --git a/include/linux/netdev_features.h b/include/linux/netdev_features.h index 3de38d6a0aea..2c6b9e416225 100644 --- a/include/linux/netdev_features.h +++ b/include/linux/netdev_features.h @@ -93,7 +93,7 @@ enum { /* * Add your fresh new feature above and remember to update - * netdev_features_strings[] in net/core/ethtool.c and maybe + * netdev_features_strings[] in net/ethtool/common.c and maybe * some feature mask #defines below. Please also describe it * in Documentation/networking/netdev-features.rst. */ diff --git a/include/uapi/linux/ethtool.h b/include/uapi/linux/ethtool.h index cfef6b08169a..67aa7134b301 100644 --- a/include/uapi/linux/ethtool.h +++ b/include/uapi/linux/ethtool.h @@ -233,7 +233,7 @@ enum tunable_id { ETHTOOL_PFC_PREVENTION_TOUT, /* timeout in msecs */ /* * Add your fresh new tunable attribute above and remember to update - * tunable_strings[] in net/core/ethtool.c + * tunable_strings[] in net/ethtool/common.c */ __ETHTOOL_TUNABLE_COUNT, }; @@ -297,7 +297,7 @@ enum phy_tunable_id { ETHTOOL_PHY_EDPD, /* * Add your fresh new phy tunable attribute above and remember to update - * phy_tunable_strings[] in net/core/ethtool.c + * phy_tunable_strings[] in net/ethtool/common.c */ __ETHTOOL_PHY_TUNABLE_COUNT, }; -- cgit v1.2.3 From 68f5d3f3b6543266b29e047cfaf9842333019b4c Mon Sep 17 00:00:00 2001 From: Johannes Berg Date: Fri, 5 Mar 2021 13:19:58 +0100 Subject: um: add PCI over virtio emulation driver To support testing of PCI/PCIe drivers in UML, add a PCI bus support driver. This driver uses virtio, which in UML is really just vhost-user, to talk to devices, and adds the devices to the virtual PCI bus in the system. Since virtio already allows DMA/bus mastering this really isn't all that hard, of course we need the logic_iomem infrastructure that was added by a previous patch. The protocol to talk to the device is has a few fairly simple messages for reading to/writing from config and IO spaces, and messages for the device to send the various interrupts (INT#, MSI/MSI-X and while suspended PME#). Note that currently no offical virtio device ID is assigned for this protocol, as a consequence this patch requires defining it in the Kconfig, with a default that makes the driver refuse to work at all. Finally, in order to add support for MSI/MSI-X interrupts, some small changes are needed in the UML IRQ code, it needs to have more interrupts, changing NR_IRQS from 64 to 128 if this driver is enabled, but not actually use them for anything so that the generic IRQ domain/MSI infrastructure can allocate IRQ numbers. Signed-off-by: Johannes Berg Signed-off-by: Richard Weinberger --- arch/um/Kconfig | 13 +- arch/um/drivers/Kconfig | 20 + arch/um/drivers/Makefile | 1 + arch/um/drivers/virt-pci.c | 885 +++++++++++++++++++++++++++++++++++++ arch/um/include/asm/Kbuild | 1 - arch/um/include/asm/io.h | 7 + arch/um/include/asm/irq.h | 8 +- arch/um/include/asm/msi.h | 1 + arch/um/include/asm/pci.h | 39 ++ arch/um/kernel/Makefile | 1 + arch/um/kernel/ioport.c | 13 + arch/um/kernel/irq.c | 7 +- include/uapi/linux/virtio_pcidev.h | 64 +++ 13 files changed, 1054 insertions(+), 6 deletions(-) create mode 100644 arch/um/drivers/virt-pci.c create mode 100644 arch/um/include/asm/msi.h create mode 100644 arch/um/include/asm/pci.h create mode 100644 arch/um/kernel/ioport.c create mode 100644 include/uapi/linux/virtio_pcidev.h (limited to 'include/uapi') diff --git a/arch/um/Kconfig b/arch/um/Kconfig index e72b4b393367..328ba973c07d 100644 --- a/arch/um/Kconfig +++ b/arch/um/Kconfig @@ -15,7 +15,7 @@ config UML select HAVE_FUTEX_CMPXCHG if FUTEX select HAVE_DEBUG_KMEMLEAK select HAVE_DEBUG_BUGVERBOSE - select NO_DMA + select NO_DMA if !UML_DMA_EMULATION select GENERIC_IRQ_SHOW select GENERIC_CPU_DEVICES select HAVE_GCC_PLUGINS @@ -26,10 +26,21 @@ config MMU bool default y +config UML_DMA_EMULATION + bool + config NO_IOMEM bool "disable IOMEM" if EXPERT + depends on !INDIRECT_IOMEM default y +config UML_IOMEM_EMULATION + bool + select INDIRECT_IOMEM + select GENERIC_PCI_IOMAP + select GENERIC_IOMAP + select NO_GENERIC_PCI_IOPORT_MAP + config NO_IOPORT_MAP def_bool y diff --git a/arch/um/drivers/Kconfig b/arch/um/drivers/Kconfig index 03ba34b61115..f145842c40b9 100644 --- a/arch/um/drivers/Kconfig +++ b/arch/um/drivers/Kconfig @@ -357,3 +357,23 @@ config UML_RTC rtcwake, especially in time-travel mode. This driver enables that by providing a fake RTC clock that causes a wakeup at the right time. + +config UML_PCI_OVER_VIRTIO + bool "Enable PCI over VIRTIO device simulation" + # in theory, just VIRTIO is enough, but that causes recursion + depends on VIRTIO_UML + select FORCE_PCI + select UML_IOMEM_EMULATION + select UML_DMA_EMULATION + select PCI_MSI + select PCI_MSI_IRQ_DOMAIN + select PCI_LOCKLESS_CONFIG + +config UML_PCI_OVER_VIRTIO_DEVICE_ID + int "set the virtio device ID for PCI emulation" + default -1 + depends on UML_PCI_OVER_VIRTIO + help + There's no official device ID assigned (yet), set the one you + wish to use for experimentation here. The default of -1 is + not valid and will cause the driver to fail at probe. diff --git a/arch/um/drivers/Makefile b/arch/um/drivers/Makefile index dcc64a02f81f..803666e85414 100644 --- a/arch/um/drivers/Makefile +++ b/arch/um/drivers/Makefile @@ -64,6 +64,7 @@ obj-$(CONFIG_BLK_DEV_COW_COMMON) += cow_user.o obj-$(CONFIG_UML_RANDOM) += random.o obj-$(CONFIG_VIRTIO_UML) += virtio_uml.o obj-$(CONFIG_UML_RTC) += rtc.o +obj-$(CONFIG_UML_PCI_OVER_VIRTIO) += virt-pci.o # pcap_user.o must be added explicitly. USER_OBJS := fd.o null.o pty.o tty.o xterm.o slip_common.o pcap_user.o vde_user.o vector_user.o diff --git a/arch/um/drivers/virt-pci.c b/arch/um/drivers/virt-pci.c new file mode 100644 index 000000000000..dd85f36197aa --- /dev/null +++ b/arch/um/drivers/virt-pci.c @@ -0,0 +1,885 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2020 Intel Corporation + * Author: Johannes Berg + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define MAX_DEVICES 8 +#define MAX_MSI_VECTORS 32 +#define CFG_SPACE_SIZE 4096 + +/* for MSI-X we have a 32-bit payload */ +#define MAX_IRQ_MSG_SIZE (sizeof(struct virtio_pcidev_msg) + sizeof(u32)) +#define NUM_IRQ_MSGS 10 + +#define HANDLE_NO_FREE(ptr) ((void *)((unsigned long)(ptr) | 1)) +#define HANDLE_IS_NO_FREE(ptr) ((unsigned long)(ptr) & 1) + +struct um_pci_device { + struct virtio_device *vdev; + + /* for now just standard BARs */ + u8 resptr[PCI_STD_NUM_BARS]; + + struct virtqueue *cmd_vq, *irq_vq; + +#define UM_PCI_STAT_WAITING 0 + unsigned long status; + + int irq; +}; + +struct um_pci_device_reg { + struct um_pci_device *dev; + void __iomem *iomem; +}; + +static struct pci_host_bridge *bridge; +static DEFINE_MUTEX(um_pci_mtx); +static struct um_pci_device_reg um_pci_devices[MAX_DEVICES]; +static struct fwnode_handle *um_pci_fwnode; +static struct irq_domain *um_pci_inner_domain; +static struct irq_domain *um_pci_msi_domain; +static unsigned long um_pci_msi_used[BITS_TO_LONGS(MAX_MSI_VECTORS)]; + +#define UM_VIRT_PCI_MAXDELAY 40000 + +static int um_pci_send_cmd(struct um_pci_device *dev, + struct virtio_pcidev_msg *cmd, + unsigned int cmd_size, + const void *extra, unsigned int extra_size, + void *out, unsigned int out_size) +{ + struct scatterlist out_sg, extra_sg, in_sg; + struct scatterlist *sgs_list[] = { + [0] = &out_sg, + [1] = extra ? &extra_sg : &in_sg, + [2] = extra ? &in_sg : NULL, + }; + int delay_count = 0; + int ret, len; + bool posted; + + if (WARN_ON(cmd_size < sizeof(*cmd))) + return -EINVAL; + + switch (cmd->op) { + case VIRTIO_PCIDEV_OP_CFG_WRITE: + case VIRTIO_PCIDEV_OP_MMIO_WRITE: + case VIRTIO_PCIDEV_OP_MMIO_MEMSET: + /* in PCI, writes are posted, so don't wait */ + posted = !out; + WARN_ON(!posted); + break; + default: + posted = false; + break; + } + + if (posted) { + u8 *ncmd = kmalloc(cmd_size + extra_size, GFP_ATOMIC); + + if (ncmd) { + memcpy(ncmd, cmd, cmd_size); + if (extra) + memcpy(ncmd + cmd_size, extra, extra_size); + cmd = (void *)ncmd; + cmd_size += extra_size; + extra = NULL; + extra_size = 0; + } else { + /* try without allocating memory */ + posted = false; + } + } + + sg_init_one(&out_sg, cmd, cmd_size); + if (extra) + sg_init_one(&extra_sg, extra, extra_size); + if (out) + sg_init_one(&in_sg, out, out_size); + + /* add to internal virtio queue */ + ret = virtqueue_add_sgs(dev->cmd_vq, sgs_list, + extra ? 2 : 1, + out ? 1 : 0, + posted ? cmd : HANDLE_NO_FREE(cmd), + GFP_ATOMIC); + if (ret) + return ret; + + if (posted) { + virtqueue_kick(dev->cmd_vq); + return 0; + } + + /* kick and poll for getting a response on the queue */ + set_bit(UM_PCI_STAT_WAITING, &dev->status); + virtqueue_kick(dev->cmd_vq); + + while (1) { + void *completed = virtqueue_get_buf(dev->cmd_vq, &len); + + if (completed == HANDLE_NO_FREE(cmd)) + break; + + if (WARN_ONCE(virtqueue_is_broken(dev->cmd_vq) || + ++delay_count > UM_VIRT_PCI_MAXDELAY, + "um virt-pci delay: %d", delay_count)) { + ret = -EIO; + break; + } + udelay(1); + } + clear_bit(UM_PCI_STAT_WAITING, &dev->status); + + return ret; +} + +static unsigned long um_pci_cfgspace_read(void *priv, unsigned int offset, + int size) +{ + struct um_pci_device_reg *reg = priv; + struct um_pci_device *dev = reg->dev; + struct virtio_pcidev_msg hdr = { + .op = VIRTIO_PCIDEV_OP_CFG_READ, + .size = size, + .addr = offset, + }; + /* maximum size - we may only use parts of it */ + u8 data[8]; + + if (!dev) + return ~0ULL; + + memset(data, 0xff, sizeof(data)); + + switch (size) { + case 1: + case 2: + case 4: +#ifdef CONFIG_64BIT + case 8: +#endif + break; + default: + WARN(1, "invalid config space read size %d\n", size); + return ~0ULL; + } + + if (um_pci_send_cmd(dev, &hdr, sizeof(hdr), NULL, 0, + data, sizeof(data))) + return ~0ULL; + + switch (size) { + case 1: + return data[0]; + case 2: + return le16_to_cpup((void *)data); + case 4: + return le32_to_cpup((void *)data); +#ifdef CONFIG_64BIT + case 8: + return le64_to_cpup((void *)data); +#endif + default: + return ~0ULL; + } +} + +static void um_pci_cfgspace_write(void *priv, unsigned int offset, int size, + unsigned long val) +{ + struct um_pci_device_reg *reg = priv; + struct um_pci_device *dev = reg->dev; + struct { + struct virtio_pcidev_msg hdr; + /* maximum size - we may only use parts of it */ + u8 data[8]; + } msg = { + .hdr = { + .op = VIRTIO_PCIDEV_OP_CFG_WRITE, + .size = size, + .addr = offset, + }, + }; + + if (!dev) + return; + + switch (size) { + case 1: + msg.data[0] = (u8)val; + break; + case 2: + put_unaligned_le16(val, (void *)msg.data); + break; + case 4: + put_unaligned_le32(val, (void *)msg.data); + break; +#ifdef CONFIG_64BIT + case 8: + put_unaligned_le64(val, (void *)msg.data); + break; +#endif + default: + WARN(1, "invalid config space write size %d\n", size); + return; + } + + WARN_ON(um_pci_send_cmd(dev, &msg.hdr, sizeof(msg), NULL, 0, NULL, 0)); +} + +static const struct logic_iomem_ops um_pci_device_cfgspace_ops = { + .read = um_pci_cfgspace_read, + .write = um_pci_cfgspace_write, +}; + +static void um_pci_bar_copy_from(void *priv, void *buffer, + unsigned int offset, int size) +{ + u8 *resptr = priv; + struct um_pci_device *dev = container_of(resptr - *resptr, + struct um_pci_device, + resptr[0]); + struct virtio_pcidev_msg hdr = { + .op = VIRTIO_PCIDEV_OP_MMIO_READ, + .bar = *resptr, + .size = size, + .addr = offset, + }; + + memset(buffer, 0xff, size); + + um_pci_send_cmd(dev, &hdr, sizeof(hdr), NULL, 0, buffer, size); +} + +static unsigned long um_pci_bar_read(void *priv, unsigned int offset, + int size) +{ + /* maximum size - we may only use parts of it */ + u8 data[8]; + + switch (size) { + case 1: + case 2: + case 4: +#ifdef CONFIG_64BIT + case 8: +#endif + break; + default: + WARN(1, "invalid config space read size %d\n", size); + return ~0ULL; + } + + um_pci_bar_copy_from(priv, data, offset, size); + + switch (size) { + case 1: + return data[0]; + case 2: + return le16_to_cpup((void *)data); + case 4: + return le32_to_cpup((void *)data); +#ifdef CONFIG_64BIT + case 8: + return le64_to_cpup((void *)data); +#endif + default: + return ~0ULL; + } +} + +static void um_pci_bar_copy_to(void *priv, unsigned int offset, + const void *buffer, int size) +{ + u8 *resptr = priv; + struct um_pci_device *dev = container_of(resptr - *resptr, + struct um_pci_device, + resptr[0]); + struct virtio_pcidev_msg hdr = { + .op = VIRTIO_PCIDEV_OP_MMIO_WRITE, + .bar = *resptr, + .size = size, + .addr = offset, + }; + + um_pci_send_cmd(dev, &hdr, sizeof(hdr), buffer, size, NULL, 0); +} + +static void um_pci_bar_write(void *priv, unsigned int offset, int size, + unsigned long val) +{ + /* maximum size - we may only use parts of it */ + u8 data[8]; + + switch (size) { + case 1: + data[0] = (u8)val; + break; + case 2: + put_unaligned_le16(val, (void *)data); + break; + case 4: + put_unaligned_le32(val, (void *)data); + break; +#ifdef CONFIG_64BIT + case 8: + put_unaligned_le64(val, (void *)data); + break; +#endif + default: + WARN(1, "invalid config space write size %d\n", size); + return; + } + + um_pci_bar_copy_to(priv, offset, data, size); +} + +static void um_pci_bar_set(void *priv, unsigned int offset, u8 value, int size) +{ + u8 *resptr = priv; + struct um_pci_device *dev = container_of(resptr - *resptr, + struct um_pci_device, + resptr[0]); + struct { + struct virtio_pcidev_msg hdr; + u8 data; + } msg = { + .hdr = { + .op = VIRTIO_PCIDEV_OP_CFG_WRITE, + .bar = *resptr, + .size = size, + .addr = offset, + }, + .data = value, + }; + + um_pci_send_cmd(dev, &msg.hdr, sizeof(msg), NULL, 0, NULL, 0); +} + +static const struct logic_iomem_ops um_pci_device_bar_ops = { + .read = um_pci_bar_read, + .write = um_pci_bar_write, + .set = um_pci_bar_set, + .copy_from = um_pci_bar_copy_from, + .copy_to = um_pci_bar_copy_to, +}; + +static void __iomem *um_pci_map_bus(struct pci_bus *bus, unsigned int devfn, + int where) +{ + struct um_pci_device_reg *dev; + unsigned int busn = bus->number; + + if (busn > 0) + return NULL; + + /* not allowing functions for now ... */ + if (devfn % 8) + return NULL; + + if (devfn / 8 >= ARRAY_SIZE(um_pci_devices)) + return NULL; + + dev = &um_pci_devices[devfn / 8]; + if (!dev) + return NULL; + + return (void __iomem *)((unsigned long)dev->iomem + where); +} + +static struct pci_ops um_pci_ops = { + .map_bus = um_pci_map_bus, + .read = pci_generic_config_read, + .write = pci_generic_config_write, +}; + +static void um_pci_rescan(void) +{ + pci_lock_rescan_remove(); + pci_rescan_bus(bridge->bus); + pci_unlock_rescan_remove(); +} + +static void um_pci_irq_vq_addbuf(struct virtqueue *vq, void *buf, bool kick) +{ + struct scatterlist sg[1]; + + sg_init_one(sg, buf, MAX_IRQ_MSG_SIZE); + if (virtqueue_add_inbuf(vq, sg, 1, buf, GFP_ATOMIC)) + kfree(buf); + else if (kick) + virtqueue_kick(vq); +} + +static void um_pci_handle_irq_message(struct virtqueue *vq, + struct virtio_pcidev_msg *msg) +{ + struct virtio_device *vdev = vq->vdev; + struct um_pci_device *dev = vdev->priv; + + /* we should properly chain interrupts, but on ARCH=um we don't care */ + + switch (msg->op) { + case VIRTIO_PCIDEV_OP_INT: + generic_handle_irq(dev->irq); + break; + case VIRTIO_PCIDEV_OP_MSI: + /* our MSI message is just the interrupt number */ + if (msg->size == sizeof(u32)) + generic_handle_irq(le32_to_cpup((void *)msg->data)); + else + generic_handle_irq(le16_to_cpup((void *)msg->data)); + break; + case VIRTIO_PCIDEV_OP_PME: + /* nothing to do - we already woke up due to the message */ + break; + default: + dev_err(&vdev->dev, "unexpected virt-pci message %d\n", msg->op); + break; + } +} + +static void um_pci_cmd_vq_cb(struct virtqueue *vq) +{ + struct virtio_device *vdev = vq->vdev; + struct um_pci_device *dev = vdev->priv; + void *cmd; + int len; + + if (test_bit(UM_PCI_STAT_WAITING, &dev->status)) + return; + + while ((cmd = virtqueue_get_buf(vq, &len))) { + if (WARN_ON(HANDLE_IS_NO_FREE(cmd))) + continue; + kfree(cmd); + } +} + +static void um_pci_irq_vq_cb(struct virtqueue *vq) +{ + struct virtio_pcidev_msg *msg; + int len; + + while ((msg = virtqueue_get_buf(vq, &len))) { + if (len >= sizeof(*msg)) + um_pci_handle_irq_message(vq, msg); + + /* recycle the message buffer */ + um_pci_irq_vq_addbuf(vq, msg, true); + } +} + +static int um_pci_init_vqs(struct um_pci_device *dev) +{ + struct virtqueue *vqs[2]; + static const char *const names[2] = { "cmd", "irq" }; + vq_callback_t *cbs[2] = { um_pci_cmd_vq_cb, um_pci_irq_vq_cb }; + int err, i; + + err = virtio_find_vqs(dev->vdev, 2, vqs, cbs, names, NULL); + if (err) + return err; + + dev->cmd_vq = vqs[0]; + dev->irq_vq = vqs[1]; + + for (i = 0; i < NUM_IRQ_MSGS; i++) { + void *msg = kzalloc(MAX_IRQ_MSG_SIZE, GFP_KERNEL); + + if (msg) + um_pci_irq_vq_addbuf(dev->irq_vq, msg, false); + } + + virtqueue_kick(dev->irq_vq); + + return 0; +} + +static int um_pci_virtio_probe(struct virtio_device *vdev) +{ + struct um_pci_device *dev; + int i, free = -1; + int err = -ENOSPC; + + dev = kzalloc(sizeof(*dev), GFP_KERNEL); + if (!dev) + return -ENOMEM; + + dev->vdev = vdev; + vdev->priv = dev; + + mutex_lock(&um_pci_mtx); + for (i = 0; i < MAX_DEVICES; i++) { + if (um_pci_devices[i].dev) + continue; + free = i; + break; + } + + if (free < 0) + goto error; + + err = um_pci_init_vqs(dev); + if (err) + goto error; + + dev->irq = irq_alloc_desc(numa_node_id()); + if (dev->irq < 0) { + err = dev->irq; + goto error; + } + um_pci_devices[free].dev = dev; + vdev->priv = dev; + + mutex_unlock(&um_pci_mtx); + + device_set_wakeup_enable(&vdev->dev, true); + + um_pci_rescan(); + return 0; +error: + mutex_unlock(&um_pci_mtx); + kfree(dev); + return err; +} + +static void um_pci_virtio_remove(struct virtio_device *vdev) +{ + struct um_pci_device *dev = vdev->priv; + int i; + + /* Stop all virtqueues */ + vdev->config->reset(vdev); + vdev->config->del_vqs(vdev); + + device_set_wakeup_enable(&vdev->dev, false); + + mutex_lock(&um_pci_mtx); + for (i = 0; i < MAX_DEVICES; i++) { + if (um_pci_devices[i].dev != dev) + continue; + um_pci_devices[i].dev = NULL; + irq_free_desc(dev->irq); + } + mutex_unlock(&um_pci_mtx); + + um_pci_rescan(); + + kfree(dev); +} + +static struct virtio_device_id id_table[] = { + { CONFIG_UML_PCI_OVER_VIRTIO_DEVICE_ID, VIRTIO_DEV_ANY_ID }, + { 0 }, +}; +MODULE_DEVICE_TABLE(virtio, id_table); + +static struct virtio_driver um_pci_virtio_driver = { + .driver.name = "virtio-pci", + .driver.owner = THIS_MODULE, + .id_table = id_table, + .probe = um_pci_virtio_probe, + .remove = um_pci_virtio_remove, +}; + +static struct resource virt_cfgspace_resource = { + .name = "PCI config space", + .start = 0xf0000000 - MAX_DEVICES * CFG_SPACE_SIZE, + .end = 0xf0000000 - 1, + .flags = IORESOURCE_MEM, +}; + +static long um_pci_map_cfgspace(unsigned long offset, size_t size, + const struct logic_iomem_ops **ops, + void **priv) +{ + if (WARN_ON(size > CFG_SPACE_SIZE || offset % CFG_SPACE_SIZE)) + return -EINVAL; + + if (offset / CFG_SPACE_SIZE < MAX_DEVICES) { + *ops = &um_pci_device_cfgspace_ops; + *priv = &um_pci_devices[offset / CFG_SPACE_SIZE]; + return 0; + } + + WARN(1, "cannot map offset 0x%lx/0x%zx\n", offset, size); + return -ENOENT; +} + +static const struct logic_iomem_region_ops um_pci_cfgspace_ops = { + .map = um_pci_map_cfgspace, +}; + +static struct resource virt_iomem_resource = { + .name = "PCI iomem", + .start = 0xf0000000, + .end = 0xffffffff, + .flags = IORESOURCE_MEM, +}; + +struct um_pci_map_iomem_data { + unsigned long offset; + size_t size; + const struct logic_iomem_ops **ops; + void **priv; + long ret; +}; + +static int um_pci_map_iomem_walk(struct pci_dev *pdev, void *_data) +{ + struct um_pci_map_iomem_data *data = _data; + struct um_pci_device_reg *reg = &um_pci_devices[pdev->devfn / 8]; + struct um_pci_device *dev; + int i; + + if (!reg->dev) + return 0; + + for (i = 0; i < ARRAY_SIZE(dev->resptr); i++) { + struct resource *r = &pdev->resource[i]; + + if ((r->flags & IORESOURCE_TYPE_BITS) != IORESOURCE_MEM) + continue; + + /* + * must be the whole or part of the resource, + * not allowed to only overlap + */ + if (data->offset < r->start || data->offset > r->end) + continue; + if (data->offset + data->size - 1 > r->end) + continue; + + dev = reg->dev; + *data->ops = &um_pci_device_bar_ops; + dev->resptr[i] = i; + *data->priv = &dev->resptr[i]; + data->ret = data->offset - r->start; + + /* no need to continue */ + return 1; + } + + return 0; +} + +static long um_pci_map_iomem(unsigned long offset, size_t size, + const struct logic_iomem_ops **ops, + void **priv) +{ + struct um_pci_map_iomem_data data = { + /* we want the full address here */ + .offset = offset + virt_iomem_resource.start, + .size = size, + .ops = ops, + .priv = priv, + .ret = -ENOENT, + }; + + pci_walk_bus(bridge->bus, um_pci_map_iomem_walk, &data); + return data.ret; +} + +static const struct logic_iomem_region_ops um_pci_iomem_ops = { + .map = um_pci_map_iomem, +}; + +static void um_pci_compose_msi_msg(struct irq_data *data, struct msi_msg *msg) +{ + /* + * This is a very low address and not actually valid 'physical' memory + * in UML, so we can simply map MSI(-X) vectors to there, it cannot be + * legitimately written to by the device in any other way. + * We use the (virtual) IRQ number here as the message to simplify the + * code that receives the message, where for now we simply trust the + * device to send the correct message. + */ + msg->address_hi = 0; + msg->address_lo = 0xa0000; + msg->data = data->irq; +} + +static struct irq_chip um_pci_msi_bottom_irq_chip = { + .name = "UM virtio MSI", + .irq_compose_msi_msg = um_pci_compose_msi_msg, +}; + +static int um_pci_inner_domain_alloc(struct irq_domain *domain, + unsigned int virq, unsigned int nr_irqs, + void *args) +{ + unsigned long bit; + + WARN_ON(nr_irqs != 1); + + mutex_lock(&um_pci_mtx); + bit = find_first_zero_bit(um_pci_msi_used, MAX_MSI_VECTORS); + if (bit >= MAX_MSI_VECTORS) { + mutex_unlock(&um_pci_mtx); + return -ENOSPC; + } + + set_bit(bit, um_pci_msi_used); + mutex_unlock(&um_pci_mtx); + + irq_domain_set_info(domain, virq, bit, &um_pci_msi_bottom_irq_chip, + domain->host_data, handle_simple_irq, + NULL, NULL); + + return 0; +} + +static void um_pci_inner_domain_free(struct irq_domain *domain, + unsigned int virq, unsigned int nr_irqs) +{ + struct irq_data *d = irq_domain_get_irq_data(domain, virq); + + mutex_lock(&um_pci_mtx); + + if (!test_bit(d->hwirq, um_pci_msi_used)) + pr_err("trying to free unused MSI#%lu\n", d->hwirq); + else + __clear_bit(d->hwirq, um_pci_msi_used); + + mutex_unlock(&um_pci_mtx); +} + +static const struct irq_domain_ops um_pci_inner_domain_ops = { + .alloc = um_pci_inner_domain_alloc, + .free = um_pci_inner_domain_free, +}; + +static struct irq_chip um_pci_msi_irq_chip = { + .name = "UM virtio PCIe MSI", + .irq_mask = pci_msi_mask_irq, + .irq_unmask = pci_msi_unmask_irq, +}; + +static struct msi_domain_info um_pci_msi_domain_info = { + .flags = MSI_FLAG_USE_DEF_DOM_OPS | + MSI_FLAG_USE_DEF_CHIP_OPS | + MSI_FLAG_PCI_MSIX, + .chip = &um_pci_msi_irq_chip, +}; + +static struct resource busn_resource = { + .name = "PCI busn", + .start = 0, + .end = 0, + .flags = IORESOURCE_BUS, +}; + +static int um_pci_map_irq(const struct pci_dev *pdev, u8 slot, u8 pin) +{ + struct um_pci_device_reg *reg = &um_pci_devices[pdev->devfn / 8]; + + if (WARN_ON(!reg->dev)) + return -EINVAL; + + /* Yes, we map all pins to the same IRQ ... doesn't matter for now. */ + return reg->dev->irq; +} + +void *pci_root_bus_fwnode(struct pci_bus *bus) +{ + return um_pci_fwnode; +} + +int um_pci_init(void) +{ + int err, i; + + WARN_ON(logic_iomem_add_region(&virt_cfgspace_resource, + &um_pci_cfgspace_ops)); + WARN_ON(logic_iomem_add_region(&virt_iomem_resource, + &um_pci_iomem_ops)); + + if (WARN(CONFIG_UML_PCI_OVER_VIRTIO_DEVICE_ID < 0, + "No virtio device ID configured for PCI - no PCI support\n")) + return 0; + + bridge = pci_alloc_host_bridge(0); + if (!bridge) + return -ENOMEM; + + um_pci_fwnode = irq_domain_alloc_named_fwnode("um-pci"); + if (!um_pci_fwnode) { + err = -ENOMEM; + goto free; + } + + um_pci_inner_domain = __irq_domain_add(um_pci_fwnode, MAX_MSI_VECTORS, + MAX_MSI_VECTORS, 0, + &um_pci_inner_domain_ops, NULL); + if (!um_pci_inner_domain) { + err = -ENOMEM; + goto free; + } + + um_pci_msi_domain = pci_msi_create_irq_domain(um_pci_fwnode, + &um_pci_msi_domain_info, + um_pci_inner_domain); + if (!um_pci_msi_domain) { + err = -ENOMEM; + goto free; + } + + pci_add_resource(&bridge->windows, &virt_iomem_resource); + pci_add_resource(&bridge->windows, &busn_resource); + bridge->ops = &um_pci_ops; + bridge->map_irq = um_pci_map_irq; + + for (i = 0; i < MAX_DEVICES; i++) { + resource_size_t start; + + start = virt_cfgspace_resource.start + i * CFG_SPACE_SIZE; + um_pci_devices[i].iomem = ioremap(start, CFG_SPACE_SIZE); + if (WARN(!um_pci_devices[i].iomem, "failed to map %d\n", i)) { + err = -ENOMEM; + goto free; + } + } + + err = pci_host_probe(bridge); + if (err) + goto free; + + err = register_virtio_driver(&um_pci_virtio_driver); + if (err) + goto free; + return 0; +free: + if (um_pci_inner_domain) + irq_domain_remove(um_pci_inner_domain); + if (um_pci_fwnode) + irq_domain_free_fwnode(um_pci_fwnode); + pci_free_resource_list(&bridge->windows); + pci_free_host_bridge(bridge); + return err; +} +module_init(um_pci_init); + +void um_pci_exit(void) +{ + unregister_virtio_driver(&um_pci_virtio_driver); + irq_domain_remove(um_pci_msi_domain); + irq_domain_remove(um_pci_inner_domain); + pci_free_resource_list(&bridge->windows); + pci_free_host_bridge(bridge); +} +module_exit(um_pci_exit); diff --git a/arch/um/include/asm/Kbuild b/arch/um/include/asm/Kbuild index 10b7228b3aee..0c31d19a7a9c 100644 --- a/arch/um/include/asm/Kbuild +++ b/arch/um/include/asm/Kbuild @@ -18,7 +18,6 @@ generic-y += mcs_spinlock.h generic-y += mmiowb.h generic-y += module.lds.h generic-y += param.h -generic-y += pci.h generic-y += percpu.h generic-y += preempt.h generic-y += softirq_stack.h diff --git a/arch/um/include/asm/io.h b/arch/um/include/asm/io.h index 6ce18d343997..9ea42cc746d9 100644 --- a/arch/um/include/asm/io.h +++ b/arch/um/include/asm/io.h @@ -3,16 +3,23 @@ #define _ASM_UM_IO_H #include +/* get emulated iomem (if desired) */ +#include + +#ifndef ioremap #define ioremap ioremap static inline void __iomem *ioremap(phys_addr_t offset, size_t size) { return NULL; } +#endif /* ioremap */ +#ifndef iounmap #define iounmap iounmap static inline void iounmap(void __iomem *addr) { } +#endif /* iounmap */ #include diff --git a/arch/um/include/asm/irq.h b/arch/um/include/asm/irq.h index 3f5d3e8228fc..e187c789369d 100644 --- a/arch/um/include/asm/irq.h +++ b/arch/um/include/asm/irq.h @@ -31,7 +31,13 @@ #endif -#define NR_IRQS 64 +#define UM_LAST_SIGNAL_IRQ 64 +/* If we have (simulated) PCI MSI, allow 64 more interrupt numbers for it */ +#ifdef CONFIG_PCI_MSI +#define NR_IRQS (UM_LAST_SIGNAL_IRQ + 64) +#else +#define NR_IRQS UM_LAST_SIGNAL_IRQ +#endif /* CONFIG_PCI_MSI */ #include #endif diff --git a/arch/um/include/asm/msi.h b/arch/um/include/asm/msi.h new file mode 100644 index 000000000000..c8c6c381f394 --- /dev/null +++ b/arch/um/include/asm/msi.h @@ -0,0 +1 @@ +#include diff --git a/arch/um/include/asm/pci.h b/arch/um/include/asm/pci.h new file mode 100644 index 000000000000..da13fd5519ef --- /dev/null +++ b/arch/um/include/asm/pci.h @@ -0,0 +1,39 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef __ASM_UM_PCI_H +#define __ASM_UM_PCI_H +#include +#include + +#define PCIBIOS_MIN_IO 0 +#define PCIBIOS_MIN_MEM 0 + +#define pcibios_assign_all_busses() 1 + +extern int isa_dma_bridge_buggy; + +#ifdef CONFIG_PCI +static inline int pci_get_legacy_ide_irq(struct pci_dev *dev, int channel) +{ + /* no legacy IRQs */ + return -ENODEV; +} +#endif + +#ifdef CONFIG_PCI_DOMAINS +static inline int pci_proc_domain(struct pci_bus *bus) +{ + /* always show the domain in /proc */ + return 1; +} +#endif /* CONFIG_PCI */ + +#ifdef CONFIG_PCI_MSI_IRQ_DOMAIN +/* + * This is a bit of an annoying hack, and it assumes we only have + * the virt-pci (if anything). Which is true, but still. + */ +void *pci_root_bus_fwnode(struct pci_bus *bus); +#define pci_root_bus_fwnode pci_root_bus_fwnode +#endif + +#endif /* __ASM_UM_PCI_H */ diff --git a/arch/um/kernel/Makefile b/arch/um/kernel/Makefile index e698e0c7dbdc..2fdd00de5a72 100644 --- a/arch/um/kernel/Makefile +++ b/arch/um/kernel/Makefile @@ -23,6 +23,7 @@ obj-$(CONFIG_BLK_DEV_INITRD) += initrd.o obj-$(CONFIG_GPROF) += gprof_syms.o obj-$(CONFIG_EARLY_PRINTK) += early_printk.o obj-$(CONFIG_STACKTRACE) += stacktrace.o +obj-$(CONFIG_GENERIC_PCI_IOMAP) += ioport.o USER_OBJS := config.o diff --git a/arch/um/kernel/ioport.c b/arch/um/kernel/ioport.c new file mode 100644 index 000000000000..7220615b3beb --- /dev/null +++ b/arch/um/kernel/ioport.c @@ -0,0 +1,13 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2021 Intel Corporation + * Author: Johannes Berg + */ +#include +#include + +void __iomem *__pci_ioport_map(struct pci_dev *dev, unsigned long port, + unsigned int nr) +{ + return NULL; +} diff --git a/arch/um/kernel/irq.c b/arch/um/kernel/irq.c index f3f19da7b876..a8873d9bc28b 100644 --- a/arch/um/kernel/irq.c +++ b/arch/um/kernel/irq.c @@ -56,7 +56,7 @@ struct irq_entry { static DEFINE_SPINLOCK(irq_lock); static LIST_HEAD(active_fds); -static DECLARE_BITMAP(irqs_allocated, NR_IRQS); +static DECLARE_BITMAP(irqs_allocated, UM_LAST_SIGNAL_IRQ); static bool irqs_suspended; static void irq_io_loop(struct irq_reg *irq, struct uml_pt_regs *regs) @@ -419,7 +419,8 @@ unsigned int do_IRQ(int irq, struct uml_pt_regs *regs) void um_free_irq(int irq, void *dev) { - if (WARN(irq < 0 || irq > NR_IRQS, "freeing invalid irq %d", irq)) + if (WARN(irq < 0 || irq > UM_LAST_SIGNAL_IRQ, + "freeing invalid irq %d", irq)) return; free_irq_by_irq_and_dev(irq, dev); @@ -648,7 +649,7 @@ void __init init_IRQ(void) irq_set_chip_and_handler(TIMER_IRQ, &alarm_irq_type, handle_edge_irq); - for (i = 1; i < NR_IRQS; i++) + for (i = 1; i < UM_LAST_SIGNAL_IRQ; i++) irq_set_chip_and_handler(i, &normal_irq_type, handle_edge_irq); /* Initialize EPOLL Loop */ os_setup_epoll(); diff --git a/include/uapi/linux/virtio_pcidev.h b/include/uapi/linux/virtio_pcidev.h new file mode 100644 index 000000000000..89daa88bcfef --- /dev/null +++ b/include/uapi/linux/virtio_pcidev.h @@ -0,0 +1,64 @@ +/* SPDX-License-Identifier: ((GPL-2.0 WITH Linux-syscall-note) OR BSD-3-Clause) */ +/* + * Copyright (C) 2021 Intel Corporation + * Author: Johannes Berg + */ +#ifndef _UAPI_LINUX_VIRTIO_PCIDEV_H +#define _UAPI_LINUX_VIRTIO_PCIDEV_H +#include + +/** + * enum virtio_pcidev_ops - virtual PCI device operations + * @VIRTIO_PCIDEV_OP_CFG_READ: read config space, size is 1, 2, 4 or 8; + * the @data field should be filled in by the device (in little endian). + * @VIRTIO_PCIDEV_OP_CFG_WRITE: write config space, size is 1, 2, 4 or 8; + * the @data field contains the data to write (in little endian). + * @VIRTIO_PCIDEV_OP_BAR_READ: read BAR mem/pio, size can be variable; + * the @data field should be filled in by the device (in little endian). + * @VIRTIO_PCIDEV_OP_BAR_WRITE: write BAR mem/pio, size can be variable; + * the @data field contains the data to write (in little endian). + * @VIRTIO_PCIDEV_OP_MMIO_MEMSET: memset MMIO, size is variable but + * the @data field only has one byte (unlike @VIRTIO_PCIDEV_OP_MMIO_WRITE) + * @VIRTIO_PCIDEV_OP_INT: legacy INTx# pin interrupt, the addr field is 1-4 for + * the number + * @VIRTIO_PCIDEV_OP_MSI: MSI(-X) interrupt, this message basically transports + * the 16- or 32-bit write that would otherwise be done into memory, + * analogous to the write messages (@VIRTIO_PCIDEV_OP_MMIO_WRITE) above + * @VIRTIO_PCIDEV_OP_PME: Dummy message whose content is ignored (and should be + * all zeroes) to signal the PME# pin. + */ +enum virtio_pcidev_ops { + VIRTIO_PCIDEV_OP_RESERVED = 0, + VIRTIO_PCIDEV_OP_CFG_READ, + VIRTIO_PCIDEV_OP_CFG_WRITE, + VIRTIO_PCIDEV_OP_MMIO_READ, + VIRTIO_PCIDEV_OP_MMIO_WRITE, + VIRTIO_PCIDEV_OP_MMIO_MEMSET, + VIRTIO_PCIDEV_OP_INT, + VIRTIO_PCIDEV_OP_MSI, + VIRTIO_PCIDEV_OP_PME, +}; + +/** + * struct virtio_pcidev_msg - virtio PCI device operation + * @op: the operation to do + * @bar: the bar (only with BAR read/write messages) + * @reserved: reserved + * @size: the size of the read/write (in bytes) + * @addr: the address to read/write + * @data: the data, normally @size long, but just one byte for + * %VIRTIO_PCIDEV_OP_MMIO_MEMSET + * + * Note: the fields are all in native (CPU) endian, however, the + * @data values will often be in little endian (see the ops above.) + */ +struct virtio_pcidev_msg { + __u8 op; + __u8 bar; + __u16 reserved; + __u32 size; + __u64 addr; + __u8 data[]; +}; + +#endif /* _UAPI_LINUX_VIRTIO_PCIDEV_H */ -- cgit v1.2.3 From 8e8125f192288802267157f613c0ca654dfbde8e Mon Sep 17 00:00:00 2001 From: Yuri Nudelman Date: Tue, 25 May 2021 14:49:52 +0300 Subject: habanalabs: add debug flag to prevent failure on timeout Sometimes it is useful to allow the command to continue running despite the timeout occurred, to differentiate between really stuck or just very time consuming commands. This can be achieved by passing a new debug flag alongside the cs, HL_CS_FLAGS_SKIP_RESET_ON_TIMEOUT. Anyway, if the timeout occurred, a warning print shall be issued, however this shall not fail the submission. Signed-off-by: Yuri Nudelman Reviewed-by: Oded Gabbay Signed-off-by: Oded Gabbay --- .../misc/habanalabs/common/command_submission.c | 25 +++++++++++++++++----- drivers/misc/habanalabs/common/habanalabs.h | 5 +++++ include/uapi/misc/habanalabs.h | 1 + 3 files changed, 26 insertions(+), 5 deletions(-) (limited to 'include/uapi') diff --git a/drivers/misc/habanalabs/common/command_submission.c b/drivers/misc/habanalabs/common/command_submission.c index ecd96fbe3150..6d51f54030c1 100644 --- a/drivers/misc/habanalabs/common/command_submission.c +++ b/drivers/misc/habanalabs/common/command_submission.c @@ -556,6 +556,13 @@ out: else if (!cs->submitted) cs->fence->error = -EBUSY; + if (unlikely(cs->skip_reset_on_timeout)) { + dev_err(hdev->dev, + "Command submission %llu completed after %llu (s)\n", + cs->sequence, + div_u64(jiffies - cs->submission_time_jiffies, HZ)); + } + if (cs->timestamp) cs->fence->timestamp = ktime_get(); complete_all(&cs->fence->completion); @@ -571,6 +578,8 @@ static void cs_timedout(struct work_struct *work) int rc; struct hl_cs *cs = container_of(work, struct hl_cs, work_tdr.work); + bool skip_reset_on_timeout = cs->skip_reset_on_timeout; + rc = cs_get_unless_zero(cs); if (!rc) return; @@ -581,7 +590,8 @@ static void cs_timedout(struct work_struct *work) } /* Mark the CS is timed out so we won't try to cancel its TDR */ - cs->timedout = true; + if (likely(!skip_reset_on_timeout)) + cs->timedout = true; hdev = cs->ctx->hdev; @@ -613,10 +623,12 @@ static void cs_timedout(struct work_struct *work) cs_put(cs); - if (hdev->reset_on_lockup) - hl_device_reset(hdev, HL_RESET_TDR); - else - hdev->needs_reset = true; + if (likely(!skip_reset_on_timeout)) { + if (hdev->reset_on_lockup) + hl_device_reset(hdev, HL_RESET_TDR); + else + hdev->needs_reset = true; + } } static int allocate_cs(struct hl_device *hdev, struct hl_ctx *ctx, @@ -650,6 +662,9 @@ static int allocate_cs(struct hl_device *hdev, struct hl_ctx *ctx, cs->type = cs_type; cs->timestamp = !!(flags & HL_CS_FLAGS_TIMESTAMP); cs->timeout_jiffies = timeout; + cs->skip_reset_on_timeout = + !!(flags & HL_CS_FLAGS_SKIP_RESET_ON_TIMEOUT); + cs->submission_time_jiffies = jiffies; INIT_LIST_HEAD(&cs->job_list); INIT_DELAYED_WORK(&cs->work_tdr, cs_timedout); kref_init(&cs->refcount); diff --git a/drivers/misc/habanalabs/common/habanalabs.h b/drivers/misc/habanalabs/common/habanalabs.h index e751868b3ed3..56d2f41f8893 100644 --- a/drivers/misc/habanalabs/common/habanalabs.h +++ b/drivers/misc/habanalabs/common/habanalabs.h @@ -1421,6 +1421,7 @@ struct hl_userptr { * @staged_sequence: the sequence of the staged submission this CS is part of, * relevant only if staged_cs is set. * @timeout_jiffies: cs timeout in jiffies. + * @submission_time_jiffies: submission time of the cs * @type: CS_TYPE_*. * @submitted: true if CS was submitted to H/W. * @completed: true if CS was completed by device. @@ -1433,6 +1434,8 @@ struct hl_userptr { * @staged_first: true if this is the first staged CS and we need to receive * timeout for this CS. * @staged_cs: true if this CS is part of a staged submission. + * @skip_reset_on_timeout: true if we shall not reset the device in case + * timeout occurs (debug scenario). */ struct hl_cs { u16 *jobs_in_queue_cnt; @@ -1450,6 +1453,7 @@ struct hl_cs { u64 sequence; u64 staged_sequence; u64 timeout_jiffies; + u64 submission_time_jiffies; enum hl_cs_type type; u8 submitted; u8 completed; @@ -1460,6 +1464,7 @@ struct hl_cs { u8 staged_last; u8 staged_first; u8 staged_cs; + u8 skip_reset_on_timeout; }; /** diff --git a/include/uapi/misc/habanalabs.h b/include/uapi/misc/habanalabs.h index 6d2d34c9f375..a47485a8d411 100644 --- a/include/uapi/misc/habanalabs.h +++ b/include/uapi/misc/habanalabs.h @@ -664,6 +664,7 @@ struct hl_cs_chunk { #define HL_CS_FLAGS_STAGED_SUBMISSION_FIRST 0x80 #define HL_CS_FLAGS_STAGED_SUBMISSION_LAST 0x100 #define HL_CS_FLAGS_CUSTOM_TIMEOUT 0x200 +#define HL_CS_FLAGS_SKIP_RESET_ON_TIMEOUT 0x400 #define HL_CS_STATUS_SUCCESS 0 -- cgit v1.2.3 From e307b302be8beb7fb59aa16621d5081b69397076 Mon Sep 17 00:00:00 2001 From: Yuri Nudelman Date: Mon, 24 May 2021 11:25:21 +0300 Subject: habanalabs: added open_stats info ioctl In a system with multiple ASICs, there is a need to provide monitoring tools with information on how long a device was opened and how many times a device was opened. Therefore, we add a new opcode to the INFO ioctl to provide that information. Signed-off-by: Yuri Nudelman Reviewed-by: Oded Gabbay Signed-off-by: Oded Gabbay --- drivers/misc/habanalabs/common/device.c | 3 +++ drivers/misc/habanalabs/common/habanalabs.h | 8 ++++++++ drivers/misc/habanalabs/common/habanalabs_drv.c | 3 +++ drivers/misc/habanalabs/common/habanalabs_ioctl.c | 21 +++++++++++++++++++++ include/uapi/misc/habanalabs.h | 12 ++++++++++++ 5 files changed, 47 insertions(+) (limited to 'include/uapi') diff --git a/drivers/misc/habanalabs/common/device.c b/drivers/misc/habanalabs/common/device.c index e56f5170e338..37ce38d9a1a7 100644 --- a/drivers/misc/habanalabs/common/device.c +++ b/drivers/misc/habanalabs/common/device.c @@ -132,6 +132,9 @@ static int hl_device_release(struct inode *inode, struct file *filp) dev_warn(hdev->dev, "Device is still in use because there are live CS and/or memory mappings\n"); + hdev->last_open_session_duration_jif = + jiffies - hdev->last_successful_open_jif; + return 0; } diff --git a/drivers/misc/habanalabs/common/habanalabs.h b/drivers/misc/habanalabs/common/habanalabs.h index 244fbf209d34..6c9a81c2cfe7 100644 --- a/drivers/misc/habanalabs/common/habanalabs.h +++ b/drivers/misc/habanalabs/common/habanalabs.h @@ -2137,6 +2137,11 @@ struct hl_mmu_funcs { * the error will be ignored by the driver during * device initialization. Mainly used to debug and * workaround firmware bugs + * @last_successful_open_jif: timestamp (jiffies) of the last successful + * device open. + * @last_open_session_duration_jif: duration (jiffies) of the last device open + * session. + * @open_counter: number of successful device open operations. * @in_reset: is device in reset flow. * @curr_pll_profile: current PLL profile. * @card_type: Various ASICs have several card types. This indicates the card @@ -2259,6 +2264,9 @@ struct hl_device { u64 max_power; u64 clock_gating_mask; u64 boot_error_status_mask; + u64 last_successful_open_jif; + u64 last_open_session_duration_jif; + u64 open_counter; atomic_t in_reset; enum hl_pll_frequency curr_pll_profile; enum cpucp_card_types card_type; diff --git a/drivers/misc/habanalabs/common/habanalabs_drv.c b/drivers/misc/habanalabs/common/habanalabs_drv.c index 4d377a39df13..4194cda2d04c 100644 --- a/drivers/misc/habanalabs/common/habanalabs_drv.c +++ b/drivers/misc/habanalabs/common/habanalabs_drv.c @@ -187,6 +187,9 @@ int hl_device_open(struct inode *inode, struct file *filp) hl_debugfs_add_file(hpriv); + hdev->open_counter++; + hdev->last_successful_open_jif = jiffies; + return 0; out_err: diff --git a/drivers/misc/habanalabs/common/habanalabs_ioctl.c b/drivers/misc/habanalabs/common/habanalabs_ioctl.c index 6604d30246e6..f4dda7b4acdd 100644 --- a/drivers/misc/habanalabs/common/habanalabs_ioctl.c +++ b/drivers/misc/habanalabs/common/habanalabs_ioctl.c @@ -460,6 +460,24 @@ static int power_info(struct hl_fpriv *hpriv, struct hl_info_args *args) min((size_t) max_size, sizeof(power_info))) ? -EFAULT : 0; } +static int open_stats_info(struct hl_fpriv *hpriv, struct hl_info_args *args) +{ + struct hl_device *hdev = hpriv->hdev; + u32 max_size = args->return_size; + struct hl_open_stats_info open_stats_info = {0}; + void __user *out = (void __user *) (uintptr_t) args->return_pointer; + + if ((!max_size) || (!out)) + return -EINVAL; + + open_stats_info.last_open_period_ms = jiffies64_to_msecs( + hdev->last_open_session_duration_jif); + open_stats_info.open_counter = hdev->open_counter; + + return copy_to_user(out, &open_stats_info, + min((size_t) max_size, sizeof(open_stats_info))) ? -EFAULT : 0; +} + static int _hl_info_ioctl(struct hl_fpriv *hpriv, void *data, struct device *dev) { @@ -543,6 +561,9 @@ static int _hl_info_ioctl(struct hl_fpriv *hpriv, void *data, case HL_INFO_POWER: return power_info(hpriv, args); + case HL_INFO_OPEN_STATS: + return open_stats_info(hpriv, args); + default: dev_err(dev, "Invalid request %d\n", args->op); rc = -ENOTTY; diff --git a/include/uapi/misc/habanalabs.h b/include/uapi/misc/habanalabs.h index a47485a8d411..a47a731e4527 100644 --- a/include/uapi/misc/habanalabs.h +++ b/include/uapi/misc/habanalabs.h @@ -313,6 +313,7 @@ enum hl_device_status { * HL_INFO_SYNC_MANAGER - Retrieve sync manager info per dcore * HL_INFO_TOTAL_ENERGY - Retrieve total energy consumption * HL_INFO_PLL_FREQUENCY - Retrieve PLL frequency + * HL_INFO_OPEN_STATS - Retrieve info regarding recent device open calls */ #define HL_INFO_HW_IP_INFO 0 #define HL_INFO_HW_EVENTS 1 @@ -331,6 +332,7 @@ enum hl_device_status { #define HL_INFO_TOTAL_ENERGY 15 #define HL_INFO_PLL_FREQUENCY 16 #define HL_INFO_POWER 17 +#define HL_INFO_OPEN_STATS 18 #define HL_INFO_VERSION_MAX_LEN 128 #define HL_INFO_CARD_NAME_MAX_LEN 16 @@ -444,6 +446,16 @@ struct hl_pll_frequency_info { __u16 output[HL_PLL_NUM_OUTPUTS]; }; +/** + * struct hl_open_stats_info - device open statistics information + * @open_counter: ever growing counter, increased on each successful dev open + * @last_open_period_ms: duration (ms) device was open last time + */ +struct hl_open_stats_info { + __u64 open_counter; + __u64 last_open_period_ms; +}; + /** * struct hl_power_info - power information * @power: power consumption -- cgit v1.2.3 From 8b532109bf885b7b59b93487bc4672eb6d071b78 Mon Sep 17 00:00:00 2001 From: Andrea Mayer Date: Thu, 17 Jun 2021 19:16:44 +0200 Subject: seg6: add support for SRv6 End.DT46 Behavior IETF RFC 8986 [1] includes the definition of SRv6 End.DT4, End.DT6, and End.DT46 Behaviors. The current SRv6 code in the Linux kernel only implements End.DT4 and End.DT6 which can be used respectively to support IPv4-in-IPv6 and IPv6-in-IPv6 VPNs. With End.DT4 and End.DT6 it is not possible to create a single SRv6 VPN tunnel to carry both IPv4 and IPv6 traffic. The proposed End.DT46 implementation is meant to support the decapsulation of IPv4 and IPv6 traffic coming from a single SRv6 tunnel. The implementation of the SRv6 End.DT46 Behavior in the Linux kernel greatly simplifies the setup and operations of SRv6 VPNs. The SRv6 End.DT46 Behavior leverages the infrastructure of SRv6 End.DT{4,6} Behaviors implemented so far, because it makes use of a VRF device in order to force the routing lookup into the associated routing table. To make the End.DT46 work properly, it must be guaranteed that the routing table used for routing lookup operations is bound to one and only one VRF during the tunnel creation. Such constraint has to be enforced by enabling the VRF strict_mode sysctl parameter, i.e.: $ sysctl -wq net.vrf.strict_mode=1 Note that the same approach is used for the SRv6 End.DT4 Behavior and for the End.DT6 Behavior in VRF mode. The command used to instantiate an SRv6 End.DT46 Behavior is straightforward, i.e.: $ ip -6 route add 2001:db8::1 encap seg6local action End.DT46 vrftable 100 dev vrf100. [1] https://www.rfc-editor.org/rfc/rfc8986.html#name-enddt46-decapsulation-and-s ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Performance and impact of SRv6 End.DT46 Behavior on the SRv6 Networking ======================================================================= This patch aims to add the SRv6 End.DT46 Behavior with minimal impact on the performance of SRv6 End.DT4 and End.DT6 Behaviors. In order to verify this, we tested the performance of the newly introduced SRv6 End.DT46 Behavior and compared it with the performance of SRv6 End.DT{4,6} Behaviors, considering both the patched kernel and the kernel before applying the End.DT46 patch (referred to as vanilla kernel). In details, the following decapsulation scenarios were considered: 1.a) IPv6 traffic in SRv6 End.DT46 Behavior on patched kernel; 1.b) IPv4 traffic in SRv6 End.DT46 Behavior on patched kernel; 2.a) SRv6 End.DT6 Behavior (VRF mode) on patched kernel; 2.b) SRv6 End.DT4 Behavior on patched kernel; 3.a) SRv6 End.DT6 Behavior (VRF mode) on vanilla kernel (without the End.DT46 patch); 3.b) SRv6 End.DT4 Behavior on vanilla kernel (without the End.DT46 patch). All tests were performed on a testbed deployed on the CloudLab [2] facilities. We considered IPv{4,6} traffic handled by a single core (at 2.4 GHz on a Xeon(R) CPU E5-2630 v3) on kernel 5.13-rc1 using packets of size ~ 100 bytes. Scenario (1.a): average 684.70 kpps; std. dev. 0.7 kpps; Scenario (1.b): average 711.69 kpps; std. dev. 1.2 kpps; Scenario (2.a): average 690.70 kpps; std. dev. 1.2 kpps; Scenario (2.b): average 722.22 kpps; std. dev. 1.7 kpps; Scenario (3.a): average 690.02 kpps; std. dev. 2.6 kpps; Scenario (3.b): average 721.91 kpps; std. dev. 1.2 kpps; Considering the results for the patched kernel (1.a, 1.b, 2.a, 2.b) we observe that the performance degradation incurred in using End.DT46 rather than End.DT6 and End.DT4 respectively for IPv6 and IPv4 traffic is minimal, around 0.9% and 1.5%. Such very minimal performance degradation is the price to be paid if one prefers to use a single tunnel capable of handling both types of traffic (IPv4 and IPv6). Comparing the results for End.DT4 and End.DT6 under the patched and the vanilla kernel (2.a, 2.b, 3.a, 3.b) we observe that the introduction of the End.DT46 patch has no impact on the performance of End.DT4 and End.DT6. [2] https://www.cloudlab.us Signed-off-by: Andrea Mayer Reviewed-by: David Ahern Signed-off-by: David S. Miller --- include/uapi/linux/seg6_local.h | 2 + net/ipv6/seg6_local.c | 94 +++++++++++++++++++++++++++++++---------- 2 files changed, 74 insertions(+), 22 deletions(-) (limited to 'include/uapi') diff --git a/include/uapi/linux/seg6_local.h b/include/uapi/linux/seg6_local.h index 5ae3ace84de0..332b18f318f8 100644 --- a/include/uapi/linux/seg6_local.h +++ b/include/uapi/linux/seg6_local.h @@ -64,6 +64,8 @@ enum { SEG6_LOCAL_ACTION_END_AM = 14, /* custom BPF action */ SEG6_LOCAL_ACTION_END_BPF = 15, + /* decap and lookup of DA in v4 or v6 table */ + SEG6_LOCAL_ACTION_END_DT46 = 16, __SEG6_LOCAL_ACTION_MAX, }; diff --git a/net/ipv6/seg6_local.c b/net/ipv6/seg6_local.c index 4ff38cb08f4b..60bf3b877957 100644 --- a/net/ipv6/seg6_local.c +++ b/net/ipv6/seg6_local.c @@ -87,10 +87,10 @@ struct seg6_end_dt_info { int vrf_ifindex; int vrf_table; - /* tunneled packet proto and family (IPv4 or IPv6) */ - __be16 proto; + /* tunneled packet family (IPv4 or IPv6). + * Protocol and header length are inferred from family. + */ u16 family; - int hdrlen; }; struct pcpu_seg6_local_counters { @@ -521,19 +521,6 @@ static int __seg6_end_dt_vrf_build(struct seg6_local_lwt *slwt, const void *cfg, info->net = net; info->vrf_ifindex = vrf_ifindex; - switch (family) { - case AF_INET: - info->proto = htons(ETH_P_IP); - info->hdrlen = sizeof(struct iphdr); - break; - case AF_INET6: - info->proto = htons(ETH_P_IPV6); - info->hdrlen = sizeof(struct ipv6hdr); - break; - default: - return -EINVAL; - } - info->family = family; info->mode = DT_VRF_MODE; @@ -622,22 +609,44 @@ error: } static struct sk_buff *end_dt_vrf_core(struct sk_buff *skb, - struct seg6_local_lwt *slwt) + struct seg6_local_lwt *slwt, u16 family) { struct seg6_end_dt_info *info = &slwt->dt_info; struct net_device *vrf; + __be16 protocol; + int hdrlen; vrf = end_dt_get_vrf_rcu(skb, info); if (unlikely(!vrf)) goto drop; - skb->protocol = info->proto; + switch (family) { + case AF_INET: + protocol = htons(ETH_P_IP); + hdrlen = sizeof(struct iphdr); + break; + case AF_INET6: + protocol = htons(ETH_P_IPV6); + hdrlen = sizeof(struct ipv6hdr); + break; + case AF_UNSPEC: + fallthrough; + default: + goto drop; + } + + if (unlikely(info->family != AF_UNSPEC && info->family != family)) { + pr_warn_once("seg6local: SRv6 End.DT* family mismatch"); + goto drop; + } + + skb->protocol = protocol; skb_dst_drop(skb); - skb_set_transport_header(skb, info->hdrlen); + skb_set_transport_header(skb, hdrlen); - return end_dt_vrf_rcv(skb, info->family, vrf); + return end_dt_vrf_rcv(skb, family, vrf); drop: kfree_skb(skb); @@ -656,7 +665,7 @@ static int input_action_end_dt4(struct sk_buff *skb, if (!pskb_may_pull(skb, sizeof(struct iphdr))) goto drop; - skb = end_dt_vrf_core(skb, slwt); + skb = end_dt_vrf_core(skb, slwt, AF_INET); if (!skb) /* packet has been processed and consumed by the VRF */ return 0; @@ -739,7 +748,7 @@ static int input_action_end_dt6(struct sk_buff *skb, goto legacy_mode; /* DT6_VRF_MODE */ - skb = end_dt_vrf_core(skb, slwt); + skb = end_dt_vrf_core(skb, slwt, AF_INET6); if (!skb) /* packet has been processed and consumed by the VRF */ return 0; @@ -767,6 +776,36 @@ drop: return -EINVAL; } +#ifdef CONFIG_NET_L3_MASTER_DEV +static int seg6_end_dt46_build(struct seg6_local_lwt *slwt, const void *cfg, + struct netlink_ext_ack *extack) +{ + return __seg6_end_dt_vrf_build(slwt, cfg, AF_UNSPEC, extack); +} + +static int input_action_end_dt46(struct sk_buff *skb, + struct seg6_local_lwt *slwt) +{ + unsigned int off = 0; + int nexthdr; + + nexthdr = ipv6_find_hdr(skb, &off, -1, NULL, NULL); + if (unlikely(nexthdr < 0)) + goto drop; + + switch (nexthdr) { + case IPPROTO_IPIP: + return input_action_end_dt4(skb, slwt); + case IPPROTO_IPV6: + return input_action_end_dt6(skb, slwt); + } + +drop: + kfree_skb(skb); + return -EINVAL; +} +#endif + /* push an SRH on top of the current one */ static int input_action_end_b6(struct sk_buff *skb, struct seg6_local_lwt *slwt) { @@ -968,6 +1007,17 @@ static struct seg6_action_desc seg6_action_table[] = { #endif .input = input_action_end_dt6, }, + { + .action = SEG6_LOCAL_ACTION_END_DT46, + .attrs = SEG6_F_ATTR(SEG6_LOCAL_VRFTABLE), + .optattrs = SEG6_F_LOCAL_COUNTERS, +#ifdef CONFIG_NET_L3_MASTER_DEV + .input = input_action_end_dt46, + .slwt_ops = { + .build_state = seg6_end_dt46_build, + }, +#endif + }, { .action = SEG6_LOCAL_ACTION_END_B6, .attrs = SEG6_F_ATTR(SEG6_LOCAL_SRH), -- cgit v1.2.3 From 752e906732c69412087f716e93baa0330cb7cce3 Mon Sep 17 00:00:00 2001 From: Geliang Tang Date: Thu, 17 Jun 2021 16:46:07 -0700 Subject: mptcp: add csum_enabled in mptcp_sock This patch added a new member named csum_enabled in struct mptcp_sock, used a dummy mptcp_is_checksum_enabled() helper to initialize it. Also added a new member named mptcpi_csum_enabled in struct mptcp_info to expose the csum_enabled flag. Acked-by: Paolo Abeni Signed-off-by: Geliang Tang Signed-off-by: Mat Martineau Signed-off-by: David S. Miller --- include/uapi/linux/mptcp.h | 1 + net/mptcp/mptcp_diag.c | 1 + net/mptcp/protocol.c | 1 + net/mptcp/protocol.h | 2 ++ 4 files changed, 5 insertions(+) (limited to 'include/uapi') diff --git a/include/uapi/linux/mptcp.h b/include/uapi/linux/mptcp.h index 8eb3c0844bff..7b05f7102321 100644 --- a/include/uapi/linux/mptcp.h +++ b/include/uapi/linux/mptcp.h @@ -105,6 +105,7 @@ struct mptcp_info { __u64 mptcpi_rcv_nxt; __u8 mptcpi_local_addr_used; __u8 mptcpi_local_addr_max; + __u8 mptcpi_csum_enabled; }; /* diff --git a/net/mptcp/mptcp_diag.c b/net/mptcp/mptcp_diag.c index f16d9b5ee978..8f88ddeab6a2 100644 --- a/net/mptcp/mptcp_diag.c +++ b/net/mptcp/mptcp_diag.c @@ -144,6 +144,7 @@ static void mptcp_diag_get_info(struct sock *sk, struct inet_diag_msg *r, info->mptcpi_write_seq = READ_ONCE(msk->write_seq); info->mptcpi_snd_una = READ_ONCE(msk->snd_una); info->mptcpi_rcv_nxt = READ_ONCE(msk->ack_seq); + info->mptcpi_csum_enabled = READ_ONCE(msk->csum_enabled); unlock_sock_fast(sk, slow); } diff --git a/net/mptcp/protocol.c b/net/mptcp/protocol.c index 993095089990..2caca0dc2c1c 100644 --- a/net/mptcp/protocol.c +++ b/net/mptcp/protocol.c @@ -2453,6 +2453,7 @@ static int __mptcp_init_sock(struct sock *sk) msk->ack_hint = NULL; msk->first = NULL; inet_csk(sk)->icsk_sync_mss = mptcp_sync_mss; + WRITE_ONCE(msk->csum_enabled, mptcp_is_checksum_enabled(sock_net(sk))); mptcp_pm_data_init(msk); diff --git a/net/mptcp/protocol.h b/net/mptcp/protocol.h index 89f6b73783d5..1fc6693e257e 100644 --- a/net/mptcp/protocol.h +++ b/net/mptcp/protocol.h @@ -234,6 +234,7 @@ struct mptcp_sock { bool snd_data_fin_enable; bool rcv_fastclose; bool use_64bit_ack; /* Set when we received a 64-bit DSN */ + bool csum_enabled; spinlock_t join_list_lock; struct sock *ack_hint; struct work_struct work; @@ -525,6 +526,7 @@ static inline void mptcp_subflow_delegated_done(struct mptcp_subflow_context *su int mptcp_is_enabled(struct net *net); unsigned int mptcp_get_add_addr_timeout(struct net *net); +static inline int mptcp_is_checksum_enabled(struct net *net) { return false; } void mptcp_subflow_fully_established(struct mptcp_subflow_context *subflow, struct mptcp_options_received *mp_opt); bool mptcp_subflow_data_available(struct sock *sk); -- cgit v1.2.3 From 321827477360934dc040e9d3c626bf1de6c3ab3c Mon Sep 17 00:00:00 2001 From: Toke Høiland-Jørgensen Date: Fri, 18 Jun 2021 13:04:35 +0200 Subject: icmp: don't send out ICMP messages with a source address of 0.0.0.0 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When constructing ICMP response messages, the kernel will try to pick a suitable source address for the outgoing packet. However, if no IPv4 addresses are configured on the system at all, this will fail and we end up producing an ICMP message with a source address of 0.0.0.0. This can happen on a box routing IPv4 traffic via v6 nexthops, for instance. Since 0.0.0.0 is not generally routable on the internet, there's a good chance that such ICMP messages will never make it back to the sender of the original packet that the ICMP message was sent in response to. This, in turn, can create connectivity and PMTUd problems for senders. Fortunately, RFC7600 reserves a dummy address to be used as a source for ICMP messages (192.0.0.8/32), so let's teach the kernel to substitute that address as a last resort if the regular source address selection procedure fails. Below is a quick example reproducing this issue with network namespaces: ip netns add ns0 ip l add type veth peer netns ns0 ip l set dev veth0 up ip a add 10.0.0.1/24 dev veth0 ip a add fc00:dead:cafe:42::1/64 dev veth0 ip r add 10.1.0.0/24 via inet6 fc00:dead:cafe:42::2 ip -n ns0 l set dev veth0 up ip -n ns0 a add fc00:dead:cafe:42::2/64 dev veth0 ip -n ns0 r add 10.0.0.0/24 via inet6 fc00:dead:cafe:42::1 ip netns exec ns0 sysctl -w net.ipv4.icmp_ratelimit=0 ip netns exec ns0 sysctl -w net.ipv4.ip_forward=1 tcpdump -tpni veth0 -c 2 icmp & ping -w 1 10.1.0.1 > /dev/null tcpdump: verbose output suppressed, use -v[v]... for full protocol decode listening on veth0, link-type EN10MB (Ethernet), snapshot length 262144 bytes IP 10.0.0.1 > 10.1.0.1: ICMP echo request, id 29, seq 1, length 64 IP 0.0.0.0 > 10.0.0.1: ICMP net 10.1.0.1 unreachable, length 92 2 packets captured 2 packets received by filter 0 packets dropped by kernel With this patch the above capture changes to: IP 10.0.0.1 > 10.1.0.1: ICMP echo request, id 31127, seq 1, length 64 IP 192.0.0.8 > 10.0.0.1: ICMP net 10.1.0.1 unreachable, length 92 Fixes: 1da177e4c3f4 ("Linux-2.6.12-rc2") Reported-by: Juliusz Chroboczek Reviewed-by: David Ahern Signed-off-by: Toke Høiland-Jørgensen Signed-off-by: David S. Miller --- include/uapi/linux/in.h | 3 +++ net/ipv4/icmp.c | 7 +++++++ 2 files changed, 10 insertions(+) (limited to 'include/uapi') diff --git a/include/uapi/linux/in.h b/include/uapi/linux/in.h index 7d6687618d80..d1b327036ae4 100644 --- a/include/uapi/linux/in.h +++ b/include/uapi/linux/in.h @@ -289,6 +289,9 @@ struct sockaddr_in { /* Address indicating an error return. */ #define INADDR_NONE ((unsigned long int) 0xffffffff) +/* Dummy address for src of ICMP replies if no real address is set (RFC7600). */ +#define INADDR_DUMMY ((unsigned long int) 0xc0000008) + /* Network number for local host loopback. */ #define IN_LOOPBACKNET 127 diff --git a/net/ipv4/icmp.c b/net/ipv4/icmp.c index 7b6931a4d775..752e392083e6 100644 --- a/net/ipv4/icmp.c +++ b/net/ipv4/icmp.c @@ -759,6 +759,13 @@ void __icmp_send(struct sk_buff *skb_in, int type, int code, __be32 info, icmp_param.data_len = room; icmp_param.head_len = sizeof(struct icmphdr); + /* if we don't have a source address at this point, fall back to the + * dummy address instead of sending out a packet with a source address + * of 0.0.0.0 + */ + if (!fl4.saddr) + fl4.saddr = htonl(INADDR_DUMMY); + icmp_push_reply(&icmp_param, &fl4, &ipc, &rt); ende: ip_rt_put(rt); -- cgit v1.2.3 From 1f3c98eaddec857e16a7a1c6cd83317b3dc89438 Mon Sep 17 00:00:00 2001 From: Yejune Deng Date: Fri, 18 Jun 2021 22:32:47 +0800 Subject: net: add pf_family_names[] for protocol family Modify the pr_info content from int to char *, this looks more readable. Signed-off-by: Yejune Deng Signed-off-by: David S. Miller --- include/uapi/linux/net.h | 48 ++++++++++++++++++++++++++++++++++++++++++++++++ net/socket.c | 2 +- 2 files changed, 49 insertions(+), 1 deletion(-) (limited to 'include/uapi') diff --git a/include/uapi/linux/net.h b/include/uapi/linux/net.h index 4dabec6bd957..a28caaf620c7 100644 --- a/include/uapi/linux/net.h +++ b/include/uapi/linux/net.h @@ -55,4 +55,52 @@ typedef enum { #define __SO_ACCEPTCON (1 << 16) /* performed a listen */ +static const char * const pf_family_names[] = { + [PF_UNSPEC] = "PF_UNSPEC", + [PF_UNIX] = "PF_UNIX/PF_LOCAL", + [PF_INET] = "PF_INET", + [PF_AX25] = "PF_AX25", + [PF_IPX] = "PF_IPX", + [PF_APPLETALK] = "PF_APPLETALK", + [PF_NETROM] = "PF_NETROM", + [PF_BRIDGE] = "PF_BRIDGE", + [PF_ATMPVC] = "PF_ATMPVC", + [PF_X25] = "PF_X25", + [PF_INET6] = "PF_INET6", + [PF_ROSE] = "PF_ROSE", + [PF_DECnet] = "PF_DECnet", + [PF_NETBEUI] = "PF_NETBEUI", + [PF_SECURITY] = "PF_SECURITY", + [PF_KEY] = "PF_KEY", + [PF_NETLINK] = "PF_NETLINK/PF_ROUTE", + [PF_PACKET] = "PF_PACKET", + [PF_ASH] = "PF_ASH", + [PF_ECONET] = "PF_ECONET", + [PF_ATMSVC] = "PF_ATMSVC", + [PF_RDS] = "PF_RDS", + [PF_SNA] = "PF_SNA", + [PF_IRDA] = "PF_IRDA", + [PF_PPPOX] = "PF_PPPOX", + [PF_WANPIPE] = "PF_WANPIPE", + [PF_LLC] = "PF_LLC", + [PF_IB] = "PF_IB", + [PF_MPLS] = "PF_MPLS", + [PF_CAN] = "PF_CAN", + [PF_TIPC] = "PF_TIPC", + [PF_BLUETOOTH] = "PF_BLUETOOTH", + [PF_IUCV] = "PF_IUCV", + [PF_RXRPC] = "PF_RXRPC", + [PF_ISDN] = "PF_ISDN", + [PF_PHONET] = "PF_PHONET", + [PF_IEEE802154] = "PF_IEEE802154", + [PF_CAIF] = "PF_CAIF", + [PF_ALG] = "PF_ALG", + [PF_NFC] = "PF_NFC", + [PF_VSOCK] = "PF_VSOCK", + [PF_KCM] = "PF_KCM", + [PF_QIPCRTR] = "PF_QIPCRTR", + [PF_SMC] = "PF_SMC", + [PF_XDP] = "PF_XDP", +}; + #endif /* _UAPI_LINUX_NET_H */ diff --git a/net/socket.c b/net/socket.c index 27e3e7d53f8e..ff544cf50321 100644 --- a/net/socket.c +++ b/net/socket.c @@ -2988,7 +2988,7 @@ int sock_register(const struct net_proto_family *ops) } spin_unlock(&net_family_lock); - pr_info("NET: Registered protocol family %d\n", ops->family); + pr_info("NET: Registered %s protocol family\n", pf_family_names[ops->family]); return err; } EXPORT_SYMBOL(sock_register); -- cgit v1.2.3 From 103ebe658a262ef5b5db7f01d83857cf82a087d0 Mon Sep 17 00:00:00 2001 From: "David S. Miller" Date: Fri, 18 Jun 2021 13:02:45 -0700 Subject: Revert "net: add pf_family_names[] for protocol family" This reverts commit 1f3c98eaddec857e16a7a1c6cd83317b3dc89438. Does not build... Signed-off-by: David S. Miller --- include/uapi/linux/net.h | 48 ------------------------------------------------ net/socket.c | 2 +- 2 files changed, 1 insertion(+), 49 deletions(-) (limited to 'include/uapi') diff --git a/include/uapi/linux/net.h b/include/uapi/linux/net.h index a28caaf620c7..4dabec6bd957 100644 --- a/include/uapi/linux/net.h +++ b/include/uapi/linux/net.h @@ -55,52 +55,4 @@ typedef enum { #define __SO_ACCEPTCON (1 << 16) /* performed a listen */ -static const char * const pf_family_names[] = { - [PF_UNSPEC] = "PF_UNSPEC", - [PF_UNIX] = "PF_UNIX/PF_LOCAL", - [PF_INET] = "PF_INET", - [PF_AX25] = "PF_AX25", - [PF_IPX] = "PF_IPX", - [PF_APPLETALK] = "PF_APPLETALK", - [PF_NETROM] = "PF_NETROM", - [PF_BRIDGE] = "PF_BRIDGE", - [PF_ATMPVC] = "PF_ATMPVC", - [PF_X25] = "PF_X25", - [PF_INET6] = "PF_INET6", - [PF_ROSE] = "PF_ROSE", - [PF_DECnet] = "PF_DECnet", - [PF_NETBEUI] = "PF_NETBEUI", - [PF_SECURITY] = "PF_SECURITY", - [PF_KEY] = "PF_KEY", - [PF_NETLINK] = "PF_NETLINK/PF_ROUTE", - [PF_PACKET] = "PF_PACKET", - [PF_ASH] = "PF_ASH", - [PF_ECONET] = "PF_ECONET", - [PF_ATMSVC] = "PF_ATMSVC", - [PF_RDS] = "PF_RDS", - [PF_SNA] = "PF_SNA", - [PF_IRDA] = "PF_IRDA", - [PF_PPPOX] = "PF_PPPOX", - [PF_WANPIPE] = "PF_WANPIPE", - [PF_LLC] = "PF_LLC", - [PF_IB] = "PF_IB", - [PF_MPLS] = "PF_MPLS", - [PF_CAN] = "PF_CAN", - [PF_TIPC] = "PF_TIPC", - [PF_BLUETOOTH] = "PF_BLUETOOTH", - [PF_IUCV] = "PF_IUCV", - [PF_RXRPC] = "PF_RXRPC", - [PF_ISDN] = "PF_ISDN", - [PF_PHONET] = "PF_PHONET", - [PF_IEEE802154] = "PF_IEEE802154", - [PF_CAIF] = "PF_CAIF", - [PF_ALG] = "PF_ALG", - [PF_NFC] = "PF_NFC", - [PF_VSOCK] = "PF_VSOCK", - [PF_KCM] = "PF_KCM", - [PF_QIPCRTR] = "PF_QIPCRTR", - [PF_SMC] = "PF_SMC", - [PF_XDP] = "PF_XDP", -}; - #endif /* _UAPI_LINUX_NET_H */ diff --git a/net/socket.c b/net/socket.c index ff544cf50321..27e3e7d53f8e 100644 --- a/net/socket.c +++ b/net/socket.c @@ -2988,7 +2988,7 @@ int sock_register(const struct net_proto_family *ops) } spin_unlock(&net_family_lock); - pr_info("NET: Registered %s protocol family\n", pf_family_names[ops->family]); + pr_info("NET: Registered protocol family %d\n", ops->family); return err; } EXPORT_SYMBOL(sock_register); -- cgit v1.2.3 From 879740517daba2a1d3229f8a54fc2b1cb78a4f07 Mon Sep 17 00:00:00 2001 From: Devesh Sharma Date: Thu, 17 Jun 2021 01:58:17 +0530 Subject: RDMA/bnxt_re: Update ABI to pass wqe-mode to user space Changing ucontext ABI response structure to pass wqe_mode to user library. A flag in comp_mask has been set to indicate presence of wqe_mode. Moved wqe-mode ABI to uapi/rdma/bnxt_re-abi.h Link: https://lore.kernel.org/r/20210616202817.1185276-1-devesh.sharma@broadcom.com Signed-off-by: Devesh Sharma Reviewed-by: Leon Romanovsky Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/bnxt_re/ib_verbs.c | 3 +++ drivers/infiniband/hw/bnxt_re/qplib_fp.h | 2 ++ drivers/infiniband/hw/bnxt_re/qplib_res.h | 6 ------ include/uapi/rdma/bnxt_re-abi.h | 11 ++++++++++- 4 files changed, 15 insertions(+), 7 deletions(-) (limited to 'include/uapi') diff --git a/drivers/infiniband/hw/bnxt_re/ib_verbs.c b/drivers/infiniband/hw/bnxt_re/ib_verbs.c index a113d8d9e9ed..5955713234cb 100644 --- a/drivers/infiniband/hw/bnxt_re/ib_verbs.c +++ b/drivers/infiniband/hw/bnxt_re/ib_verbs.c @@ -3882,6 +3882,9 @@ int bnxt_re_alloc_ucontext(struct ib_ucontext *ctx, struct ib_udata *udata) resp.max_cqd = dev_attr->max_cq_wqes; resp.rsvd = 0; + resp.comp_mask |= BNXT_RE_UCNTX_CMASK_HAVE_MODE; + resp.mode = rdev->chip_ctx->modes.wqe_mode; + rc = ib_copy_to_udata(udata, &resp, min(udata->outlen, sizeof(resp))); if (rc) { ibdev_err(ibdev, "Failed to copy user context"); diff --git a/drivers/infiniband/hw/bnxt_re/qplib_fp.h b/drivers/infiniband/hw/bnxt_re/qplib_fp.h index f50784405e27..037501952543 100644 --- a/drivers/infiniband/hw/bnxt_re/qplib_fp.h +++ b/drivers/infiniband/hw/bnxt_re/qplib_fp.h @@ -39,6 +39,8 @@ #ifndef __BNXT_QPLIB_FP_H__ #define __BNXT_QPLIB_FP_H__ +#include + /* Few helper structures temporarily defined here * should get rid of these when roce_hsi.h is updated * in original code base diff --git a/drivers/infiniband/hw/bnxt_re/qplib_res.h b/drivers/infiniband/hw/bnxt_re/qplib_res.h index d2aea52bd1d8..c291f495ae91 100644 --- a/drivers/infiniband/hw/bnxt_re/qplib_res.h +++ b/drivers/infiniband/hw/bnxt_re/qplib_res.h @@ -45,12 +45,6 @@ extern const struct bnxt_qplib_gid bnxt_qplib_gid_zero; #define CHIP_NUM_57504 0x1751 #define CHIP_NUM_57502 0x1752 -enum bnxt_qplib_wqe_mode { - BNXT_QPLIB_WQE_MODE_STATIC = 0x00, - BNXT_QPLIB_WQE_MODE_VARIABLE = 0x01, - BNXT_QPLIB_WQE_MODE_INVALID = 0x02 -}; - struct bnxt_qplib_drv_modes { u8 wqe_mode; /* Other modes to follow here */ diff --git a/include/uapi/rdma/bnxt_re-abi.h b/include/uapi/rdma/bnxt_re-abi.h index dc52e3cf574c..b1de99bf56ce 100644 --- a/include/uapi/rdma/bnxt_re-abi.h +++ b/include/uapi/rdma/bnxt_re-abi.h @@ -49,7 +49,14 @@ #define BNXT_RE_CHIP_ID0_CHIP_MET_SFT 0x18 enum { - BNXT_RE_UCNTX_CMASK_HAVE_CCTX = 0x1ULL + BNXT_RE_UCNTX_CMASK_HAVE_CCTX = 0x1ULL, + BNXT_RE_UCNTX_CMASK_HAVE_MODE = 0x02ULL, +}; + +enum bnxt_re_wqe_mode { + BNXT_QPLIB_WQE_MODE_STATIC = 0x00, + BNXT_QPLIB_WQE_MODE_VARIABLE = 0x01, + BNXT_QPLIB_WQE_MODE_INVALID = 0x02, }; struct bnxt_re_uctx_resp { @@ -62,6 +69,8 @@ struct bnxt_re_uctx_resp { __aligned_u64 comp_mask; __u32 chip_id0; __u32 chip_id1; + __u32 mode; + __u32 rsvd1; /* padding */ }; /* -- cgit v1.2.3 From 2d82ab251ef0f6e7716279b04e9b5a01a86ca530 Mon Sep 17 00:00:00 2001 From: Greg Kurz Date: Thu, 20 May 2021 17:46:54 +0200 Subject: virtiofs: propagate sync() to file server Even if POSIX doesn't mandate it, linux users legitimately expect sync() to flush all data and metadata to physical storage when it is located on the same system. This isn't happening with virtiofs though: sync() inside the guest returns right away even though data still needs to be flushed from the host page cache. This is easily demonstrated by doing the following in the guest: $ dd if=/dev/zero of=/mnt/foo bs=1M count=5K ; strace -T -e sync sync 5120+0 records in 5120+0 records out 5368709120 bytes (5.4 GB, 5.0 GiB) copied, 5.22224 s, 1.0 GB/s sync() = 0 <0.024068> and start the following in the host when the 'dd' command completes in the guest: $ strace -T -e fsync /usr/bin/sync virtiofs/foo fsync(3) = 0 <10.371640> There are no good reasons not to honor the expected behavior of sync() actually: it gives an unrealistic impression that virtiofs is super fast and that data has safely landed on HW, which isn't the case obviously. Implement a ->sync_fs() superblock operation that sends a new FUSE_SYNCFS request type for this purpose. Provision a 64-bit placeholder for possible future extensions. Since the file server cannot handle the wait == 0 case, we skip it to avoid a gratuitous roundtrip. Note that this is per-superblock: a FUSE_SYNCFS is send for the root mount and for each submount. Like with FUSE_FSYNC and FUSE_FSYNCDIR, lack of support for FUSE_SYNCFS in the file server is treated as permanent success. This ensures compatibility with older file servers: the client will get the current behavior of sync() not being propagated to the file server. Note that such an operation allows the file server to DoS sync(). Since a typical FUSE file server is an untrusted piece of software running in userspace, this is disabled by default. Only enable it with virtiofs for now since virtiofsd is supposedly trusted by the guest kernel. Reported-by: Robert Krawitz Signed-off-by: Greg Kurz Signed-off-by: Miklos Szeredi --- fs/fuse/fuse_i.h | 3 +++ fs/fuse/inode.c | 40 ++++++++++++++++++++++++++++++++++++++++ fs/fuse/virtio_fs.c | 1 + include/uapi/linux/fuse.h | 10 +++++++++- 4 files changed, 53 insertions(+), 1 deletion(-) (limited to 'include/uapi') diff --git a/fs/fuse/fuse_i.h b/fs/fuse/fuse_i.h index 7e463e220053..f48dd7ff32af 100644 --- a/fs/fuse/fuse_i.h +++ b/fs/fuse/fuse_i.h @@ -761,6 +761,9 @@ struct fuse_conn { /* Auto-mount submounts announced by the server */ unsigned int auto_submounts:1; + /* Propagate syncfs() to server */ + unsigned int sync_fs:1; + /** The number of requests waiting for completion */ atomic_t num_waiting; diff --git a/fs/fuse/inode.c b/fs/fuse/inode.c index 393e36b74dc4..93d28dc1f572 100644 --- a/fs/fuse/inode.c +++ b/fs/fuse/inode.c @@ -506,6 +506,45 @@ static int fuse_statfs(struct dentry *dentry, struct kstatfs *buf) return err; } +static int fuse_sync_fs(struct super_block *sb, int wait) +{ + struct fuse_mount *fm = get_fuse_mount_super(sb); + struct fuse_conn *fc = fm->fc; + struct fuse_syncfs_in inarg; + FUSE_ARGS(args); + int err; + + /* + * Userspace cannot handle the wait == 0 case. Avoid a + * gratuitous roundtrip. + */ + if (!wait) + return 0; + + /* The filesystem is being unmounted. Nothing to do. */ + if (!sb->s_root) + return 0; + + if (!fc->sync_fs) + return 0; + + memset(&inarg, 0, sizeof(inarg)); + args.in_numargs = 1; + args.in_args[0].size = sizeof(inarg); + args.in_args[0].value = &inarg; + args.opcode = FUSE_SYNCFS; + args.nodeid = get_node_id(sb->s_root->d_inode); + args.out_numargs = 0; + + err = fuse_simple_request(fm, &args); + if (err == -ENOSYS) { + fc->sync_fs = 0; + err = 0; + } + + return err; +} + enum { OPT_SOURCE, OPT_SUBTYPE, @@ -909,6 +948,7 @@ static const struct super_operations fuse_super_operations = { .put_super = fuse_put_super, .umount_begin = fuse_umount_begin, .statfs = fuse_statfs, + .sync_fs = fuse_sync_fs, .show_options = fuse_show_options, }; diff --git a/fs/fuse/virtio_fs.c b/fs/fuse/virtio_fs.c index bcb8a02e2d8b..f9809b1b82f0 100644 --- a/fs/fuse/virtio_fs.c +++ b/fs/fuse/virtio_fs.c @@ -1447,6 +1447,7 @@ static int virtio_fs_get_tree(struct fs_context *fsc) fc->release = fuse_free_conn; fc->delete_stale = true; fc->auto_submounts = true; + fc->sync_fs = true; /* Tell FUSE to split requests that exceed the virtqueue's size */ fc->max_pages_limit = min_t(unsigned int, fc->max_pages_limit, diff --git a/include/uapi/linux/fuse.h b/include/uapi/linux/fuse.h index 271ae90a9bb7..36ed092227fa 100644 --- a/include/uapi/linux/fuse.h +++ b/include/uapi/linux/fuse.h @@ -181,6 +181,9 @@ * - add FUSE_OPEN_KILL_SUIDGID * - extend fuse_setxattr_in, add FUSE_SETXATTR_EXT * - add FUSE_SETXATTR_ACL_KILL_SGID + * + * 7.34 + * - add FUSE_SYNCFS */ #ifndef _LINUX_FUSE_H @@ -216,7 +219,7 @@ #define FUSE_KERNEL_VERSION 7 /** Minor version number of this interface */ -#define FUSE_KERNEL_MINOR_VERSION 33 +#define FUSE_KERNEL_MINOR_VERSION 34 /** The node ID of the root inode */ #define FUSE_ROOT_ID 1 @@ -509,6 +512,7 @@ enum fuse_opcode { FUSE_COPY_FILE_RANGE = 47, FUSE_SETUPMAPPING = 48, FUSE_REMOVEMAPPING = 49, + FUSE_SYNCFS = 50, /* CUSE specific operations */ CUSE_INIT = 4096, @@ -971,4 +975,8 @@ struct fuse_removemapping_one { #define FUSE_REMOVEMAPPING_MAX_ENTRY \ (PAGE_SIZE / sizeof(struct fuse_removemapping_one)) +struct fuse_syncfs_in { + uint64_t padding; +}; + #endif /* _LINUX_FUSE_H */ -- cgit v1.2.3 From 1a9fd4172d5c8ba64735b3aef7eed643d398ce05 Mon Sep 17 00:00:00 2001 From: David Sterba Date: Fri, 21 May 2021 17:42:23 +0200 Subject: btrfs: fix typos in comments Fix typos that have snuck in since the last round. Found by codespell. Signed-off-by: David Sterba --- fs/btrfs/backref.c | 2 +- fs/btrfs/ctree.h | 6 +++--- fs/btrfs/delalloc-space.c | 2 +- fs/btrfs/dev-replace.c | 2 +- fs/btrfs/discard.c | 2 +- fs/btrfs/disk-io.c | 2 +- fs/btrfs/extent-tree.c | 2 +- fs/btrfs/file-item.c | 2 +- fs/btrfs/inode.c | 4 ++-- fs/btrfs/ioctl.c | 2 +- fs/btrfs/locking.c | 4 ++-- fs/btrfs/props.c | 2 +- fs/btrfs/qgroup.c | 2 +- fs/btrfs/scrub.c | 2 +- fs/btrfs/send.c | 2 +- fs/btrfs/space-info.c | 4 ++-- fs/btrfs/tests/extent-map-tests.c | 2 +- fs/btrfs/volumes.c | 8 ++++---- fs/btrfs/zoned.c | 4 ++-- include/uapi/linux/btrfs.h | 4 ++-- include/uapi/linux/btrfs_tree.h | 4 ++-- 21 files changed, 32 insertions(+), 32 deletions(-) (limited to 'include/uapi') diff --git a/fs/btrfs/backref.c b/fs/btrfs/backref.c index 117d423fdb93..7a8a2fc19533 100644 --- a/fs/btrfs/backref.c +++ b/fs/btrfs/backref.c @@ -2675,7 +2675,7 @@ static int handle_direct_tree_backref(struct btrfs_backref_cache *cache, * * @ref_key: The same as @ref_key in handle_direct_tree_backref() * @tree_key: The first key of this tree block. - * @path: A clean (released) path, to avoid allocating path everytime + * @path: A clean (released) path, to avoid allocating path every time * the function get called. */ static int handle_indirect_tree_backref(struct btrfs_backref_cache *cache, diff --git a/fs/btrfs/ctree.h b/fs/btrfs/ctree.h index 82e58f2fbe0a..cbdabecffb05 100644 --- a/fs/btrfs/ctree.h +++ b/fs/btrfs/ctree.h @@ -2757,9 +2757,9 @@ enum btrfs_reserve_flush_enum { /* * Flush space by above mentioned methods and by: * - Running delayed iputs - * - Commiting transaction + * - Committing transaction * - * Can be interruped by fatal signal. + * Can be interrupted by a fatal signal. */ BTRFS_RESERVE_FLUSH_DATA, BTRFS_RESERVE_FLUSH_FREE_SPACE_INODE, @@ -2769,7 +2769,7 @@ enum btrfs_reserve_flush_enum { * Pretty much the same as FLUSH_ALL, but can also steal space from * global rsv. * - * Can be interruped by fatal signal. + * Can be interrupted by a fatal signal. */ BTRFS_RESERVE_FLUSH_ALL_STEAL, }; diff --git a/fs/btrfs/delalloc-space.c b/fs/btrfs/delalloc-space.c index 56642ca7af10..2059d1504149 100644 --- a/fs/btrfs/delalloc-space.c +++ b/fs/btrfs/delalloc-space.c @@ -89,7 +89,7 @@ * ->outstanding_extents += 1 (current value is 1) * * -> set_delalloc - * ->outstanding_extents += 1 (currrent value is 2) + * ->outstanding_extents += 1 (current value is 2) * * -> btrfs_delalloc_release_extents() * ->outstanding_extents -= 1 (current value is 1) diff --git a/fs/btrfs/dev-replace.c b/fs/btrfs/dev-replace.c index d05f73530af7..d029be40ea6f 100644 --- a/fs/btrfs/dev-replace.c +++ b/fs/btrfs/dev-replace.c @@ -37,7 +37,7 @@ * - Write duplication * * All new writes will be written to both target and source devices, so even - * if replace gets canceled, sources device still contans up-to-date data. + * if replace gets canceled, sources device still contains up-to-date data. * * Location: handle_ops_on_dev_replace() from __btrfs_map_block() * Start: btrfs_dev_replace_start() diff --git a/fs/btrfs/discard.c b/fs/btrfs/discard.c index 306ff20af70f..e1b7bd927d69 100644 --- a/fs/btrfs/discard.c +++ b/fs/btrfs/discard.c @@ -624,7 +624,7 @@ void btrfs_discard_update_discardable(struct btrfs_block_group *block_group) * @fs_info: fs_info of interest * * The unused_bgs list needs to be punted to the discard lists because the - * order of operations is changed. In the normal sychronous discard path, the + * order of operations is changed. In the normal synchronous discard path, the * block groups are trimmed via a single large trim in transaction commit. This * is ultimately what we are trying to avoid with asynchronous discard. Thus, * it must be done before going down the unused_bgs path. diff --git a/fs/btrfs/disk-io.c b/fs/btrfs/disk-io.c index d1d5091a8385..544bb7a82e57 100644 --- a/fs/btrfs/disk-io.c +++ b/fs/btrfs/disk-io.c @@ -3471,7 +3471,7 @@ int __cold open_ctree(struct super_block *sb, struct btrfs_fs_devices *fs_device * At this point we know all the devices that make this filesystem, * including the seed devices but we don't know yet if the replace * target is required. So free devices that are not part of this - * filesystem but skip the replace traget device which is checked + * filesystem but skip the replace target device which is checked * below in btrfs_init_dev_replace(). */ btrfs_free_extra_devids(fs_devices); diff --git a/fs/btrfs/extent-tree.c b/fs/btrfs/extent-tree.c index d2f39a122d89..421120d6a14b 100644 --- a/fs/btrfs/extent-tree.c +++ b/fs/btrfs/extent-tree.c @@ -1425,7 +1425,7 @@ int btrfs_inc_extent_ref(struct btrfs_trans_handle *trans, * bytenr of the parent block. Since new extents are always * created with indirect references, this will only be the case * when relocating a shared extent. In that case, root_objectid - * will be BTRFS_TREE_RELOC_OBJECTID. Otheriwse, parent must + * will be BTRFS_TREE_RELOC_OBJECTID. Otherwise, parent must * be 0 * * @root_objectid: The id of the root where this modification has originated, diff --git a/fs/btrfs/file-item.c b/fs/btrfs/file-item.c index 441cee7fbb62..df6631eefc65 100644 --- a/fs/btrfs/file-item.c +++ b/fs/btrfs/file-item.c @@ -618,7 +618,7 @@ fail: * @file_start: offset in file this bio begins to describe * @contig: Boolean. If true/1 means all bio vecs in this bio are * contiguous and they begin at @file_start in the file. False/0 - * means this bio can contains potentially discontigous bio vecs + * means this bio can contain potentially discontiguous bio vecs * so the logical offset of each should be calculated separately. */ blk_status_t btrfs_csum_one_bio(struct btrfs_inode *inode, struct bio *bio, diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c index 794d906cba6c..a2494c645681 100644 --- a/fs/btrfs/inode.c +++ b/fs/btrfs/inode.c @@ -2784,7 +2784,7 @@ static int insert_reserved_file_extent(struct btrfs_trans_handle *trans, /* * If we dropped an inline extent here, we know the range where it is * was not marked with the EXTENT_DELALLOC_NEW bit, so we update the - * number of bytes only for that range contaning the inline extent. + * number of bytes only for that range containing the inline extent. * The remaining of the range will be processed when clearning the * EXTENT_DELALLOC_BIT bit through the ordered extent completion. */ @@ -4114,7 +4114,7 @@ static int btrfs_unlink_subvol(struct btrfs_trans_handle *trans, * This is a placeholder inode for a subvolume we didn't have a * reference to at the time of the snapshot creation. In the meantime * we could have renamed the real subvol link into our snapshot, so - * depending on btrfs_del_root_ref to return -ENOENT here is incorret. + * depending on btrfs_del_root_ref to return -ENOENT here is incorrect. * Instead simply lookup the dir_index_item for this entry so we can * remove it. Otherwise we know we have a ref to the root and we can * call btrfs_del_root_ref, and it _shouldn't_ fail. diff --git a/fs/btrfs/ioctl.c b/fs/btrfs/ioctl.c index f83eb4a225cc..0ba98e08a029 100644 --- a/fs/btrfs/ioctl.c +++ b/fs/btrfs/ioctl.c @@ -2984,7 +2984,7 @@ static noinline int btrfs_ioctl_snap_destroy(struct file *file, err = PTR_ERR(subvol_name_ptr); goto free_parent; } - /* subvol_name_ptr is already NULL termined */ + /* subvol_name_ptr is already nul terminated */ subvol_name = (char *)kbasename(subvol_name_ptr); } } else { diff --git a/fs/btrfs/locking.c b/fs/btrfs/locking.c index 5fafc5e89bb7..313d9d685adb 100644 --- a/fs/btrfs/locking.c +++ b/fs/btrfs/locking.c @@ -57,7 +57,7 @@ void btrfs_tree_read_lock(struct extent_buffer *eb) /* * Try-lock for read. * - * Retrun 1 if the rwlock has been taken, 0 otherwise + * Return 1 if the rwlock has been taken, 0 otherwise */ int btrfs_try_tree_read_lock(struct extent_buffer *eb) { @@ -72,7 +72,7 @@ int btrfs_try_tree_read_lock(struct extent_buffer *eb) /* * Try-lock for write. * - * Retrun 1 if the rwlock has been taken, 0 otherwise + * Return 1 if the rwlock has been taken, 0 otherwise */ int btrfs_try_tree_write_lock(struct extent_buffer *eb) { diff --git a/fs/btrfs/props.c b/fs/btrfs/props.c index 2dcb1cb21634..a17e53e700b1 100644 --- a/fs/btrfs/props.c +++ b/fs/btrfs/props.c @@ -348,7 +348,7 @@ static int inherit_props(struct btrfs_trans_handle *trans, /* * This is not strictly necessary as the property should be - * valid, but in case it isn't, don't propagate it futher. + * valid, but in case it isn't, don't propagate it further. */ ret = h->validate(value, strlen(value)); if (ret) diff --git a/fs/btrfs/qgroup.c b/fs/btrfs/qgroup.c index 3ded812f522c..d72885903b8c 100644 --- a/fs/btrfs/qgroup.c +++ b/fs/btrfs/qgroup.c @@ -2521,7 +2521,7 @@ int btrfs_qgroup_account_extent(struct btrfs_trans_handle *trans, u64 bytenr, int ret = 0; /* - * If quotas get disabled meanwhile, the resouces need to be freed and + * If quotas get disabled meanwhile, the resources need to be freed and * we can't just exit here. */ if (!test_bit(BTRFS_FS_QUOTA_ENABLED, &fs_info->flags)) diff --git a/fs/btrfs/scrub.c b/fs/btrfs/scrub.c index b60466db5654..088641ba7a8e 100644 --- a/fs/btrfs/scrub.c +++ b/fs/btrfs/scrub.c @@ -2486,7 +2486,7 @@ static void drop_csum_range(struct scrub_ctx *sctx, struct btrfs_ordered_sum *su * the csum into @csum. * * The search source is sctx->csum_list, which is a pre-populated list - * storing bytenr ordered csum ranges. We're reponsible to cleanup any range + * storing bytenr ordered csum ranges. We're responsible to cleanup any range * that is before @logical. * * Return 0 if there is no csum for the range. diff --git a/fs/btrfs/send.c b/fs/btrfs/send.c index 974274c7e26e..6e69302828ef 100644 --- a/fs/btrfs/send.c +++ b/fs/btrfs/send.c @@ -6514,7 +6514,7 @@ static int changed_extent(struct send_ctx *sctx, * updates the inode item, but it only changes the iversion (sequence * field in the inode item) of the inode, so if a file is deduplicated * the same amount of times in both the parent and send snapshots, its - * iversion becames the same in both snapshots, whence the inode item is + * iversion becomes the same in both snapshots, whence the inode item is * the same on both snapshots. */ if (sctx->cur_ino != sctx->cmp_key->objectid) diff --git a/fs/btrfs/space-info.c b/fs/btrfs/space-info.c index 42d0fa2092d4..f26fdb7a17e8 100644 --- a/fs/btrfs/space-info.c +++ b/fs/btrfs/space-info.c @@ -389,7 +389,7 @@ again: ticket = list_first_entry(head, struct reserve_ticket, list); - /* Check and see if our ticket can be satisified now. */ + /* Check and see if our ticket can be satisfied now. */ if ((used + ticket->bytes <= space_info->total_bytes) || btrfs_can_overcommit(fs_info, space_info, ticket->bytes, flush)) { @@ -961,7 +961,7 @@ static bool maybe_fail_all_tickets(struct btrfs_fs_info *fs_info, * if it doesn't feel like the space reclaimed by the commit * would result in the ticket succeeding. However if we have a * smaller ticket in the queue it may be small enough to be - * satisified by committing the transaction, so if any + * satisfied by committing the transaction, so if any * subsequent ticket is smaller than the first ticket go ahead * and send us back for another loop through the enospc flushing * code. diff --git a/fs/btrfs/tests/extent-map-tests.c b/fs/btrfs/tests/extent-map-tests.c index c0aefe6dee0b..319fed82d741 100644 --- a/fs/btrfs/tests/extent-map-tests.c +++ b/fs/btrfs/tests/extent-map-tests.c @@ -557,7 +557,7 @@ int btrfs_test_extent_map(void) { /* * Test a chunk with 2 data stripes one of which - * interesects the physical address of the super block + * intersects the physical address of the super block * is correctly recognised. */ .raid_type = BTRFS_BLOCK_GROUP_RAID1, diff --git a/fs/btrfs/volumes.c b/fs/btrfs/volumes.c index 80e962788396..582695cee9d1 100644 --- a/fs/btrfs/volumes.c +++ b/fs/btrfs/volumes.c @@ -717,7 +717,7 @@ static struct btrfs_fs_devices *find_fsid_changed( /* * Handles the case where scanned device is part of an fs that had - * multiple successful changes of FSID but curently device didn't + * multiple successful changes of FSID but currently device didn't * observe it. Meaning our fsid will be different than theirs. We need * to handle two subcases : * 1 - The fs still continues to have different METADATA/FSID uuids. @@ -1550,7 +1550,7 @@ static bool dev_extent_hole_check(struct btrfs_device *device, u64 *hole_start, * check to ensure dev extents are not double allocated. * This makes the function safe to allocate dev extents but may not report * correct usable device space, as device extent freed in current transaction - * is not reported as avaiable. + * is not reported as available. */ static int find_free_dev_extent_start(struct btrfs_device *device, u64 num_bytes, u64 search_start, u64 *start, @@ -6152,7 +6152,7 @@ int btrfs_get_io_geometry(struct btrfs_fs_info *fs_info, struct extent_map *em, offset = logical - em->start; /* Len of a stripe in a chunk */ stripe_len = map->stripe_len; - /* Stripe wher this block falls in */ + /* Stripe where this block falls in */ stripe_nr = div64_u64(offset, stripe_len); /* Offset of stripe in the chunk */ stripe_offset = stripe_nr * stripe_len; @@ -7863,7 +7863,7 @@ static int verify_one_dev_extent(struct btrfs_fs_info *fs_info, ret = -EUCLEAN; } - /* Make sure no dev extent is beyond device bondary */ + /* Make sure no dev extent is beyond device boundary */ dev = btrfs_find_device(fs_info->fs_devices, devid, NULL, NULL); if (!dev) { btrfs_err(fs_info, "failed to find devid %llu", devid); diff --git a/fs/btrfs/zoned.c b/fs/btrfs/zoned.c index 549912120cfe..297c0b1c0634 100644 --- a/fs/btrfs/zoned.c +++ b/fs/btrfs/zoned.c @@ -81,7 +81,7 @@ static int sb_write_pointer(struct block_device *bdev, struct blk_zone *zones, * *: Special case, no superblock is written * 0: Use write pointer of zones[0] * 1: Use write pointer of zones[1] - * C: Compare super blcoks from zones[0] and zones[1], use the latest + * C: Compare super blocks from zones[0] and zones[1], use the latest * one determined by generation * x: Invalid state */ @@ -433,7 +433,7 @@ int btrfs_get_dev_zone_info(struct btrfs_device *device) } /* - * If zones[0] is conventional, always use the beggining of the + * If zones[0] is conventional, always use the beginning of the * zone to record superblock. No need to validate in that case. */ if (zone_info->sb_zones[BTRFS_NR_SB_LOG_ZONES * i].type == diff --git a/include/uapi/linux/btrfs.h b/include/uapi/linux/btrfs.h index 5df73001aad4..22cd037123fa 100644 --- a/include/uapi/linux/btrfs.h +++ b/include/uapi/linux/btrfs.h @@ -154,7 +154,7 @@ struct btrfs_scrub_progress { __u64 tree_bytes_scrubbed; /* # of tree bytes scrubbed */ __u64 read_errors; /* # of read errors encountered (EIO) */ __u64 csum_errors; /* # of failed csum checks */ - __u64 verify_errors; /* # of occurences, where the metadata + __u64 verify_errors; /* # of occurrences, where the metadata * of a tree block did not match the * expected values, like generation or * logical */ @@ -174,7 +174,7 @@ struct btrfs_scrub_progress { __u64 last_physical; /* last physical address scrubbed. In * case a scrub was aborted, this can * be used to restart the scrub */ - __u64 unverified_errors; /* # of occurences where a read for a + __u64 unverified_errors; /* # of occurrences where a read for a * full (64k) bio failed, but the re- * check succeeded for each 4k piece. * Intermittent error. */ diff --git a/include/uapi/linux/btrfs_tree.h b/include/uapi/linux/btrfs_tree.h index 58d7cff9afb1..ccdb40fe40dc 100644 --- a/include/uapi/linux/btrfs_tree.h +++ b/include/uapi/linux/btrfs_tree.h @@ -59,7 +59,7 @@ /* for storing balance parameters in the root tree */ #define BTRFS_BALANCE_OBJECTID -4ULL -/* orhpan objectid for tracking unlinked/truncated files */ +/* orphan objectid for tracking unlinked/truncated files */ #define BTRFS_ORPHAN_OBJECTID -5ULL /* does write ahead logging to speed up fsyncs */ @@ -275,7 +275,7 @@ #define BTRFS_PERSISTENT_ITEM_KEY 249 /* - * Persistantly stores the device replace state in the device tree. + * Persistently stores the device replace state in the device tree. * The key is built like this: (0, BTRFS_DEV_REPLACE_KEY, 0). */ #define BTRFS_DEV_REPLACE_KEY 250 -- cgit v1.2.3 From ea7fc1bb1cd1b92b42b1d9273ce7e231d3dc9321 Mon Sep 17 00:00:00 2001 From: Steven Price Date: Mon, 21 Jun 2021 12:17:12 +0100 Subject: KVM: arm64: Introduce MTE VM feature Add a new VM feature 'KVM_ARM_CAP_MTE' which enables memory tagging for a VM. This will expose the feature to the guest and automatically tag memory pages touched by the VM as PG_mte_tagged (and clear the tag storage) to ensure that the guest cannot see stale tags, and so that the tags are correctly saved/restored across swap. Actually exposing the new capability to user space happens in a later patch. Reviewed-by: Catalin Marinas Signed-off-by: Steven Price [maz: move VM_SHARED sampling into the critical section] Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20210621111716.37157-3-steven.price@arm.com --- arch/arm64/include/asm/kvm_emulate.h | 3 ++ arch/arm64/include/asm/kvm_host.h | 4 +++ arch/arm64/kvm/hyp/exception.c | 3 +- arch/arm64/kvm/mmu.c | 67 +++++++++++++++++++++++++++++++++++- arch/arm64/kvm/sys_regs.c | 7 ++++ include/uapi/linux/kvm.h | 1 + 6 files changed, 83 insertions(+), 2 deletions(-) (limited to 'include/uapi') diff --git a/arch/arm64/include/asm/kvm_emulate.h b/arch/arm64/include/asm/kvm_emulate.h index 01b9857757f2..fd418955e31e 100644 --- a/arch/arm64/include/asm/kvm_emulate.h +++ b/arch/arm64/include/asm/kvm_emulate.h @@ -84,6 +84,9 @@ static inline void vcpu_reset_hcr(struct kvm_vcpu *vcpu) if (cpus_have_const_cap(ARM64_MISMATCHED_CACHE_TYPE) || vcpu_el1_is_32bit(vcpu)) vcpu->arch.hcr_el2 |= HCR_TID2; + + if (kvm_has_mte(vcpu->kvm)) + vcpu->arch.hcr_el2 |= HCR_ATA; } static inline unsigned long *vcpu_hcr(struct kvm_vcpu *vcpu) diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index 7cd7d5c8c4bc..1c4293c46ef6 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -132,6 +132,9 @@ struct kvm_arch { u8 pfr0_csv2; u8 pfr0_csv3; + + /* Memory Tagging Extension enabled for the guest */ + bool mte_enabled; }; struct kvm_vcpu_fault_info { @@ -769,6 +772,7 @@ bool kvm_arm_vcpu_is_finalized(struct kvm_vcpu *vcpu); #define kvm_arm_vcpu_sve_finalized(vcpu) \ ((vcpu)->arch.flags & KVM_ARM64_VCPU_SVE_FINALIZED) +#define kvm_has_mte(kvm) (system_supports_mte() && (kvm)->arch.mte_enabled) #define kvm_vcpu_has_pmu(vcpu) \ (test_bit(KVM_ARM_VCPU_PMU_V3, (vcpu)->arch.features)) diff --git a/arch/arm64/kvm/hyp/exception.c b/arch/arm64/kvm/hyp/exception.c index 11541b94b328..0418399e0a20 100644 --- a/arch/arm64/kvm/hyp/exception.c +++ b/arch/arm64/kvm/hyp/exception.c @@ -112,7 +112,8 @@ static void enter_exception64(struct kvm_vcpu *vcpu, unsigned long target_mode, new |= (old & PSR_C_BIT); new |= (old & PSR_V_BIT); - // TODO: TCO (if/when ARMv8.5-MemTag is exposed to guests) + if (kvm_has_mte(vcpu->kvm)) + new |= PSR_TCO_BIT; new |= (old & PSR_DIT_BIT); diff --git a/arch/arm64/kvm/mmu.c b/arch/arm64/kvm/mmu.c index c10207fed2f3..c6a97d463892 100644 --- a/arch/arm64/kvm/mmu.c +++ b/arch/arm64/kvm/mmu.c @@ -822,6 +822,45 @@ transparent_hugepage_adjust(struct kvm_memory_slot *memslot, return PAGE_SIZE; } +/* + * The page will be mapped in stage 2 as Normal Cacheable, so the VM will be + * able to see the page's tags and therefore they must be initialised first. If + * PG_mte_tagged is set, tags have already been initialised. + * + * The race in the test/set of the PG_mte_tagged flag is handled by: + * - preventing VM_SHARED mappings in a memslot with MTE preventing two VMs + * racing to santise the same page + * - mmap_lock protects between a VM faulting a page in and the VMM performing + * an mprotect() to add VM_MTE + */ +static int sanitise_mte_tags(struct kvm *kvm, kvm_pfn_t pfn, + unsigned long size) +{ + unsigned long i, nr_pages = size >> PAGE_SHIFT; + struct page *page; + + if (!kvm_has_mte(kvm)) + return 0; + + /* + * pfn_to_online_page() is used to reject ZONE_DEVICE pages + * that may not support tags. + */ + page = pfn_to_online_page(pfn); + + if (!page) + return -EFAULT; + + for (i = 0; i < nr_pages; i++, page++) { + if (!test_bit(PG_mte_tagged, &page->flags)) { + mte_clear_page_tags(page_address(page)); + set_bit(PG_mte_tagged, &page->flags); + } + } + + return 0; +} + static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa, struct kvm_memory_slot *memslot, unsigned long hva, unsigned long fault_status) @@ -830,6 +869,7 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa, bool write_fault, writable, force_pte = false; bool exec_fault; bool device = false; + bool shared; unsigned long mmu_seq; struct kvm *kvm = vcpu->kvm; struct kvm_mmu_memory_cache *memcache = &vcpu->arch.mmu_page_cache; @@ -873,6 +913,8 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa, vma_shift = PAGE_SHIFT; } + shared = (vma->vm_flags & VM_PFNMAP); + switch (vma_shift) { #ifndef __PAGETABLE_PMD_FOLDED case PUD_SHIFT: @@ -971,8 +1013,18 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa, if (writable) prot |= KVM_PGTABLE_PROT_W; - if (fault_status != FSC_PERM && !device) + if (fault_status != FSC_PERM && !device) { + /* Check the VMM hasn't introduced a new VM_SHARED VMA */ + if (kvm_has_mte(kvm) && shared) { + ret = -EFAULT; + goto out_unlock; + } + ret = sanitise_mte_tags(kvm, pfn, vma_pagesize); + if (ret) + goto out_unlock; + clean_dcache_guest_page(pfn, vma_pagesize); + } if (exec_fault) { prot |= KVM_PGTABLE_PROT_X; @@ -1168,12 +1220,17 @@ bool kvm_unmap_gfn_range(struct kvm *kvm, struct kvm_gfn_range *range) bool kvm_set_spte_gfn(struct kvm *kvm, struct kvm_gfn_range *range) { kvm_pfn_t pfn = pte_pfn(range->pte); + int ret; if (!kvm->arch.mmu.pgt) return false; WARN_ON(range->end - range->start != 1); + ret = sanitise_mte_tags(kvm, pfn, PAGE_SIZE); + if (ret) + return false; + /* * We've moved a page around, probably through CoW, so let's treat it * just like a translation fault and clean the cache to the PoC. @@ -1381,6 +1438,14 @@ int kvm_arch_prepare_memory_region(struct kvm *kvm, if (!vma) break; + /* + * VM_SHARED mappings are not allowed with MTE to avoid races + * when updating the PG_mte_tagged page flag, see + * sanitise_mte_tags for more details. + */ + if (kvm_has_mte(kvm) && vma->vm_flags & VM_SHARED) + return -EINVAL; + /* * Take the intersection of this VMA with the memory region */ diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c index 1a7968ad078c..36f67f8deae1 100644 --- a/arch/arm64/kvm/sys_regs.c +++ b/arch/arm64/kvm/sys_regs.c @@ -1047,6 +1047,13 @@ static u64 read_id_reg(const struct kvm_vcpu *vcpu, break; case SYS_ID_AA64PFR1_EL1: val &= ~FEATURE(ID_AA64PFR1_MTE); + if (kvm_has_mte(vcpu->kvm)) { + u64 pfr, mte; + + pfr = read_sanitised_ftr_reg(SYS_ID_AA64PFR1_EL1); + mte = cpuid_feature_extract_unsigned_field(pfr, ID_AA64PFR1_MTE_SHIFT); + val |= FIELD_PREP(FEATURE(ID_AA64PFR1_MTE), mte); + } break; case SYS_ID_AA64ISAR1_EL1: if (!vcpu_has_ptrauth(vcpu)) diff --git a/include/uapi/linux/kvm.h b/include/uapi/linux/kvm.h index 79d9c44d1ad7..d4da58ddcad7 100644 --- a/include/uapi/linux/kvm.h +++ b/include/uapi/linux/kvm.h @@ -1083,6 +1083,7 @@ struct kvm_ppc_resize_hpt { #define KVM_CAP_SGX_ATTRIBUTE 196 #define KVM_CAP_VM_COPY_ENC_CONTEXT_FROM 197 #define KVM_CAP_PTP_KVM 198 +#define KVM_CAP_ARM_MTE 199 #ifdef KVM_CAP_IRQ_ROUTING -- cgit v1.2.3 From f0376edb1ddcab19a473b4bf1fbd5b6bbed3705b Mon Sep 17 00:00:00 2001 From: Steven Price Date: Mon, 21 Jun 2021 12:17:15 +0100 Subject: KVM: arm64: Add ioctl to fetch/store tags in a guest The VMM may not wish to have it's own mapping of guest memory mapped with PROT_MTE because this causes problems if the VMM has tag checking enabled (the guest controls the tags in physical RAM and it's unlikely the tags are correct for the VMM). Instead add a new ioctl which allows the VMM to easily read/write the tags from guest memory, allowing the VMM's mapping to be non-PROT_MTE while the VMM can still read/write the tags for the purpose of migration. Reviewed-by: Catalin Marinas Signed-off-by: Steven Price Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20210621111716.37157-6-steven.price@arm.com --- arch/arm64/include/asm/kvm_host.h | 3 ++ arch/arm64/include/asm/mte-def.h | 1 + arch/arm64/include/uapi/asm/kvm.h | 11 ++++++ arch/arm64/kvm/arm.c | 7 ++++ arch/arm64/kvm/guest.c | 82 +++++++++++++++++++++++++++++++++++++++ include/uapi/linux/kvm.h | 1 + 6 files changed, 105 insertions(+) (limited to 'include/uapi') diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index 74a7447a83a1..c93a7198c242 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -730,6 +730,9 @@ int kvm_arm_vcpu_arch_get_attr(struct kvm_vcpu *vcpu, int kvm_arm_vcpu_arch_has_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr); +long kvm_vm_ioctl_mte_copy_tags(struct kvm *kvm, + struct kvm_arm_copy_mte_tags *copy_tags); + /* Guest/host FPSIMD coordination helpers */ int kvm_arch_vcpu_run_map_fp(struct kvm_vcpu *vcpu); void kvm_arch_vcpu_load_fp(struct kvm_vcpu *vcpu); diff --git a/arch/arm64/include/asm/mte-def.h b/arch/arm64/include/asm/mte-def.h index cf241b0f0a42..626d359b396e 100644 --- a/arch/arm64/include/asm/mte-def.h +++ b/arch/arm64/include/asm/mte-def.h @@ -7,6 +7,7 @@ #define MTE_GRANULE_SIZE UL(16) #define MTE_GRANULE_MASK (~(MTE_GRANULE_SIZE - 1)) +#define MTE_GRANULES_PER_PAGE (PAGE_SIZE / MTE_GRANULE_SIZE) #define MTE_TAG_SHIFT 56 #define MTE_TAG_SIZE 4 #define MTE_TAG_MASK GENMASK((MTE_TAG_SHIFT + (MTE_TAG_SIZE - 1)), MTE_TAG_SHIFT) diff --git a/arch/arm64/include/uapi/asm/kvm.h b/arch/arm64/include/uapi/asm/kvm.h index 24223adae150..b3edde68bc3e 100644 --- a/arch/arm64/include/uapi/asm/kvm.h +++ b/arch/arm64/include/uapi/asm/kvm.h @@ -184,6 +184,17 @@ struct kvm_vcpu_events { __u32 reserved[12]; }; +struct kvm_arm_copy_mte_tags { + __u64 guest_ipa; + __u64 length; + void __user *addr; + __u64 flags; + __u64 reserved[2]; +}; + +#define KVM_ARM_TAGS_TO_GUEST 0 +#define KVM_ARM_TAGS_FROM_GUEST 1 + /* If you need to interpret the index values, here is the key: */ #define KVM_REG_ARM_COPROC_MASK 0x000000000FFF0000 #define KVM_REG_ARM_COPROC_SHIFT 16 diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c index 28ce26a68f09..511f3716fe33 100644 --- a/arch/arm64/kvm/arm.c +++ b/arch/arm64/kvm/arm.c @@ -1359,6 +1359,13 @@ long kvm_arch_vm_ioctl(struct file *filp, return 0; } + case KVM_ARM_MTE_COPY_TAGS: { + struct kvm_arm_copy_mte_tags copy_tags; + + if (copy_from_user(©_tags, argp, sizeof(copy_tags))) + return -EFAULT; + return kvm_vm_ioctl_mte_copy_tags(kvm, ©_tags); + } default: return -EINVAL; } diff --git a/arch/arm64/kvm/guest.c b/arch/arm64/kvm/guest.c index 5cb4a1cd5603..4ddb20017b2f 100644 --- a/arch/arm64/kvm/guest.c +++ b/arch/arm64/kvm/guest.c @@ -995,3 +995,85 @@ int kvm_arm_vcpu_arch_has_attr(struct kvm_vcpu *vcpu, return ret; } + +long kvm_vm_ioctl_mte_copy_tags(struct kvm *kvm, + struct kvm_arm_copy_mte_tags *copy_tags) +{ + gpa_t guest_ipa = copy_tags->guest_ipa; + size_t length = copy_tags->length; + void __user *tags = copy_tags->addr; + gpa_t gfn; + bool write = !(copy_tags->flags & KVM_ARM_TAGS_FROM_GUEST); + int ret = 0; + + if (!kvm_has_mte(kvm)) + return -EINVAL; + + if (copy_tags->reserved[0] || copy_tags->reserved[1]) + return -EINVAL; + + if (copy_tags->flags & ~KVM_ARM_TAGS_FROM_GUEST) + return -EINVAL; + + if (length & ~PAGE_MASK || guest_ipa & ~PAGE_MASK) + return -EINVAL; + + gfn = gpa_to_gfn(guest_ipa); + + mutex_lock(&kvm->slots_lock); + + while (length > 0) { + kvm_pfn_t pfn = gfn_to_pfn_prot(kvm, gfn, write, NULL); + void *maddr; + unsigned long num_tags; + struct page *page; + + if (is_error_noslot_pfn(pfn)) { + ret = -EFAULT; + goto out; + } + + page = pfn_to_online_page(pfn); + if (!page) { + /* Reject ZONE_DEVICE memory */ + ret = -EFAULT; + goto out; + } + maddr = page_address(page); + + if (!write) { + if (test_bit(PG_mte_tagged, &page->flags)) + num_tags = mte_copy_tags_to_user(tags, maddr, + MTE_GRANULES_PER_PAGE); + else + /* No tags in memory, so write zeros */ + num_tags = MTE_GRANULES_PER_PAGE - + clear_user(tags, MTE_GRANULES_PER_PAGE); + kvm_release_pfn_clean(pfn); + } else { + num_tags = mte_copy_tags_from_user(maddr, tags, + MTE_GRANULES_PER_PAGE); + kvm_release_pfn_dirty(pfn); + } + + if (num_tags != MTE_GRANULES_PER_PAGE) { + ret = -EFAULT; + goto out; + } + + /* Set the flag after checking the write completed fully */ + if (write) + set_bit(PG_mte_tagged, &page->flags); + + gfn++; + tags += num_tags; + length -= PAGE_SIZE; + } + +out: + mutex_unlock(&kvm->slots_lock); + /* If some data has been copied report the number of bytes copied */ + if (length != copy_tags->length) + return copy_tags->length - length; + return ret; +} diff --git a/include/uapi/linux/kvm.h b/include/uapi/linux/kvm.h index d4da58ddcad7..da1edd2b4046 100644 --- a/include/uapi/linux/kvm.h +++ b/include/uapi/linux/kvm.h @@ -1429,6 +1429,7 @@ struct kvm_s390_ucas_mapping { /* Available with KVM_CAP_PMU_EVENT_FILTER */ #define KVM_SET_PMU_EVENT_FILTER _IOW(KVMIO, 0xb2, struct kvm_pmu_event_filter) #define KVM_PPC_SVM_OFF _IO(KVMIO, 0xb3) +#define KVM_ARM_MTE_COPY_TAGS _IOR(KVMIO, 0xb4, struct kvm_arm_copy_mte_tags) /* ioctl for vm fd */ #define KVM_CREATE_DEVICE _IOWR(KVMIO, 0xe0, struct kvm_create_device) -- cgit v1.2.3 From b87cc116c7e1bc62a84d8c46acd401db179edb11 Mon Sep 17 00:00:00 2001 From: Bharata B Rao Date: Mon, 21 Jun 2021 14:20:02 +0530 Subject: KVM: PPC: Book3S HV: Add KVM_CAP_PPC_RPT_INVALIDATE capability Now that we have H_RPT_INVALIDATE fully implemented, enable support for the same via KVM_CAP_PPC_RPT_INVALIDATE KVM capability Signed-off-by: Bharata B Rao Reviewed-by: David Gibson Signed-off-by: Michael Ellerman Link: https://lore.kernel.org/r/20210621085003.904767-6-bharata@linux.ibm.com --- Documentation/virt/kvm/api.rst | 18 ++++++++++++++++++ arch/powerpc/kvm/powerpc.c | 3 +++ include/uapi/linux/kvm.h | 1 + 3 files changed, 22 insertions(+) (limited to 'include/uapi') diff --git a/Documentation/virt/kvm/api.rst b/Documentation/virt/kvm/api.rst index 7fcb2fd38f42..9977e845633f 100644 --- a/Documentation/virt/kvm/api.rst +++ b/Documentation/virt/kvm/api.rst @@ -6362,6 +6362,24 @@ default. See Documentation/x86/sgx/2.Kernel-internals.rst for more details. +7.26 KVM_CAP_PPC_RPT_INVALIDATE +------------------------------- + +:Capability: KVM_CAP_PPC_RPT_INVALIDATE +:Architectures: ppc +:Type: vm + +This capability indicates that the kernel is capable of handling +H_RPT_INVALIDATE hcall. + +In order to enable the use of H_RPT_INVALIDATE in the guest, +user space might have to advertise it for the guest. For example, +IBM pSeries (sPAPR) guest starts using it if "hcall-rpt-invalidate" is +present in the "ibm,hypertas-functions" device-tree property. + +This capability is enabled for hypervisors on platforms like POWER9 +that support radix MMU. + 8. Other capabilities. ====================== diff --git a/arch/powerpc/kvm/powerpc.c b/arch/powerpc/kvm/powerpc.c index a2a68a958fa0..be33b5321a76 100644 --- a/arch/powerpc/kvm/powerpc.c +++ b/arch/powerpc/kvm/powerpc.c @@ -682,6 +682,9 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) r = !!(hv_enabled && kvmppc_hv_ops->enable_dawr1 && !kvmppc_hv_ops->enable_dawr1(NULL)); break; + case KVM_CAP_PPC_RPT_INVALIDATE: + r = 1; + break; #endif default: r = 0; diff --git a/include/uapi/linux/kvm.h b/include/uapi/linux/kvm.h index 3fd9a7e9d90c..613198a94c43 100644 --- a/include/uapi/linux/kvm.h +++ b/include/uapi/linux/kvm.h @@ -1082,6 +1082,7 @@ struct kvm_ppc_resize_hpt { #define KVM_CAP_SGX_ATTRIBUTE 196 #define KVM_CAP_VM_COPY_ENC_CONTEXT_FROM 197 #define KVM_CAP_PTP_KVM 198 +#define KVM_CAP_PPC_RPT_INVALIDATE 199 #ifdef KVM_CAP_IRQ_ROUTING -- cgit v1.2.3 From bf22a6976897977b0a3f1aeba6823c959fc4fdae Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Thu, 22 Apr 2021 21:44:23 +0200 Subject: futex: Provide FUTEX_LOCK_PI2 to support clock selection The FUTEX_LOCK_PI futex operand uses a CLOCK_REALTIME based absolute timeout since it was implemented, but it does not require that the FUTEX_CLOCK_REALTIME flag is set, because that was introduced later. In theory as none of the user space implementations can set the FUTEX_CLOCK_REALTIME flag on this operand, it would be possible to creatively abuse it and make the meaning invers, i.e. select CLOCK_REALTIME when not set and CLOCK_MONOTONIC when set. But that's a nasty hackery. Another option would be to have a new FUTEX_CLOCK_MONOTONIC flag only for FUTEX_LOCK_PI, but that's also awkward because it does not allow libraries to handle the timeout clock selection consistently. So provide a new FUTEX_LOCK_PI2 operand which implements the timeout semantics which the other operands use and leave FUTEX_LOCK_PI alone. Reported-by: Kurt Kanzenbach Signed-off-by: Thomas Gleixner Signed-off-by: Peter Zijlstra (Intel) Link: https://lore.kernel.org/r/20210422194705.440773992@linutronix.de --- include/uapi/linux/futex.h | 2 ++ kernel/futex.c | 7 ++++++- 2 files changed, 8 insertions(+), 1 deletion(-) (limited to 'include/uapi') diff --git a/include/uapi/linux/futex.h b/include/uapi/linux/futex.h index a89eb0accd5e..235e5b2facaa 100644 --- a/include/uapi/linux/futex.h +++ b/include/uapi/linux/futex.h @@ -21,6 +21,7 @@ #define FUTEX_WAKE_BITSET 10 #define FUTEX_WAIT_REQUEUE_PI 11 #define FUTEX_CMP_REQUEUE_PI 12 +#define FUTEX_LOCK_PI2 13 #define FUTEX_PRIVATE_FLAG 128 #define FUTEX_CLOCK_REALTIME 256 @@ -32,6 +33,7 @@ #define FUTEX_CMP_REQUEUE_PRIVATE (FUTEX_CMP_REQUEUE | FUTEX_PRIVATE_FLAG) #define FUTEX_WAKE_OP_PRIVATE (FUTEX_WAKE_OP | FUTEX_PRIVATE_FLAG) #define FUTEX_LOCK_PI_PRIVATE (FUTEX_LOCK_PI | FUTEX_PRIVATE_FLAG) +#define FUTEX_LOCK_PI2_PRIVATE (FUTEX_LOCK_PI2 | FUTEX_PRIVATE_FLAG) #define FUTEX_UNLOCK_PI_PRIVATE (FUTEX_UNLOCK_PI | FUTEX_PRIVATE_FLAG) #define FUTEX_TRYLOCK_PI_PRIVATE (FUTEX_TRYLOCK_PI | FUTEX_PRIVATE_FLAG) #define FUTEX_WAIT_BITSET_PRIVATE (FUTEX_WAIT_BITSET | FUTEX_PRIVATE_FLAG) diff --git a/kernel/futex.c b/kernel/futex.c index f820439a8aae..f832b6434625 100644 --- a/kernel/futex.c +++ b/kernel/futex.c @@ -3707,12 +3707,14 @@ long do_futex(u32 __user *uaddr, int op, u32 val, ktime_t *timeout, if (op & FUTEX_CLOCK_REALTIME) { flags |= FLAGS_CLOCKRT; - if (cmd != FUTEX_WAIT_BITSET && cmd != FUTEX_WAIT_REQUEUE_PI) + if (cmd != FUTEX_WAIT_BITSET && cmd != FUTEX_WAIT_REQUEUE_PI && + cmd != FUTEX_LOCK_PI2) return -ENOSYS; } switch (cmd) { case FUTEX_LOCK_PI: + case FUTEX_LOCK_PI2: case FUTEX_UNLOCK_PI: case FUTEX_TRYLOCK_PI: case FUTEX_WAIT_REQUEUE_PI: @@ -3740,6 +3742,8 @@ long do_futex(u32 __user *uaddr, int op, u32 val, ktime_t *timeout, return futex_wake_op(uaddr, flags, uaddr2, val, val2, val3); case FUTEX_LOCK_PI: flags |= FLAGS_CLOCKRT; + fallthrough; + case FUTEX_LOCK_PI2: return futex_lock_pi(uaddr, flags, timeout, 0); case FUTEX_UNLOCK_PI: return futex_unlock_pi(uaddr, flags); @@ -3760,6 +3764,7 @@ static __always_inline bool futex_cmd_has_timeout(u32 cmd) switch (cmd) { case FUTEX_WAIT: case FUTEX_LOCK_PI: + case FUTEX_LOCK_PI2: case FUTEX_WAIT_BITSET: case FUTEX_WAIT_REQUEUE_PI: return true; -- cgit v1.2.3 From 6d33cabf2baf304730d01a942095416b3a8329ab Mon Sep 17 00:00:00 2001 From: Kees Cook Date: Wed, 16 Jun 2021 13:26:15 -0700 Subject: RDMA/core: Use flexible array for mad data In preparation for FORTIFY_SOURCE performing compile-time and run-time field bounds checking for memcpy(), memmove(), and memset(), avoid intentionally reading across neighboring array fields. Without a flexible array, this looks like an attempt to perform a memcpy() read beyond the end of the packet->mad.data array: drivers/infiniband/core/user_mad.c: memcpy(packet->msg->mad, packet->mad.data, IB_MGMT_MAD_HDR); Switch from [0] to [] to use the appropriately handled type for trailing bytes. Link: https://lore.kernel.org/r/20210616202615.1247242-1-keescook@chromium.org Signed-off-by: Kees Cook Signed-off-by: Jason Gunthorpe --- include/uapi/rdma/ib_user_mad.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/uapi') diff --git a/include/uapi/rdma/ib_user_mad.h b/include/uapi/rdma/ib_user_mad.h index 90c0cf228020..10b5f6a4c677 100644 --- a/include/uapi/rdma/ib_user_mad.h +++ b/include/uapi/rdma/ib_user_mad.h @@ -143,7 +143,7 @@ struct ib_user_mad_hdr { */ struct ib_user_mad { struct ib_user_mad_hdr hdr; - __aligned_u64 data[0]; + __aligned_u64 data[]; }; /* -- cgit v1.2.3 From 336529518e9724d4cecabc622e57bcdce02e7c61 Mon Sep 17 00:00:00 2001 From: Aharon Landau Date: Wed, 16 Jun 2021 10:57:39 +0300 Subject: RDMA/mlx5: Support real-time timestamp directly from the device Currently, if the user asks for a real-time timestamp, the device will return a free-running one, and the timestamp will be translated to real-time in the user-space. When the device supports only real-time timestamp and not free-running, the creation of the QP will fail even though the user needs supported the real-time one. To prevent this, we will return the real-time timestamp directly from the device. Link: https://lore.kernel.org/r/c6cfc8e6f038575c5c2de6505830f7e74e4de80d.1623829775.git.leonro@nvidia.com Signed-off-by: Aharon Landau Reviewed-by: Maor Gottlieb Signed-off-by: Leon Romanovsky Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/mlx5/cq.c | 6 +++++- drivers/infiniband/hw/mlx5/main.c | 6 ++++++ drivers/infiniband/hw/mlx5/mlx5_ib.h | 7 +++++++ drivers/infiniband/hw/mlx5/qp.c | 30 +++++++++++++++++++++++++----- include/uapi/rdma/mlx5-abi.h | 2 ++ 5 files changed, 45 insertions(+), 6 deletions(-) (limited to 'include/uapi') diff --git a/drivers/infiniband/hw/mlx5/cq.c b/drivers/infiniband/hw/mlx5/cq.c index 9ce01f729673..d130fc71c395 100644 --- a/drivers/infiniband/hw/mlx5/cq.c +++ b/drivers/infiniband/hw/mlx5/cq.c @@ -725,7 +725,8 @@ static int create_cq_user(struct mlx5_ib_dev *dev, struct ib_udata *udata, return -EFAULT; if ((ucmd.flags & ~(MLX5_IB_CREATE_CQ_FLAGS_CQE_128B_PAD | - MLX5_IB_CREATE_CQ_FLAGS_UAR_PAGE_INDEX))) + MLX5_IB_CREATE_CQ_FLAGS_UAR_PAGE_INDEX | + MLX5_IB_CREATE_CQ_FLAGS_REAL_TIME_TS))) return -EINVAL; if ((ucmd.cqe_size != 64 && ucmd.cqe_size != 128) || @@ -826,6 +827,9 @@ static int create_cq_user(struct mlx5_ib_dev *dev, struct ib_udata *udata, cq->private_flags |= MLX5_IB_CQ_PR_FLAGS_CQE_128_PAD; } + if (ucmd.flags & MLX5_IB_CREATE_CQ_FLAGS_REAL_TIME_TS) + cq->private_flags |= MLX5_IB_CQ_PR_FLAGS_REAL_TIME_TS; + MLX5_SET(create_cq_in, *cqb, uid, context->devx_uid); return 0; diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c index 644d5d0ac544..a273cbb9369f 100644 --- a/drivers/infiniband/hw/mlx5/main.c +++ b/drivers/infiniband/hw/mlx5/main.c @@ -1816,6 +1816,12 @@ static int set_ucontext_resp(struct ib_ucontext *uctx, if (MLX5_CAP_GEN(dev->mdev, ece_support)) resp->comp_mask |= MLX5_IB_ALLOC_UCONTEXT_RESP_MASK_ECE; + if (rt_supported(MLX5_CAP_GEN(dev->mdev, sq_ts_format)) && + rt_supported(MLX5_CAP_GEN(dev->mdev, rq_ts_format)) && + rt_supported(MLX5_CAP_ROCE(dev->mdev, qp_ts_format))) + resp->comp_mask |= + MLX5_IB_ALLOC_UCONTEXT_RESP_MASK_REAL_TIME_TS; + resp->num_dyn_bfregs = bfregi->num_dyn_bfregs; return 0; } diff --git a/drivers/infiniband/hw/mlx5/mlx5_ib.h b/drivers/infiniband/hw/mlx5/mlx5_ib.h index e9a3f34a30b8..a69874784387 100644 --- a/drivers/infiniband/hw/mlx5/mlx5_ib.h +++ b/drivers/infiniband/hw/mlx5/mlx5_ib.h @@ -550,6 +550,7 @@ static inline const struct mlx5_umr_wr *umr_wr(const struct ib_send_wr *wr) enum mlx5_ib_cq_pr_flags { MLX5_IB_CQ_PR_FLAGS_CQE_128_PAD = 1 << 0, + MLX5_IB_CQ_PR_FLAGS_REAL_TIME_TS = 1 << 1, }; struct mlx5_ib_cq { @@ -1614,4 +1615,10 @@ static inline bool mlx5_ib_lag_should_assign_affinity(struct mlx5_ib_dev *dev) (MLX5_CAP_GEN(dev->mdev, num_lag_ports) > 1 && MLX5_CAP_GEN(dev->mdev, lag_tx_port_affinity)); } + +static inline bool rt_supported(int ts_cap) +{ + return ts_cap == MLX5_TIMESTAMP_FORMAT_CAP_REAL_TIME || + ts_cap == MLX5_TIMESTAMP_FORMAT_CAP_FREE_RUNNING_AND_REAL_TIME; +} #endif /* MLX5_IB_H */ diff --git a/drivers/infiniband/hw/mlx5/qp.c b/drivers/infiniband/hw/mlx5/qp.c index 0e3babac62db..737b1f6eba0b 100644 --- a/drivers/infiniband/hw/mlx5/qp.c +++ b/drivers/infiniband/hw/mlx5/qp.c @@ -1180,8 +1180,16 @@ static bool fr_supported(int ts_cap) } static int get_ts_format(struct mlx5_ib_dev *dev, struct mlx5_ib_cq *cq, - bool fr_sup) + bool fr_sup, bool rt_sup) { + if (cq->private_flags & MLX5_IB_CQ_PR_FLAGS_REAL_TIME_TS) { + if (!rt_sup) { + mlx5_ib_dbg(dev, + "Real time TS format is not supported\n"); + return -EOPNOTSUPP; + } + return MLX5_TIMESTAMP_FORMAT_REAL_TIME; + } if (cq->create_flags & IB_UVERBS_CQ_FLAGS_TIMESTAMP_COMPLETION) { if (!fr_sup) { mlx5_ib_dbg(dev, @@ -1198,14 +1206,16 @@ static int get_rq_ts_format(struct mlx5_ib_dev *dev, struct mlx5_ib_cq *recv_cq) { u8 ts_cap = MLX5_CAP_GEN(dev->mdev, rq_ts_format); - return get_ts_format(dev, recv_cq, fr_supported(ts_cap)); + return get_ts_format(dev, recv_cq, fr_supported(ts_cap), + rt_supported(ts_cap)); } static int get_sq_ts_format(struct mlx5_ib_dev *dev, struct mlx5_ib_cq *send_cq) { u8 ts_cap = MLX5_CAP_GEN(dev->mdev, sq_ts_format); - return get_ts_format(dev, send_cq, fr_supported(ts_cap)); + return get_ts_format(dev, send_cq, fr_supported(ts_cap), + rt_supported(ts_cap)); } static int get_qp_ts_format(struct mlx5_ib_dev *dev, struct mlx5_ib_cq *send_cq, @@ -1213,18 +1223,28 @@ static int get_qp_ts_format(struct mlx5_ib_dev *dev, struct mlx5_ib_cq *send_cq, { u8 ts_cap = MLX5_CAP_ROCE(dev->mdev, qp_ts_format); bool fr_sup = fr_supported(ts_cap); + bool rt_sup = rt_supported(ts_cap); u8 default_ts = fr_sup ? MLX5_TIMESTAMP_FORMAT_FREE_RUNNING : MLX5_TIMESTAMP_FORMAT_DEFAULT; int send_ts_format = - send_cq ? get_ts_format(dev, send_cq, fr_sup) : + send_cq ? get_ts_format(dev, send_cq, fr_sup, rt_sup) : default_ts; int recv_ts_format = - recv_cq ? get_ts_format(dev, recv_cq, fr_sup) : + recv_cq ? get_ts_format(dev, recv_cq, fr_sup, rt_sup) : default_ts; if (send_ts_format < 0 || recv_ts_format < 0) return -EOPNOTSUPP; + if (send_ts_format != MLX5_TIMESTAMP_FORMAT_DEFAULT && + recv_ts_format != MLX5_TIMESTAMP_FORMAT_DEFAULT && + send_ts_format != recv_ts_format) { + mlx5_ib_dbg( + dev, + "The send ts_format does not match the receive ts_format\n"); + return -EOPNOTSUPP; + } + return send_ts_format == default_ts ? recv_ts_format : send_ts_format; } diff --git a/include/uapi/rdma/mlx5-abi.h b/include/uapi/rdma/mlx5-abi.h index 27905a0268c9..82e3bc1eb57d 100644 --- a/include/uapi/rdma/mlx5-abi.h +++ b/include/uapi/rdma/mlx5-abi.h @@ -101,6 +101,7 @@ enum mlx5_ib_alloc_ucontext_resp_mask { MLX5_IB_ALLOC_UCONTEXT_RESP_MASK_CORE_CLOCK_OFFSET = 1UL << 0, MLX5_IB_ALLOC_UCONTEXT_RESP_MASK_DUMP_FILL_MKEY = 1UL << 1, MLX5_IB_ALLOC_UCONTEXT_RESP_MASK_ECE = 1UL << 2, + MLX5_IB_ALLOC_UCONTEXT_RESP_MASK_REAL_TIME_TS = 1UL << 4, }; enum mlx5_user_cmds_supp_uhw { @@ -270,6 +271,7 @@ struct mlx5_ib_query_device_resp { enum mlx5_ib_create_cq_flags { MLX5_IB_CREATE_CQ_FLAGS_CQE_128B_PAD = 1 << 0, MLX5_IB_CREATE_CQ_FLAGS_UAR_PAGE_INDEX = 1 << 1, + MLX5_IB_CREATE_CQ_FLAGS_REAL_TIME_TS = 1 << 2, }; struct mlx5_ib_create_cq { -- cgit v1.2.3 From 913d026fbfaf114ff87afcc77fa4e9309f87f114 Mon Sep 17 00:00:00 2001 From: Ido Schimmel Date: Tue, 22 Jun 2021 09:50:47 +0300 Subject: ethtool: Document correct attribute type 'ETHTOOL_A_MODULE_EEPROM_DATA' is a binary attribute, not a nested one. Signed-off-by: Ido Schimmel Signed-off-by: David S. Miller --- Documentation/networking/ethtool-netlink.rst | 2 +- include/uapi/linux/ethtool_netlink.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'include/uapi') diff --git a/Documentation/networking/ethtool-netlink.rst b/Documentation/networking/ethtool-netlink.rst index c3600f9c8988..8ae644f800f0 100644 --- a/Documentation/networking/ethtool-netlink.rst +++ b/Documentation/networking/ethtool-netlink.rst @@ -1388,7 +1388,7 @@ Kernel response contents: +---------------------------------------------+--------+---------------------+ | ``ETHTOOL_A_MODULE_EEPROM_HEADER`` | nested | reply header | +---------------------------------------------+--------+---------------------+ - | ``ETHTOOL_A_MODULE_EEPROM_DATA`` | nested | array of bytes from | + | ``ETHTOOL_A_MODULE_EEPROM_DATA`` | binary | array of bytes from | | | | module EEPROM | +---------------------------------------------+--------+---------------------+ diff --git a/include/uapi/linux/ethtool_netlink.h b/include/uapi/linux/ethtool_netlink.h index 825cfda1c5d5..c7135c9c37a5 100644 --- a/include/uapi/linux/ethtool_netlink.h +++ b/include/uapi/linux/ethtool_netlink.h @@ -675,7 +675,7 @@ enum { ETHTOOL_A_MODULE_EEPROM_PAGE, /* u8 */ ETHTOOL_A_MODULE_EEPROM_BANK, /* u8 */ ETHTOOL_A_MODULE_EEPROM_I2C_ADDRESS, /* u8 */ - ETHTOOL_A_MODULE_EEPROM_DATA, /* nested */ + ETHTOOL_A_MODULE_EEPROM_DATA, /* binary */ __ETHTOOL_A_MODULE_EEPROM_CNT, ETHTOOL_A_MODULE_EEPROM_MAX = (__ETHTOOL_A_MODULE_EEPROM_CNT - 1) -- cgit v1.2.3 From 3190b649b4d9391be7bde3edd8e924e451c5d2f6 Mon Sep 17 00:00:00 2001 From: Xin Long Date: Tue, 22 Jun 2021 14:04:49 -0400 Subject: sctp: add SCTP_PLPMTUD_PROBE_INTERVAL sockopt for sock/asoc/transport With this socket option, users can change probe_interval for a transport, asoc or sock after it's created. Note that if the change is for an asoc, also apply the change to each transport in this asoc. Signed-off-by: Xin Long Acked-by: Marcelo Ricardo Leitner Signed-off-by: David S. Miller --- include/uapi/linux/sctp.h | 8 ++++ net/sctp/socket.c | 118 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 126 insertions(+) (limited to 'include/uapi') diff --git a/include/uapi/linux/sctp.h b/include/uapi/linux/sctp.h index cb78e7a739da..c4ff1ebd8bcc 100644 --- a/include/uapi/linux/sctp.h +++ b/include/uapi/linux/sctp.h @@ -141,6 +141,7 @@ typedef __s32 sctp_assoc_t; #define SCTP_EXPOSE_POTENTIALLY_FAILED_STATE 131 #define SCTP_EXPOSE_PF_STATE SCTP_EXPOSE_POTENTIALLY_FAILED_STATE #define SCTP_REMOTE_UDP_ENCAPS_PORT 132 +#define SCTP_PLPMTUD_PROBE_INTERVAL 133 /* PR-SCTP policies */ #define SCTP_PR_SCTP_NONE 0x0000 @@ -1213,4 +1214,11 @@ enum sctp_sched_type { SCTP_SS_MAX = SCTP_SS_RR }; +/* Probe Interval socket option */ +struct sctp_probeinterval { + sctp_assoc_t spi_assoc_id; + struct sockaddr_storage spi_address; + __u32 spi_interval; +}; + #endif /* _UAPI_SCTP_H */ diff --git a/net/sctp/socket.c b/net/sctp/socket.c index d2960ab665a5..aba576f53458 100644 --- a/net/sctp/socket.c +++ b/net/sctp/socket.c @@ -4481,6 +4481,58 @@ static int sctp_setsockopt_encap_port(struct sock *sk, return 0; } +static int sctp_setsockopt_probe_interval(struct sock *sk, + struct sctp_probeinterval *params, + unsigned int optlen) +{ + struct sctp_association *asoc; + struct sctp_transport *t; + __u32 probe_interval; + + if (optlen != sizeof(*params)) + return -EINVAL; + + probe_interval = params->spi_interval; + if (probe_interval && probe_interval < SCTP_PROBE_TIMER_MIN) + return -EINVAL; + + /* If an address other than INADDR_ANY is specified, and + * no transport is found, then the request is invalid. + */ + if (!sctp_is_any(sk, (union sctp_addr *)¶ms->spi_address)) { + t = sctp_addr_id2transport(sk, ¶ms->spi_address, + params->spi_assoc_id); + if (!t) + return -EINVAL; + + t->probe_interval = msecs_to_jiffies(probe_interval); + return 0; + } + + /* Get association, if assoc_id != SCTP_FUTURE_ASSOC and the + * socket is a one to many style socket, and an association + * was not found, then the id was invalid. + */ + asoc = sctp_id2assoc(sk, params->spi_assoc_id); + if (!asoc && params->spi_assoc_id != SCTP_FUTURE_ASSOC && + sctp_style(sk, UDP)) + return -EINVAL; + + /* If changes are for association, also apply probe_interval to + * each transport. + */ + if (asoc) { + list_for_each_entry(t, &asoc->peer.transport_addr_list, transports) + t->probe_interval = msecs_to_jiffies(probe_interval); + + asoc->probe_interval = msecs_to_jiffies(probe_interval); + return 0; + } + + sctp_sk(sk)->probe_interval = probe_interval; + return 0; +} + /* API 6.2 setsockopt(), getsockopt() * * Applications use setsockopt() and getsockopt() to set or retrieve @@ -4703,6 +4755,9 @@ static int sctp_setsockopt(struct sock *sk, int level, int optname, case SCTP_REMOTE_UDP_ENCAPS_PORT: retval = sctp_setsockopt_encap_port(sk, kopt, optlen); break; + case SCTP_PLPMTUD_PROBE_INTERVAL: + retval = sctp_setsockopt_probe_interval(sk, kopt, optlen); + break; default: retval = -ENOPROTOOPT; break; @@ -7906,6 +7961,66 @@ out: return 0; } +static int sctp_getsockopt_probe_interval(struct sock *sk, int len, + char __user *optval, + int __user *optlen) +{ + struct sctp_probeinterval params; + struct sctp_association *asoc; + struct sctp_transport *t; + __u32 probe_interval; + + if (len < sizeof(params)) + return -EINVAL; + + len = sizeof(params); + if (copy_from_user(¶ms, optval, len)) + return -EFAULT; + + /* If an address other than INADDR_ANY is specified, and + * no transport is found, then the request is invalid. + */ + if (!sctp_is_any(sk, (union sctp_addr *)¶ms.spi_address)) { + t = sctp_addr_id2transport(sk, ¶ms.spi_address, + params.spi_assoc_id); + if (!t) { + pr_debug("%s: failed no transport\n", __func__); + return -EINVAL; + } + + probe_interval = jiffies_to_msecs(t->probe_interval); + goto out; + } + + /* Get association, if assoc_id != SCTP_FUTURE_ASSOC and the + * socket is a one to many style socket, and an association + * was not found, then the id was invalid. + */ + asoc = sctp_id2assoc(sk, params.spi_assoc_id); + if (!asoc && params.spi_assoc_id != SCTP_FUTURE_ASSOC && + sctp_style(sk, UDP)) { + pr_debug("%s: failed no association\n", __func__); + return -EINVAL; + } + + if (asoc) { + probe_interval = jiffies_to_msecs(asoc->probe_interval); + goto out; + } + + probe_interval = sctp_sk(sk)->probe_interval; + +out: + params.spi_interval = probe_interval; + if (copy_to_user(optval, ¶ms, len)) + return -EFAULT; + + if (put_user(len, optlen)) + return -EFAULT; + + return 0; +} + static int sctp_getsockopt(struct sock *sk, int level, int optname, char __user *optval, int __user *optlen) { @@ -8129,6 +8244,9 @@ static int sctp_getsockopt(struct sock *sk, int level, int optname, case SCTP_REMOTE_UDP_ENCAPS_PORT: retval = sctp_getsockopt_encap_port(sk, len, optval, optlen); break; + case SCTP_PLPMTUD_PROBE_INTERVAL: + retval = sctp_getsockopt_probe_interval(sk, len, optval, optlen); + break; default: retval = -ENOPROTOOPT; break; -- cgit v1.2.3 From dd3e4fc75b4ab8186a133cfe9d49666a2f8186e0 Mon Sep 17 00:00:00 2001 From: Avraham Stern Date: Fri, 18 Jun 2021 13:41:36 +0300 Subject: nl80211/cfg80211: add BSS color to NDP ranging parameters In NDP ranging, the initiator need to set the BSS color in the NDP to the BSS color of the responder. Add the BSS color as a parameter for NDP ranging. Signed-off-by: Avraham Stern Signed-off-by: Luca Coelho Link: https://lore.kernel.org/r/iwlwifi.20210618133832.f097a6144b59.I27dec8b994df52e691925ea61be4dd4fa6d396c0@changeid Signed-off-by: Johannes Berg --- include/net/cfg80211.h | 8 ++++++-- include/uapi/linux/nl80211.h | 6 +++++- net/wireless/pmsr.c | 12 ++++++++++++ 3 files changed, 23 insertions(+), 3 deletions(-) (limited to 'include/uapi') diff --git a/include/net/cfg80211.h b/include/net/cfg80211.h index b3bc58ec9098..c6812945b4b8 100644 --- a/include/net/cfg80211.h +++ b/include/net/cfg80211.h @@ -7,7 +7,7 @@ * Copyright 2006-2010 Johannes Berg * Copyright 2013-2014 Intel Mobile Communications GmbH * Copyright 2015-2017 Intel Deutschland GmbH - * Copyright (C) 2018-2020 Intel Corporation + * Copyright (C) 2018-2021 Intel Corporation */ #include @@ -3521,7 +3521,10 @@ struct cfg80211_pmsr_result { * If neither @trigger_based nor @non_trigger_based is set, * EDCA based ranging will be used. * @lmr_feedback: negotiate for I2R LMR feedback. Only valid if either - * @trigger_based or @non_trigger_based is set. + * @trigger_based or @non_trigger_based is set. + * @bss_color: the bss color of the responder. Optional. Set to zero to + * indicate the driver should set the BSS color. Only valid if + * @non_trigger_based or @trigger_based is set. * * See also nl80211 for the respective attribute documentation. */ @@ -3539,6 +3542,7 @@ struct cfg80211_pmsr_ftm_request_peer { u8 burst_duration; u8 ftms_per_burst; u8 ftmr_retries; + u8 bss_color; }; /** diff --git a/include/uapi/linux/nl80211.h b/include/uapi/linux/nl80211.h index f962c06e9818..771f238ccff1 100644 --- a/include/uapi/linux/nl80211.h +++ b/include/uapi/linux/nl80211.h @@ -11,7 +11,7 @@ * Copyright 2008 Jouni Malinen * Copyright 2008 Colin McCabe * Copyright 2015-2017 Intel Deutschland GmbH - * Copyright (C) 2018-2020 Intel Corporation + * Copyright (C) 2018-2021 Intel Corporation * * Permission to use, copy, modify, and/or distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -6912,6 +6912,9 @@ enum nl80211_peer_measurement_ftm_capa { * @NL80211_PMSR_FTM_REQ_ATTR_LMR_FEEDBACK: negotiate for LMR feedback. Only * valid if either %NL80211_PMSR_FTM_REQ_ATTR_TRIGGER_BASED or * %NL80211_PMSR_FTM_REQ_ATTR_NON_TRIGGER_BASED is set. + * @NL80211_PMSR_FTM_REQ_ATTR_BSS_COLOR: optional. The BSS color of the + * responder. Only valid if %NL80211_PMSR_FTM_REQ_ATTR_NON_TRIGGER_BASED + * or %NL80211_PMSR_FTM_REQ_ATTR_TRIGGER_BASED is set. * * @NUM_NL80211_PMSR_FTM_REQ_ATTR: internal * @NL80211_PMSR_FTM_REQ_ATTR_MAX: highest attribute number @@ -6931,6 +6934,7 @@ enum nl80211_peer_measurement_ftm_req { NL80211_PMSR_FTM_REQ_ATTR_TRIGGER_BASED, NL80211_PMSR_FTM_REQ_ATTR_NON_TRIGGER_BASED, NL80211_PMSR_FTM_REQ_ATTR_LMR_FEEDBACK, + NL80211_PMSR_FTM_REQ_ATTR_BSS_COLOR, /* keep last */ NUM_NL80211_PMSR_FTM_REQ_ATTR, diff --git a/net/wireless/pmsr.c b/net/wireless/pmsr.c index d245968b74cb..328cf54bda82 100644 --- a/net/wireless/pmsr.c +++ b/net/wireless/pmsr.c @@ -168,6 +168,18 @@ static int pmsr_parse_ftm(struct cfg80211_registered_device *rdev, return -EINVAL; } + if (tb[NL80211_PMSR_FTM_REQ_ATTR_BSS_COLOR]) { + if (!out->ftm.non_trigger_based && !out->ftm.trigger_based) { + NL_SET_ERR_MSG_ATTR(info->extack, + tb[NL80211_PMSR_FTM_REQ_ATTR_BSS_COLOR], + "FTM: BSS color set for EDCA based ranging"); + return -EINVAL; + } + + out->ftm.bss_color = + nla_get_u8(tb[NL80211_PMSR_FTM_REQ_ATTR_BSS_COLOR]); + } + return 0; } -- cgit v1.2.3 From f4f8650588d35deafaa4a4e28cceb3557a71e711 Mon Sep 17 00:00:00 2001 From: Johannes Berg Date: Fri, 18 Jun 2021 13:41:52 +0300 Subject: cfg80211: allow advertising vendor-specific capabilities There may be cases where vendor-specific elements need to be used over the air. Rather than have driver or firmware add them and possibly cause problems that way, add them to the iftype-data band capabilities. This way we can advertise to userspace first, and use them in mac80211 next. Signed-off-by: Johannes Berg Signed-off-by: Luca Coelho Link: https://lore.kernel.org/r/iwlwifi.20210618133832.e8c4f0347276.Iee5964682b3e9ec51fc1cd57a7c62383eaf6ddd7@changeid Signed-off-by: Johannes Berg --- include/net/cfg80211.h | 7 +++++++ include/uapi/linux/nl80211.h | 3 +++ net/wireless/nl80211.c | 5 +++++ 3 files changed, 15 insertions(+) (limited to 'include/uapi') diff --git a/include/net/cfg80211.h b/include/net/cfg80211.h index 481e4e24800f..c93a2cd77920 100644 --- a/include/net/cfg80211.h +++ b/include/net/cfg80211.h @@ -371,11 +371,18 @@ struct ieee80211_sta_he_cap { * @he_cap: holds the HE capabilities * @he_6ghz_capa: HE 6 GHz capabilities, must be filled in for a * 6 GHz band channel (and 0 may be valid value). + * @vendor_elems: vendor element(s) to advertise + * @vendor_elems.data: vendor element(s) data + * @vendor_elems.len: vendor element(s) length */ struct ieee80211_sband_iftype_data { u16 types_mask; struct ieee80211_sta_he_cap he_cap; struct ieee80211_he_6ghz_capa he_6ghz_capa; + struct { + const u8 *data; + unsigned int len; + } vendor_elems; }; /** diff --git a/include/uapi/linux/nl80211.h b/include/uapi/linux/nl80211.h index 771f238ccff1..db474994fa73 100644 --- a/include/uapi/linux/nl80211.h +++ b/include/uapi/linux/nl80211.h @@ -3654,6 +3654,8 @@ enum nl80211_mpath_info { * defined * @NL80211_BAND_IFTYPE_ATTR_HE_6GHZ_CAPA: HE 6GHz band capabilities (__le16), * given for all 6 GHz band channels + * @NL80211_BAND_IFTYPE_ATTR_VENDOR_ELEMS: vendor element capabilities that are + * advertised on this band/for this iftype (binary) * @__NL80211_BAND_IFTYPE_ATTR_AFTER_LAST: internal use */ enum nl80211_band_iftype_attr { @@ -3665,6 +3667,7 @@ enum nl80211_band_iftype_attr { NL80211_BAND_IFTYPE_ATTR_HE_CAP_MCS_SET, NL80211_BAND_IFTYPE_ATTR_HE_CAP_PPE, NL80211_BAND_IFTYPE_ATTR_HE_6GHZ_CAPA, + NL80211_BAND_IFTYPE_ATTR_VENDOR_ELEMS, /* keep last */ __NL80211_BAND_IFTYPE_ATTR_AFTER_LAST, diff --git a/net/wireless/nl80211.c b/net/wireless/nl80211.c index 4c61cf66fe05..50eb405b0690 100644 --- a/net/wireless/nl80211.c +++ b/net/wireless/nl80211.c @@ -1731,6 +1731,11 @@ nl80211_send_iftype_data(struct sk_buff *msg, &iftdata->he_6ghz_capa)) return -ENOBUFS; + if (iftdata->vendor_elems.data && iftdata->vendor_elems.len && + nla_put(msg, NL80211_BAND_IFTYPE_ATTR_VENDOR_ELEMS, + iftdata->vendor_elems.len, iftdata->vendor_elems.data)) + return -ENOBUFS; + return 0; } -- cgit v1.2.3 From d12e339044a00ecae993b06672c38c168a92f0c3 Mon Sep 17 00:00:00 2001 From: Jonathan Marek Date: Fri, 23 Apr 2021 15:08:20 -0400 Subject: drm/msm: add MSM_BO_CACHED_COHERENT Add a new cache mode for creating coherent host-cached BOs. Signed-off-by: Jonathan Marek Reviewed-by: Jordan Crouse Link: https://lore.kernel.org/r/20210423190833.25319-5-jonathan@marek.ca Signed-off-by: Rob Clark --- drivers/gpu/drm/msm/adreno/adreno_device.c | 1 + drivers/gpu/drm/msm/msm_drv.c | 3 ++- drivers/gpu/drm/msm/msm_drv.h | 1 + drivers/gpu/drm/msm/msm_gem.c | 8 ++++++++ include/uapi/drm/msm_drm.h | 5 ++--- 5 files changed, 14 insertions(+), 4 deletions(-) (limited to 'include/uapi') diff --git a/drivers/gpu/drm/msm/adreno/adreno_device.c b/drivers/gpu/drm/msm/adreno/adreno_device.c index 600d445fabe8..b3337b93be91 100644 --- a/drivers/gpu/drm/msm/adreno/adreno_device.c +++ b/drivers/gpu/drm/msm/adreno/adreno_device.c @@ -466,6 +466,7 @@ static int adreno_bind(struct device *dev, struct device *master, void *data) config.rev.minor, config.rev.patchid); priv->is_a2xx = config.rev.core == 2; + priv->has_cached_coherent = config.rev.core >= 6; gpu = info->init(drm); if (IS_ERR(gpu)) { diff --git a/drivers/gpu/drm/msm/msm_drv.c b/drivers/gpu/drm/msm/msm_drv.c index 3ecf617e5f1a..917856bfdb94 100644 --- a/drivers/gpu/drm/msm/msm_drv.c +++ b/drivers/gpu/drm/msm/msm_drv.c @@ -41,9 +41,10 @@ * - 1.5.0 - Add SUBMITQUERY_QUERY ioctl * - 1.6.0 - Syncobj support * - 1.7.0 - Add MSM_PARAM_SUSPENDS to access suspend count + * - 1.8.0 - Add MSM_BO_CACHED_COHERENT for supported GPUs (a6xx) */ #define MSM_VERSION_MAJOR 1 -#define MSM_VERSION_MINOR 7 +#define MSM_VERSION_MINOR 8 #define MSM_VERSION_PATCHLEVEL 0 static const struct drm_mode_config_funcs mode_config_funcs = { diff --git a/drivers/gpu/drm/msm/msm_drv.h b/drivers/gpu/drm/msm/msm_drv.h index c33fc1293789..2f72eab83aa6 100644 --- a/drivers/gpu/drm/msm/msm_drv.h +++ b/drivers/gpu/drm/msm/msm_drv.h @@ -167,6 +167,7 @@ struct msm_drm_private { struct msm_file_private *lastctx; /* gpu is only set on open(), but we need this info earlier */ bool is_a2xx; + bool has_cached_coherent; struct drm_fb_helper *fbdev; diff --git a/drivers/gpu/drm/msm/msm_gem.c b/drivers/gpu/drm/msm/msm_gem.c index ffe94bd97219..8d19a0f289fd 100644 --- a/drivers/gpu/drm/msm/msm_gem.c +++ b/drivers/gpu/drm/msm/msm_gem.c @@ -445,6 +445,9 @@ static int msm_gem_pin_iova(struct drm_gem_object *obj, if (msm_obj->flags & MSM_BO_MAP_PRIV) prot |= IOMMU_PRIV; + if (msm_obj->flags & MSM_BO_CACHED_COHERENT) + prot |= IOMMU_CACHE; + GEM_WARN_ON(!msm_gem_is_locked(obj)); if (GEM_WARN_ON(msm_obj->madv != MSM_MADV_WILLNEED)) @@ -1158,6 +1161,7 @@ static int msm_gem_new_impl(struct drm_device *dev, uint32_t size, uint32_t flags, struct drm_gem_object **obj) { + struct msm_drm_private *priv = dev->dev_private; struct msm_gem_object *msm_obj; switch (flags & MSM_BO_CACHE_MASK) { @@ -1165,6 +1169,10 @@ static int msm_gem_new_impl(struct drm_device *dev, case MSM_BO_CACHED: case MSM_BO_WC: break; + case MSM_BO_CACHED_COHERENT: + if (priv->has_cached_coherent) + break; + /* fallthrough */ default: DRM_DEV_ERROR(dev->dev, "invalid cache flag: %x\n", (flags & MSM_BO_CACHE_MASK)); diff --git a/include/uapi/drm/msm_drm.h b/include/uapi/drm/msm_drm.h index 5596d7c37f9e..a92d90a6d96f 100644 --- a/include/uapi/drm/msm_drm.h +++ b/include/uapi/drm/msm_drm.h @@ -95,12 +95,11 @@ struct drm_msm_param { #define MSM_BO_CACHED 0x00010000 #define MSM_BO_WC 0x00020000 #define MSM_BO_UNCACHED 0x00040000 +#define MSM_BO_CACHED_COHERENT 0x080000 #define MSM_BO_FLAGS (MSM_BO_SCANOUT | \ MSM_BO_GPU_READONLY | \ - MSM_BO_CACHED | \ - MSM_BO_WC | \ - MSM_BO_UNCACHED) + MSM_BO_CACHE_MASK) struct drm_msm_gem_new { __u64 size; /* in */ -- cgit v1.2.3 From 9ef364432db4a11ff2dbee398d7ed06e93bdfe5e Mon Sep 17 00:00:00 2001 From: Jonathan Marek Date: Fri, 23 Apr 2021 15:08:21 -0400 Subject: drm/msm: deprecate MSM_BO_UNCACHED (map as writecombine instead) There shouldn't be any reason to ever use uncached over writecombine, so just use writecombine for MSM_BO_UNCACHED. Note: userspace never used MSM_BO_UNCACHED anyway Signed-off-by: Jonathan Marek Acked-by: Jordan Crouse Link: https://lore.kernel.org/r/20210423190833.25319-6-jonathan@marek.ca Signed-off-by: Rob Clark --- drivers/gpu/drm/msm/msm_gem.c | 4 +--- include/uapi/drm/msm_drm.h | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) (limited to 'include/uapi') diff --git a/drivers/gpu/drm/msm/msm_gem.c b/drivers/gpu/drm/msm/msm_gem.c index 8d19a0f289fd..1865919368f2 100644 --- a/drivers/gpu/drm/msm/msm_gem.c +++ b/drivers/gpu/drm/msm/msm_gem.c @@ -213,10 +213,8 @@ void msm_gem_put_pages(struct drm_gem_object *obj) static pgprot_t msm_gem_pgprot(struct msm_gem_object *msm_obj, pgprot_t prot) { - if (msm_obj->flags & MSM_BO_WC) + if (msm_obj->flags & (MSM_BO_WC|MSM_BO_UNCACHED)) return pgprot_writecombine(prot); - if (msm_obj->flags & MSM_BO_UNCACHED) - return pgprot_noncached(prot); return prot; } diff --git a/include/uapi/drm/msm_drm.h b/include/uapi/drm/msm_drm.h index a92d90a6d96f..f075851021c3 100644 --- a/include/uapi/drm/msm_drm.h +++ b/include/uapi/drm/msm_drm.h @@ -94,7 +94,7 @@ struct drm_msm_param { /* cache modes */ #define MSM_BO_CACHED 0x00010000 #define MSM_BO_WC 0x00020000 -#define MSM_BO_UNCACHED 0x00040000 +#define MSM_BO_UNCACHED 0x00040000 /* deprecated, use MSM_BO_WC */ #define MSM_BO_CACHED_COHERENT 0x080000 #define MSM_BO_FLAGS (MSM_BO_SCANOUT | \ -- cgit v1.2.3 From 55d444b310c64b084dcc62ba3e4dc3862269fb96 Mon Sep 17 00:00:00 2001 From: Kuniyuki Iwashima Date: Wed, 23 Jun 2021 08:35:29 +0900 Subject: tcp: Add stats for socket migration. This commit adds two stats for the socket migration feature to evaluate the effectiveness: LINUX_MIB_TCPMIGRATEREQ(SUCCESS|FAILURE). If the migration fails because of the own_req race in receiving ACK and sending SYN+ACK paths, we do not increment the failure stat. Then another CPU is responsible for the req. Link: https://lore.kernel.org/bpf/CAK6E8=cgFKuGecTzSCSQ8z3YJ_163C0uwO9yRvfDSE7vOe9mJA@mail.gmail.com/ Suggested-by: Yuchung Cheng Signed-off-by: Kuniyuki Iwashima Acked-by: Yuchung Cheng Signed-off-by: David S. Miller --- include/uapi/linux/snmp.h | 2 ++ net/core/sock_reuseport.c | 15 +++++++++++---- net/ipv4/inet_connection_sock.c | 15 +++++++++++++-- net/ipv4/proc.c | 2 ++ net/ipv4/tcp_minisocks.c | 3 +++ 5 files changed, 31 insertions(+), 6 deletions(-) (limited to 'include/uapi') diff --git a/include/uapi/linux/snmp.h b/include/uapi/linux/snmp.h index 26fc60ce9298..904909d020e2 100644 --- a/include/uapi/linux/snmp.h +++ b/include/uapi/linux/snmp.h @@ -290,6 +290,8 @@ enum LINUX_MIB_TCPDUPLICATEDATAREHASH, /* TCPDuplicateDataRehash */ LINUX_MIB_TCPDSACKRECVSEGS, /* TCPDSACKRecvSegs */ LINUX_MIB_TCPDSACKIGNOREDDUBIOUS, /* TCPDSACKIgnoredDubious */ + LINUX_MIB_TCPMIGRATEREQSUCCESS, /* TCPMigrateReqSuccess */ + LINUX_MIB_TCPMIGRATEREQFAILURE, /* TCPMigrateReqFailure */ __LINUX_MIB_MAX }; diff --git a/net/core/sock_reuseport.c b/net/core/sock_reuseport.c index de5ee3ae86d5..3f00a28fe762 100644 --- a/net/core/sock_reuseport.c +++ b/net/core/sock_reuseport.c @@ -6,6 +6,7 @@ * selecting the socket index from the array of available sockets. */ +#include #include #include #include @@ -536,7 +537,7 @@ struct sock *reuseport_migrate_sock(struct sock *sk, socks = READ_ONCE(reuse->num_socks); if (unlikely(!socks)) - goto out; + goto failure; /* paired with smp_wmb() in __reuseport_add_sock() */ smp_rmb(); @@ -546,13 +547,13 @@ struct sock *reuseport_migrate_sock(struct sock *sk, if (!prog || prog->expected_attach_type != BPF_SK_REUSEPORT_SELECT_OR_MIGRATE) { if (sock_net(sk)->ipv4.sysctl_tcp_migrate_req) goto select_by_hash; - goto out; + goto failure; } if (!skb) { skb = alloc_skb(0, GFP_ATOMIC); if (!skb) - goto out; + goto failure; allocated = true; } @@ -565,12 +566,18 @@ select_by_hash: if (!nsk) nsk = reuseport_select_sock_by_hash(reuse, hash, socks); - if (IS_ERR_OR_NULL(nsk) || unlikely(!refcount_inc_not_zero(&nsk->sk_refcnt))) + if (IS_ERR_OR_NULL(nsk) || unlikely(!refcount_inc_not_zero(&nsk->sk_refcnt))) { nsk = NULL; + goto failure; + } out: rcu_read_unlock(); return nsk; + +failure: + __NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPMIGRATEREQFAILURE); + goto out; } EXPORT_SYMBOL(reuseport_migrate_sock); diff --git a/net/ipv4/inet_connection_sock.c b/net/ipv4/inet_connection_sock.c index 0eea878edc30..754013fa393b 100644 --- a/net/ipv4/inet_connection_sock.c +++ b/net/ipv4/inet_connection_sock.c @@ -703,6 +703,8 @@ static struct request_sock *inet_reqsk_clone(struct request_sock *req, nreq = kmem_cache_alloc(req->rsk_ops->slab, GFP_ATOMIC | __GFP_NOWARN); if (!nreq) { + __NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPMIGRATEREQFAILURE); + /* paired with refcount_inc_not_zero() in reuseport_migrate_sock() */ sock_put(sk); return NULL; @@ -876,9 +878,10 @@ static void reqsk_timer_handler(struct timer_list *t) if (!inet_ehash_insert(req_to_sk(nreq), req_to_sk(oreq), NULL)) { /* delete timer */ inet_csk_reqsk_queue_drop(sk_listener, nreq); - goto drop; + goto no_ownership; } + __NET_INC_STATS(net, LINUX_MIB_TCPMIGRATEREQSUCCESS); reqsk_migrate_reset(oreq); reqsk_queue_removed(&inet_csk(oreq->rsk_listener)->icsk_accept_queue, oreq); reqsk_put(oreq); @@ -887,17 +890,19 @@ static void reqsk_timer_handler(struct timer_list *t) return; } -drop: /* Even if we can clone the req, we may need not retransmit any more * SYN+ACKs (nreq->num_timeout > max_syn_ack_retries, etc), or another * CPU may win the "own_req" race so that inet_ehash_insert() fails. */ if (nreq) { + __NET_INC_STATS(net, LINUX_MIB_TCPMIGRATEREQFAILURE); +no_ownership: reqsk_migrate_reset(nreq); reqsk_queue_removed(queue, nreq); __reqsk_free(nreq); } +drop: inet_csk_reqsk_queue_drop_and_put(oreq->rsk_listener, oreq); } @@ -1135,11 +1140,13 @@ struct sock *inet_csk_complete_hashdance(struct sock *sk, struct sock *child, refcount_set(&nreq->rsk_refcnt, 1); if (inet_csk_reqsk_queue_add(sk, nreq, child)) { + __NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPMIGRATEREQSUCCESS); reqsk_migrate_reset(req); reqsk_put(req); return child; } + __NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPMIGRATEREQFAILURE); reqsk_migrate_reset(nreq); __reqsk_free(nreq); } else if (inet_csk_reqsk_queue_add(sk, req, child)) { @@ -1188,8 +1195,12 @@ void inet_csk_listen_stop(struct sock *sk) refcount_set(&nreq->rsk_refcnt, 1); if (inet_csk_reqsk_queue_add(nsk, nreq, child)) { + __NET_INC_STATS(sock_net(nsk), + LINUX_MIB_TCPMIGRATEREQSUCCESS); reqsk_migrate_reset(req); } else { + __NET_INC_STATS(sock_net(nsk), + LINUX_MIB_TCPMIGRATEREQFAILURE); reqsk_migrate_reset(nreq); __reqsk_free(nreq); } diff --git a/net/ipv4/proc.c b/net/ipv4/proc.c index 6d46297a99f8..b0d3a09dc84e 100644 --- a/net/ipv4/proc.c +++ b/net/ipv4/proc.c @@ -295,6 +295,8 @@ static const struct snmp_mib snmp4_net_list[] = { SNMP_MIB_ITEM("TcpDuplicateDataRehash", LINUX_MIB_TCPDUPLICATEDATAREHASH), SNMP_MIB_ITEM("TCPDSACKRecvSegs", LINUX_MIB_TCPDSACKRECVSEGS), SNMP_MIB_ITEM("TCPDSACKIgnoredDubious", LINUX_MIB_TCPDSACKIGNOREDDUBIOUS), + SNMP_MIB_ITEM("TCPMigrateReqSuccess", LINUX_MIB_TCPMIGRATEREQSUCCESS), + SNMP_MIB_ITEM("TCPMigrateReqFailure", LINUX_MIB_TCPMIGRATEREQFAILURE), SNMP_MIB_SENTINEL }; diff --git a/net/ipv4/tcp_minisocks.c b/net/ipv4/tcp_minisocks.c index f258a4c0da71..0a4f3f16140a 100644 --- a/net/ipv4/tcp_minisocks.c +++ b/net/ipv4/tcp_minisocks.c @@ -786,6 +786,9 @@ struct sock *tcp_check_req(struct sock *sk, struct sk_buff *skb, return inet_csk_complete_hashdance(sk, child, req, own_req); listen_overflow: + if (sk != req->rsk_listener) + __NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPMIGRATEREQFAILURE); + if (!sock_net(sk)->ipv4.sysctl_tcp_abort_on_overflow) { inet_rsk(req)->acked = 1; return NULL; -- cgit v1.2.3 From cb082bfab59a224a49ae803fed52cd03e8d6b5e0 Mon Sep 17 00:00:00 2001 From: Jing Zhang Date: Fri, 18 Jun 2021 22:27:04 +0000 Subject: KVM: stats: Add fd-based API to read binary stats data This commit defines the API for userspace and prepare the common functionalities to support per VM/VCPU binary stats data readings. The KVM stats now is only accessible by debugfs, which has some shortcomings this change series are supposed to fix: 1. The current debugfs stats solution in KVM could be disabled when kernel Lockdown mode is enabled, which is a potential rick for production. 2. The current debugfs stats solution in KVM is organized as "one stats per file", it is good for debugging, but not efficient for production. 3. The stats read/clear in current debugfs solution in KVM are protected by the global kvm_lock. Besides that, there are some other benefits with this change: 1. All KVM VM/VCPU stats can be read out in a bulk by one copy to userspace. 2. A schema is used to describe KVM statistics. From userspace's perspective, the KVM statistics are self-describing. 3. With the fd-based solution, a separate telemetry would be able to read KVM stats in a less privileged environment. 4. After the initial setup by reading in stats descriptors, a telemetry only needs to read the stats data itself, no more parsing or setup is needed. Reviewed-by: David Matlack Reviewed-by: Ricardo Koller Reviewed-by: Krish Sadhukhan Reviewed-by: Fuad Tabba Tested-by: Fuad Tabba #arm64 Signed-off-by: Jing Zhang Message-Id: <20210618222709.1858088-3-jingzhangos@google.com> Signed-off-by: Paolo Bonzini --- arch/arm64/kvm/Makefile | 2 +- arch/mips/kvm/Makefile | 2 +- arch/powerpc/kvm/Makefile | 2 +- arch/s390/kvm/Makefile | 3 +- arch/x86/kvm/Makefile | 2 +- include/linux/kvm_host.h | 82 +++++++++++++++++++++++++- include/linux/kvm_types.h | 2 + include/uapi/linux/kvm.h | 73 +++++++++++++++++++++++ virt/kvm/binary_stats.c | 146 ++++++++++++++++++++++++++++++++++++++++++++++ 9 files changed, 307 insertions(+), 7 deletions(-) create mode 100644 virt/kvm/binary_stats.c (limited to 'include/uapi') diff --git a/arch/arm64/kvm/Makefile b/arch/arm64/kvm/Makefile index 589921392cb1..989bb5dad2c8 100644 --- a/arch/arm64/kvm/Makefile +++ b/arch/arm64/kvm/Makefile @@ -11,7 +11,7 @@ obj-$(CONFIG_KVM) += kvm.o obj-$(CONFIG_KVM) += hyp/ kvm-y := $(KVM)/kvm_main.o $(KVM)/coalesced_mmio.o $(KVM)/eventfd.o \ - $(KVM)/vfio.o $(KVM)/irqchip.o \ + $(KVM)/vfio.o $(KVM)/irqchip.o $(KVM)/binary_stats.o \ arm.o mmu.o mmio.o psci.o perf.o hypercalls.o pvtime.o \ inject_fault.o va_layout.o handle_exit.o \ guest.o debug.o reset.o sys_regs.o \ diff --git a/arch/mips/kvm/Makefile b/arch/mips/kvm/Makefile index 30cc060857c7..c67250a956b8 100644 --- a/arch/mips/kvm/Makefile +++ b/arch/mips/kvm/Makefile @@ -2,7 +2,7 @@ # Makefile for KVM support for MIPS # -common-objs-y = $(addprefix ../../../virt/kvm/, kvm_main.o coalesced_mmio.o eventfd.o) +common-objs-y = $(addprefix ../../../virt/kvm/, kvm_main.o coalesced_mmio.o eventfd.o binary_stats.o) EXTRA_CFLAGS += -Ivirt/kvm -Iarch/mips/kvm diff --git a/arch/powerpc/kvm/Makefile b/arch/powerpc/kvm/Makefile index ab241317481c..583c14ef596e 100644 --- a/arch/powerpc/kvm/Makefile +++ b/arch/powerpc/kvm/Makefile @@ -6,7 +6,7 @@ ccflags-y := -Ivirt/kvm -Iarch/powerpc/kvm KVM := ../../../virt/kvm -common-objs-y = $(KVM)/kvm_main.o $(KVM)/eventfd.o +common-objs-y = $(KVM)/kvm_main.o $(KVM)/eventfd.o $(KVM)/binary_stats.o common-objs-$(CONFIG_KVM_VFIO) += $(KVM)/vfio.o common-objs-$(CONFIG_KVM_MMIO) += $(KVM)/coalesced_mmio.o diff --git a/arch/s390/kvm/Makefile b/arch/s390/kvm/Makefile index 12decca22e7c..b3aaadc60ead 100644 --- a/arch/s390/kvm/Makefile +++ b/arch/s390/kvm/Makefile @@ -4,7 +4,8 @@ # Copyright IBM Corp. 2008 KVM := ../../../virt/kvm -common-objs = $(KVM)/kvm_main.o $(KVM)/eventfd.o $(KVM)/async_pf.o $(KVM)/irqchip.o $(KVM)/vfio.o +common-objs = $(KVM)/kvm_main.o $(KVM)/eventfd.o $(KVM)/async_pf.o \ + $(KVM)/irqchip.o $(KVM)/vfio.o $(KVM)/binary_stats.o ccflags-y := -Ivirt/kvm -Iarch/s390/kvm diff --git a/arch/x86/kvm/Makefile b/arch/x86/kvm/Makefile index 83331376b779..75dfd27b6e8a 100644 --- a/arch/x86/kvm/Makefile +++ b/arch/x86/kvm/Makefile @@ -11,7 +11,7 @@ KVM := ../../../virt/kvm kvm-y += $(KVM)/kvm_main.o $(KVM)/coalesced_mmio.o \ $(KVM)/eventfd.o $(KVM)/irqchip.o $(KVM)/vfio.o \ - $(KVM)/dirty_ring.o + $(KVM)/dirty_ring.o $(KVM)/binary_stats.o kvm-$(CONFIG_KVM_ASYNC_PF) += $(KVM)/async_pf.o kvm-y += x86.o emulate.o i8259.o irq.o lapic.o \ diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h index 37cbb56ccd09..9ee7f350473b 100644 --- a/include/linux/kvm_host.h +++ b/include/linux/kvm_host.h @@ -1272,16 +1272,94 @@ struct kvm_stats_debugfs_item { int mode; }; +struct _kvm_stats_desc { + struct kvm_stats_desc desc; + char name[KVM_STATS_NAME_SIZE]; +}; + #define KVM_DBGFS_GET_MODE(dbgfs_item) \ ((dbgfs_item)->mode ? (dbgfs_item)->mode : 0644) -#define VM_STAT(n, x, ...) \ +#define VM_STAT(n, x, ...) \ { n, offsetof(struct kvm, stat.x), KVM_STAT_VM, ## __VA_ARGS__ } -#define VCPU_STAT(n, x, ...) \ +#define VCPU_STAT(n, x, ...) \ { n, offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU, ## __VA_ARGS__ } +#define STATS_DESC_COMMON(type, unit, base, exp) \ + .flags = type | unit | base | \ + BUILD_BUG_ON_ZERO(type & ~KVM_STATS_TYPE_MASK) | \ + BUILD_BUG_ON_ZERO(unit & ~KVM_STATS_UNIT_MASK) | \ + BUILD_BUG_ON_ZERO(base & ~KVM_STATS_BASE_MASK), \ + .exponent = exp, \ + .size = 1 + +#define VM_GENERIC_STATS_DESC(stat, type, unit, base, exp) \ + { \ + { \ + STATS_DESC_COMMON(type, unit, base, exp), \ + .offset = offsetof(struct kvm_vm_stat, generic.stat) \ + }, \ + .name = #stat, \ + } +#define VCPU_GENERIC_STATS_DESC(stat, type, unit, base, exp) \ + { \ + { \ + STATS_DESC_COMMON(type, unit, base, exp), \ + .offset = offsetof(struct kvm_vcpu_stat, generic.stat) \ + }, \ + .name = #stat, \ + } +#define VM_STATS_DESC(stat, type, unit, base, exp) \ + { \ + { \ + STATS_DESC_COMMON(type, unit, base, exp), \ + .offset = offsetof(struct kvm_vm_stat, stat) \ + }, \ + .name = #stat, \ + } +#define VCPU_STATS_DESC(stat, type, unit, base, exp) \ + { \ + { \ + STATS_DESC_COMMON(type, unit, base, exp), \ + .offset = offsetof(struct kvm_vcpu_stat, stat) \ + }, \ + .name = #stat, \ + } +/* SCOPE: VM, VM_GENERIC, VCPU, VCPU_GENERIC */ +#define STATS_DESC(SCOPE, stat, type, unit, base, exp) \ + SCOPE##_STATS_DESC(stat, type, unit, base, exp) + +#define STATS_DESC_CUMULATIVE(SCOPE, name, unit, base, exponent) \ + STATS_DESC(SCOPE, name, KVM_STATS_TYPE_CUMULATIVE, unit, base, exponent) +#define STATS_DESC_INSTANT(SCOPE, name, unit, base, exponent) \ + STATS_DESC(SCOPE, name, KVM_STATS_TYPE_INSTANT, unit, base, exponent) +#define STATS_DESC_PEAK(SCOPE, name, unit, base, exponent) \ + STATS_DESC(SCOPE, name, KVM_STATS_TYPE_PEAK, unit, base, exponent) + +/* Cumulative counter, read/write */ +#define STATS_DESC_COUNTER(SCOPE, name) \ + STATS_DESC_CUMULATIVE(SCOPE, name, KVM_STATS_UNIT_NONE, \ + KVM_STATS_BASE_POW10, 0) +/* Instantaneous counter, read only */ +#define STATS_DESC_ICOUNTER(SCOPE, name) \ + STATS_DESC_INSTANT(SCOPE, name, KVM_STATS_UNIT_NONE, \ + KVM_STATS_BASE_POW10, 0) +/* Peak counter, read/write */ +#define STATS_DESC_PCOUNTER(SCOPE, name) \ + STATS_DESC_PEAK(SCOPE, name, KVM_STATS_UNIT_NONE, \ + KVM_STATS_BASE_POW10, 0) + +/* Cumulative time in nanosecond */ +#define STATS_DESC_TIME_NSEC(SCOPE, name) \ + STATS_DESC_CUMULATIVE(SCOPE, name, KVM_STATS_UNIT_SECONDS, \ + KVM_STATS_BASE_POW10, -9) + extern struct kvm_stats_debugfs_item debugfs_entries[]; extern struct dentry *kvm_debugfs_dir; +ssize_t kvm_stats_read(char *id, const struct kvm_stats_header *header, + const struct _kvm_stats_desc *desc, + void *stats, size_t size_stats, + char __user *user_buffer, size_t size, loff_t *offset); #if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER) static inline int mmu_notifier_retry(struct kvm *kvm, unsigned long mmu_seq) diff --git a/include/linux/kvm_types.h b/include/linux/kvm_types.h index 48db778291b7..ed6a985c5680 100644 --- a/include/linux/kvm_types.h +++ b/include/linux/kvm_types.h @@ -89,4 +89,6 @@ struct kvm_vcpu_stat_generic { u64 halt_poll_fail_ns; }; +#define KVM_STATS_NAME_SIZE 48 + #endif /* __KVM_TYPES_H__ */ diff --git a/include/uapi/linux/kvm.h b/include/uapi/linux/kvm.h index 330835f1005b..f1ba602260f6 100644 --- a/include/uapi/linux/kvm.h +++ b/include/uapi/linux/kvm.h @@ -1087,6 +1087,7 @@ struct kvm_ppc_resize_hpt { #define KVM_CAP_SREGS2 200 #define KVM_CAP_EXIT_HYPERCALL 201 #define KVM_CAP_PPC_RPT_INVALIDATE 202 +#define KVM_CAP_BINARY_STATS_FD 203 #ifdef KVM_CAP_IRQ_ROUTING @@ -1906,4 +1907,76 @@ struct kvm_dirty_gfn { #define KVM_BUS_LOCK_DETECTION_OFF (1 << 0) #define KVM_BUS_LOCK_DETECTION_EXIT (1 << 1) +/** + * struct kvm_stats_header - Header of per vm/vcpu binary statistics data. + * @flags: Some extra information for header, always 0 for now. + * @name_size: The size in bytes of the memory which contains statistics + * name string including trailing '\0'. The memory is allocated + * at the send of statistics descriptor. + * @num_desc: The number of statistics the vm or vcpu has. + * @id_offset: The offset of the vm/vcpu stats' id string in the file pointed + * by vm/vcpu stats fd. + * @desc_offset: The offset of the vm/vcpu stats' descriptor block in the file + * pointd by vm/vcpu stats fd. + * @data_offset: The offset of the vm/vcpu stats' data block in the file + * pointed by vm/vcpu stats fd. + * + * This is the header userspace needs to read from stats fd before any other + * readings. It is used by userspace to discover all the information about the + * vm/vcpu's binary statistics. + * Userspace reads this header from the start of the vm/vcpu's stats fd. + */ +struct kvm_stats_header { + __u32 flags; + __u32 name_size; + __u32 num_desc; + __u32 id_offset; + __u32 desc_offset; + __u32 data_offset; +}; + +#define KVM_STATS_TYPE_SHIFT 0 +#define KVM_STATS_TYPE_MASK (0xF << KVM_STATS_TYPE_SHIFT) +#define KVM_STATS_TYPE_CUMULATIVE (0x0 << KVM_STATS_TYPE_SHIFT) +#define KVM_STATS_TYPE_INSTANT (0x1 << KVM_STATS_TYPE_SHIFT) +#define KVM_STATS_TYPE_PEAK (0x2 << KVM_STATS_TYPE_SHIFT) +#define KVM_STATS_TYPE_MAX KVM_STATS_TYPE_PEAK + +#define KVM_STATS_UNIT_SHIFT 4 +#define KVM_STATS_UNIT_MASK (0xF << KVM_STATS_UNIT_SHIFT) +#define KVM_STATS_UNIT_NONE (0x0 << KVM_STATS_UNIT_SHIFT) +#define KVM_STATS_UNIT_BYTES (0x1 << KVM_STATS_UNIT_SHIFT) +#define KVM_STATS_UNIT_SECONDS (0x2 << KVM_STATS_UNIT_SHIFT) +#define KVM_STATS_UNIT_CYCLES (0x3 << KVM_STATS_UNIT_SHIFT) +#define KVM_STATS_UNIT_MAX KVM_STATS_UNIT_CYCLES + +#define KVM_STATS_BASE_SHIFT 8 +#define KVM_STATS_BASE_MASK (0xF << KVM_STATS_BASE_SHIFT) +#define KVM_STATS_BASE_POW10 (0x0 << KVM_STATS_BASE_SHIFT) +#define KVM_STATS_BASE_POW2 (0x1 << KVM_STATS_BASE_SHIFT) +#define KVM_STATS_BASE_MAX KVM_STATS_BASE_POW2 + +/** + * struct kvm_stats_desc - Descriptor of a KVM statistics. + * @flags: Annotations of the stats, like type, unit, etc. + * @exponent: Used together with @flags to determine the unit. + * @size: The number of data items for this stats. + * Every data item is of type __u64. + * @offset: The offset of the stats to the start of stat structure in + * struture kvm or kvm_vcpu. + * @unused: Unused field for future usage. Always 0 for now. + * @name: The name string for the stats. Its size is indicated by the + * &kvm_stats_header->name_size. + */ +struct kvm_stats_desc { + __u32 flags; + __s16 exponent; + __u16 size; + __u32 offset; + __u32 unused; + char name[]; +}; + +#define KVM_GET_STATS_FD _IO(KVMIO, 0xce) + #endif /* __LINUX_KVM_H */ diff --git a/virt/kvm/binary_stats.c b/virt/kvm/binary_stats.c new file mode 100644 index 000000000000..e609d428811a --- /dev/null +++ b/virt/kvm/binary_stats.c @@ -0,0 +1,146 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * KVM binary statistics interface implementation + * + * Copyright 2021 Google LLC + */ + +#include +#include +#include +#include + +/** + * kvm_stats_read() - Common function to read from the binary statistics + * file descriptor. + * + * @id: identification string of the stats + * @header: stats header for a vm or a vcpu + * @desc: start address of an array of stats descriptors for a vm or a vcpu + * @stats: start address of stats data block for a vm or a vcpu + * @size_stats: the size of stats data block pointed by @stats + * @user_buffer: start address of userspace buffer + * @size: requested read size from userspace + * @offset: the start position from which the content will be read for the + * corresponding vm or vcp file descriptor + * + * The file content of a vm/vcpu file descriptor is now defined as below: + * +-------------+ + * | Header | + * +-------------+ + * | id string | + * +-------------+ + * | Descriptors | + * +-------------+ + * | Stats Data | + * +-------------+ + * Although this function allows userspace to read any amount of data (as long + * as in the limit) from any position, the typical usage would follow below + * steps: + * 1. Read header from offset 0. Get the offset of descriptors and stats data + * and some other necessary information. This is a one-time work for the + * lifecycle of the corresponding vm/vcpu stats fd. + * 2. Read id string from its offset. This is a one-time work for the lifecycle + * of the corresponding vm/vcpu stats fd. + * 3. Read descriptors from its offset and discover all the stats by parsing + * descriptors. This is a one-time work for the lifecycle of the + * corresponding vm/vcpu stats fd. + * 4. Periodically read stats data from its offset using pread. + * + * Return: the number of bytes that has been successfully read + */ +ssize_t kvm_stats_read(char *id, const struct kvm_stats_header *header, + const struct _kvm_stats_desc *desc, + void *stats, size_t size_stats, + char __user *user_buffer, size_t size, loff_t *offset) +{ + ssize_t len; + ssize_t copylen; + ssize_t remain = size; + size_t size_desc; + size_t size_header; + void *src; + loff_t pos = *offset; + char __user *dest = user_buffer; + + size_header = sizeof(*header); + size_desc = header->num_desc * sizeof(*desc); + + len = KVM_STATS_NAME_SIZE + size_header + size_desc + size_stats - pos; + len = min(len, remain); + if (len <= 0) + return 0; + remain = len; + + /* + * Copy kvm stats header. + * The header is the first block of content userspace usually read out. + * The pos is 0 and the copylen and remain would be the size of header. + * The copy of the header would be skipped if offset is larger than the + * size of header. That usually happens when userspace reads stats + * descriptors and stats data. + */ + copylen = size_header - pos; + copylen = min(copylen, remain); + if (copylen > 0) { + src = (void *)header + pos; + if (copy_to_user(dest, src, copylen)) + return -EFAULT; + remain -= copylen; + pos += copylen; + dest += copylen; + } + + /* + * Copy kvm stats header id string. + * The id string is unique for every vm/vcpu, which is stored in kvm + * and kvm_vcpu structure. + * The id string is part of the stat header from the perspective of + * userspace, it is usually read out together with previous constant + * header part and could be skipped for later descriptors and stats + * data readings. + */ + copylen = header->id_offset + KVM_STATS_NAME_SIZE - pos; + copylen = min(copylen, remain); + if (copylen > 0) { + src = id + pos - header->id_offset; + if (copy_to_user(dest, src, copylen)) + return -EFAULT; + remain -= copylen; + pos += copylen; + dest += copylen; + } + + /* + * Copy kvm stats descriptors. + * The descriptors copy would be skipped in the typical case that + * userspace periodically read stats data, since the pos would be + * greater than the end address of descriptors + * (header->header.desc_offset + size_desc) causing copylen <= 0. + */ + copylen = header->desc_offset + size_desc - pos; + copylen = min(copylen, remain); + if (copylen > 0) { + src = (void *)desc + pos - header->desc_offset; + if (copy_to_user(dest, src, copylen)) + return -EFAULT; + remain -= copylen; + pos += copylen; + dest += copylen; + } + + /* Copy kvm stats values */ + copylen = header->data_offset + size_stats - pos; + copylen = min(copylen, remain); + if (copylen > 0) { + src = stats + pos - header->data_offset; + if (copy_to_user(dest, src, copylen)) + return -EFAULT; + remain -= copylen; + pos += copylen; + dest += copylen; + } + + *offset = pos; + return len; +} -- cgit v1.2.3 From e8b9eab99232c4e62ada9d7976c80fd5e8118289 Mon Sep 17 00:00:00 2001 From: Martynas Pumputis Date: Wed, 23 Jun 2021 15:56:45 +0200 Subject: net: retrieve netns cookie via getsocketopt It's getting more common to run nested container environments for testing cloud software. One of such examples is Kind [1] which runs a Kubernetes cluster in Docker containers on a single host. Each container acts as a Kubernetes node, and thus can run any Pod (aka container) inside the former. This approach simplifies testing a lot, as it eliminates complicated VM setups. Unfortunately, such a setup breaks some functionality when cgroupv2 BPF programs are used for load-balancing. The load-balancer BPF program needs to detect whether a request originates from the host netns or a container netns in order to allow some access, e.g. to a service via a loopback IP address. Typically, the programs detect this by comparing netns cookies with the one of the init ns via a call to bpf_get_netns_cookie(NULL). However, in nested environments the latter cannot be used given the Kubernetes node's netns is outside the init ns. To fix this, we need to pass the Kubernetes node netns cookie to the program in a different way: by extending getsockopt() with a SO_NETNS_COOKIE option, the orchestrator which runs in the Kubernetes node netns can retrieve the cookie and pass it to the program instead. Thus, this is following up on Eric's commit 3d368ab87cf6 ("net: initialize net->net_cookie at netns setup") to allow retrieval via SO_NETNS_COOKIE. This is also in line in how we retrieve socket cookie via SO_COOKIE. [1] https://kind.sigs.k8s.io/ Signed-off-by: Lorenz Bauer Signed-off-by: Martynas Pumputis Cc: Eric Dumazet Reviewed-by: Eric Dumazet Signed-off-by: David S. Miller --- arch/alpha/include/uapi/asm/socket.h | 2 ++ arch/mips/include/uapi/asm/socket.h | 2 ++ arch/parisc/include/uapi/asm/socket.h | 2 ++ arch/sparc/include/uapi/asm/socket.h | 2 ++ include/uapi/asm-generic/socket.h | 2 ++ net/core/sock.c | 7 +++++++ 6 files changed, 17 insertions(+) (limited to 'include/uapi') diff --git a/arch/alpha/include/uapi/asm/socket.h b/arch/alpha/include/uapi/asm/socket.h index 57420356ce4c..6b3daba60987 100644 --- a/arch/alpha/include/uapi/asm/socket.h +++ b/arch/alpha/include/uapi/asm/socket.h @@ -127,6 +127,8 @@ #define SO_PREFER_BUSY_POLL 69 #define SO_BUSY_POLL_BUDGET 70 +#define SO_NETNS_COOKIE 71 + #if !defined(__KERNEL__) #if __BITS_PER_LONG == 64 diff --git a/arch/mips/include/uapi/asm/socket.h b/arch/mips/include/uapi/asm/socket.h index 2d949969313b..cdf404a831b2 100644 --- a/arch/mips/include/uapi/asm/socket.h +++ b/arch/mips/include/uapi/asm/socket.h @@ -138,6 +138,8 @@ #define SO_PREFER_BUSY_POLL 69 #define SO_BUSY_POLL_BUDGET 70 +#define SO_NETNS_COOKIE 71 + #if !defined(__KERNEL__) #if __BITS_PER_LONG == 64 diff --git a/arch/parisc/include/uapi/asm/socket.h b/arch/parisc/include/uapi/asm/socket.h index f60904329bbc..5b5351cdcb33 100644 --- a/arch/parisc/include/uapi/asm/socket.h +++ b/arch/parisc/include/uapi/asm/socket.h @@ -119,6 +119,8 @@ #define SO_PREFER_BUSY_POLL 0x4043 #define SO_BUSY_POLL_BUDGET 0x4044 +#define SO_NETNS_COOKIE 0x4045 + #if !defined(__KERNEL__) #if __BITS_PER_LONG == 64 diff --git a/arch/sparc/include/uapi/asm/socket.h b/arch/sparc/include/uapi/asm/socket.h index 848a22fbac20..92675dc380fa 100644 --- a/arch/sparc/include/uapi/asm/socket.h +++ b/arch/sparc/include/uapi/asm/socket.h @@ -120,6 +120,8 @@ #define SO_PREFER_BUSY_POLL 0x0048 #define SO_BUSY_POLL_BUDGET 0x0049 +#define SO_NETNS_COOKIE 0x0050 + #if !defined(__KERNEL__) diff --git a/include/uapi/asm-generic/socket.h b/include/uapi/asm-generic/socket.h index 4dcd13d097a9..d588c244ec2f 100644 --- a/include/uapi/asm-generic/socket.h +++ b/include/uapi/asm-generic/socket.h @@ -122,6 +122,8 @@ #define SO_PREFER_BUSY_POLL 69 #define SO_BUSY_POLL_BUDGET 70 +#define SO_NETNS_COOKIE 71 + #if !defined(__KERNEL__) #if __BITS_PER_LONG == 64 || (defined(__x86_64__) && defined(__ILP32__)) diff --git a/net/core/sock.c b/net/core/sock.c index ddfa88082a2b..a2337b37eba6 100644 --- a/net/core/sock.c +++ b/net/core/sock.c @@ -1635,6 +1635,13 @@ int sock_getsockopt(struct socket *sock, int level, int optname, v.val = sk->sk_bound_dev_if; break; + case SO_NETNS_COOKIE: + lv = sizeof(u64); + if (len != lv) + return -EINVAL; + v.val64 = sock_net(sk)->net_cookie; + break; + default: /* We implement the SO_SNDLOWAT etc to not be settable * (1003.1g 7). -- cgit v1.2.3 From 19238e75bd8ed8ffe784bf5b37586e77b2093742 Mon Sep 17 00:00:00 2001 From: Aaron Lewis Date: Mon, 10 May 2021 07:48:33 -0700 Subject: kvm: x86: Allow userspace to handle emulation errors Add a fallback mechanism to the in-kernel instruction emulator that allows userspace the opportunity to process an instruction the emulator was unable to. When the in-kernel instruction emulator fails to process an instruction it will either inject a #UD into the guest or exit to userspace with exit reason KVM_INTERNAL_ERROR. This is because it does not know how to proceed in an appropriate manner. This feature lets userspace get involved to see if it can figure out a better path forward. Signed-off-by: Aaron Lewis Reviewed-by: David Edmondson Message-Id: <20210510144834.658457-2-aaronlewis@google.com> Reviewed-by: Jim Mattson Signed-off-by: Paolo Bonzini --- Documentation/virt/kvm/api.rst | 20 ++++++++++++++++++++ arch/x86/include/asm/kvm_host.h | 6 ++++++ arch/x86/kvm/x86.c | 40 ++++++++++++++++++++++++++++++++++++---- include/uapi/linux/kvm.h | 23 +++++++++++++++++++++++ 4 files changed, 85 insertions(+), 4 deletions(-) (limited to 'include/uapi') diff --git a/Documentation/virt/kvm/api.rst b/Documentation/virt/kvm/api.rst index 5d8db4922df6..3b6e3b1628b4 100644 --- a/Documentation/virt/kvm/api.rst +++ b/Documentation/virt/kvm/api.rst @@ -6546,6 +6546,7 @@ KVM_RUN_BUS_LOCK flag is used to distinguish between them. This capability can be used to check / enable 2nd DAWR feature provided by POWER10 processor. + 7.24 KVM_CAP_VM_COPY_ENC_CONTEXT_FROM ------------------------------------- @@ -6603,6 +6604,25 @@ present in the "ibm,hypertas-functions" device-tree property. This capability is enabled for hypervisors on platforms like POWER9 that support radix MMU. +7.27 KVM_CAP_EXIT_ON_EMULATION_FAILURE +-------------------------------------- + +:Architectures: x86 +:Parameters: args[0] whether the feature should be enabled or not + +When this capability is enabled, an emulation failure will result in an exit +to userspace with KVM_INTERNAL_ERROR (except when the emulator was invoked +to handle a VMware backdoor instruction). Furthermore, KVM will now provide up +to 15 instruction bytes for any exit to userspace resulting from an emulation +failure. When these exits to userspace occur use the emulation_failure struct +instead of the internal struct. They both have the same layout, but the +emulation_failure struct matches the content better. It also explicitly +defines the 'flags' field which is used to describe the fields in the struct +that are valid (ie: if KVM_INTERNAL_ERROR_EMULATION_FLAG_INSTRUCTION_BYTES is +set in the 'flags' field then both 'insn_size' and 'insn_bytes' have valid data +in them.) + + 8. Other capabilities. ====================== diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index 3cd496c8acb8..c9ec5c76c438 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -1114,6 +1114,12 @@ struct kvm_arch { bool exception_payload_enabled; bool bus_lock_detection_enabled; + /* + * If exit_on_emulation_error is set, and the in-kernel instruction + * emulator fails to emulate an instruction, allow userspace + * the opportunity to look at it. + */ + bool exit_on_emulation_error; /* Deflect RDMSR and WRMSR to user space when they trigger a #GP */ u32 user_space_msr_mask; diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index a7c7b2b28de7..17468d983fbd 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -4010,6 +4010,7 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) #endif case KVM_CAP_VM_COPY_ENC_CONTEXT_FROM: case KVM_CAP_SREGS2: + case KVM_CAP_EXIT_ON_EMULATION_FAILURE: r = 1; break; case KVM_CAP_EXIT_HYPERCALL: @@ -5649,6 +5650,13 @@ split_irqchip_unlock: kvm->arch.hypercall_exit_enabled = cap->args[0]; r = 0; break; + case KVM_CAP_EXIT_ON_EMULATION_FAILURE: + r = -EINVAL; + if (cap->args[0] & ~1) + break; + kvm->arch.exit_on_emulation_error = cap->args[0]; + r = 0; + break; default: r = -EINVAL; break; @@ -7444,8 +7452,33 @@ void kvm_inject_realmode_interrupt(struct kvm_vcpu *vcpu, int irq, int inc_eip) } EXPORT_SYMBOL_GPL(kvm_inject_realmode_interrupt); +static void prepare_emulation_failure_exit(struct kvm_vcpu *vcpu) +{ + struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt; + u32 insn_size = ctxt->fetch.end - ctxt->fetch.data; + struct kvm_run *run = vcpu->run; + + run->exit_reason = KVM_EXIT_INTERNAL_ERROR; + run->emulation_failure.suberror = KVM_INTERNAL_ERROR_EMULATION; + run->emulation_failure.ndata = 0; + run->emulation_failure.flags = 0; + + if (insn_size) { + run->emulation_failure.ndata = 3; + run->emulation_failure.flags |= + KVM_INTERNAL_ERROR_EMULATION_FLAG_INSTRUCTION_BYTES; + run->emulation_failure.insn_size = insn_size; + memset(run->emulation_failure.insn_bytes, 0x90, + sizeof(run->emulation_failure.insn_bytes)); + memcpy(run->emulation_failure.insn_bytes, + ctxt->fetch.data, insn_size); + } +} + static int handle_emulation_failure(struct kvm_vcpu *vcpu, int emulation_type) { + struct kvm *kvm = vcpu->kvm; + ++vcpu->stat.insn_emulation_fail; trace_kvm_emulate_insn_failed(vcpu); @@ -7454,10 +7487,9 @@ static int handle_emulation_failure(struct kvm_vcpu *vcpu, int emulation_type) return 1; } - if (emulation_type & EMULTYPE_SKIP) { - vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR; - vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION; - vcpu->run->internal.ndata = 0; + if (kvm->arch.exit_on_emulation_error || + (emulation_type & EMULTYPE_SKIP)) { + prepare_emulation_failure_exit(vcpu); return 0; } diff --git a/include/uapi/linux/kvm.h b/include/uapi/linux/kvm.h index f1ba602260f6..68c9e6d8bbda 100644 --- a/include/uapi/linux/kvm.h +++ b/include/uapi/linux/kvm.h @@ -280,6 +280,9 @@ struct kvm_xen_exit { /* Encounter unexpected vm-exit reason */ #define KVM_INTERNAL_ERROR_UNEXPECTED_EXIT_REASON 4 +/* Flags that describe what fields in emulation_failure hold valid data. */ +#define KVM_INTERNAL_ERROR_EMULATION_FLAG_INSTRUCTION_BYTES (1ULL << 0) + /* for KVM_RUN, returned by mmap(vcpu_fd, offset=0) */ struct kvm_run { /* in */ @@ -383,6 +386,25 @@ struct kvm_run { __u32 ndata; __u64 data[16]; } internal; + /* + * KVM_INTERNAL_ERROR_EMULATION + * + * "struct emulation_failure" is an overlay of "struct internal" + * that is used for the KVM_INTERNAL_ERROR_EMULATION sub-type of + * KVM_EXIT_INTERNAL_ERROR. Note, unlike other internal error + * sub-types, this struct is ABI! It also needs to be backwards + * compatible with "struct internal". Take special care that + * "ndata" is correct, that new fields are enumerated in "flags", + * and that each flag enumerates fields that are 64-bit aligned + * and sized (so that ndata+internal.data[] is valid/accurate). + */ + struct { + __u32 suberror; + __u32 ndata; + __u64 flags; + __u8 insn_size; + __u8 insn_bytes[15]; + } emulation_failure; /* KVM_EXIT_OSI */ struct { __u64 gprs[32]; @@ -1088,6 +1110,7 @@ struct kvm_ppc_resize_hpt { #define KVM_CAP_EXIT_HYPERCALL 201 #define KVM_CAP_PPC_RPT_INVALIDATE 202 #define KVM_CAP_BINARY_STATS_FD 203 +#define KVM_CAP_EXIT_ON_EMULATION_FAILURE 204 #ifdef KVM_CAP_IRQ_ROUTING -- cgit v1.2.3 From 808e9df477757955a9644ca323010339be0c40ee Mon Sep 17 00:00:00 2001 From: Gleb Fotengauer-Malinovskiy Date: Fri, 25 Jun 2021 20:36:55 +0300 Subject: userfaultfd: uapi: fix UFFDIO_CONTINUE ioctl request definition This ioctl request reads from uffdio_continue structure written by userspace which justifies _IOC_WRITE flag. It also writes back to that structure which justifies _IOC_READ flag. See NOTEs in include/uapi/asm-generic/ioctl.h for more information. Fixes: f619147104c8 ("userfaultfd: add UFFDIO_CONTINUE ioctl") Signed-off-by: Gleb Fotengauer-Malinovskiy Acked-by: Peter Xu Reviewed-by: Axel Rasmussen Reviewed-by: Dmitry V. Levin Signed-off-by: Linus Torvalds --- include/uapi/linux/userfaultfd.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'include/uapi') diff --git a/include/uapi/linux/userfaultfd.h b/include/uapi/linux/userfaultfd.h index bafbeb1a2624..650480f41f1d 100644 --- a/include/uapi/linux/userfaultfd.h +++ b/include/uapi/linux/userfaultfd.h @@ -80,8 +80,8 @@ struct uffdio_zeropage) #define UFFDIO_WRITEPROTECT _IOWR(UFFDIO, _UFFDIO_WRITEPROTECT, \ struct uffdio_writeprotect) -#define UFFDIO_CONTINUE _IOR(UFFDIO, _UFFDIO_CONTINUE, \ - struct uffdio_continue) +#define UFFDIO_CONTINUE _IOWR(UFFDIO, _UFFDIO_CONTINUE, \ + struct uffdio_continue) /* read() structure */ struct uffd_msg { -- cgit v1.2.3 From 0ae71c7720e3ae3aabd2e8a072d27f7bd173d25c Mon Sep 17 00:00:00 2001 From: Rodrigo Campos Date: Mon, 17 May 2021 12:39:07 -0700 Subject: seccomp: Support atomic "addfd + send reply" Alban Crequy reported a race condition userspace faces when we want to add some fds and make the syscall return them[1] using seccomp notify. The problem is that currently two different ioctl() calls are needed by the process handling the syscalls (agent) for another userspace process (target): SECCOMP_IOCTL_NOTIF_ADDFD to allocate the fd and SECCOMP_IOCTL_NOTIF_SEND to return that value. Therefore, it is possible for the agent to do the first ioctl to add a file descriptor but the target is interrupted (EINTR) before the agent does the second ioctl() call. This patch adds a flag to the ADDFD ioctl() so it adds the fd and returns that value atomically to the target program, as suggested by Kees Cook[2]. This is done by simply allowing seccomp_do_user_notification() to add the fd and return it in this case. Therefore, in this case the target wakes up from the wait in seccomp_do_user_notification() either to interrupt the syscall or to add the fd and return it. This "allocate an fd and return" functionality is useful for syscalls that return a file descriptor only, like connect(2). Other syscalls that return a file descriptor but not as return value (or return more than one fd), like socketpair(), pipe(), recvmsg with SCM_RIGHTs, will not work with this flag. This effectively combines SECCOMP_IOCTL_NOTIF_ADDFD and SECCOMP_IOCTL_NOTIF_SEND into an atomic opteration. The notification's return value, nor error can be set by the user. Upon successful invocation of the SECCOMP_IOCTL_NOTIF_ADDFD ioctl with the SECCOMP_ADDFD_FLAG_SEND flag, the notifying process's errno will be 0, and the return value will be the file descriptor number that was installed. [1]: https://lore.kernel.org/lkml/CADZs7q4sw71iNHmV8EOOXhUKJMORPzF7thraxZYddTZsxta-KQ@mail.gmail.com/ [2]: https://lore.kernel.org/lkml/202012011322.26DCBC64F2@keescook/ Signed-off-by: Rodrigo Campos Signed-off-by: Sargun Dhillon Acked-by: Tycho Andersen Acked-by: Christian Brauner Signed-off-by: Kees Cook Link: https://lore.kernel.org/r/20210517193908.3113-4-sargun@sargun.me --- Documentation/userspace-api/seccomp_filter.rst | 12 ++++++ include/uapi/linux/seccomp.h | 1 + kernel/seccomp.c | 51 +++++++++++++++++++++++--- 3 files changed, 58 insertions(+), 6 deletions(-) (limited to 'include/uapi') diff --git a/Documentation/userspace-api/seccomp_filter.rst b/Documentation/userspace-api/seccomp_filter.rst index 6efb41cc8072..d61219889e49 100644 --- a/Documentation/userspace-api/seccomp_filter.rst +++ b/Documentation/userspace-api/seccomp_filter.rst @@ -259,6 +259,18 @@ and ``ioctl(SECCOMP_IOCTL_NOTIF_SEND)`` a response, indicating what should be returned to userspace. The ``id`` member of ``struct seccomp_notif_resp`` should be the same ``id`` as in ``struct seccomp_notif``. +Userspace can also add file descriptors to the notifying process via +``ioctl(SECCOMP_IOCTL_NOTIF_ADDFD)``. The ``id`` member of +``struct seccomp_notif_addfd`` should be the same ``id`` as in +``struct seccomp_notif``. The ``newfd_flags`` flag may be used to set flags +like O_EXEC on the file descriptor in the notifying process. If the supervisor +wants to inject the file descriptor with a specific number, the +``SECCOMP_ADDFD_FLAG_SETFD`` flag can be used, and set the ``newfd`` member to +the specific number to use. If that file descriptor is already open in the +notifying process it will be replaced. The supervisor can also add an FD, and +respond atomically by using the ``SECCOMP_ADDFD_FLAG_SEND`` flag and the return +value will be the injected file descriptor number. + It is worth noting that ``struct seccomp_data`` contains the values of register arguments to the syscall, but does not contain pointers to memory. The task's memory is accessible to suitably privileged traces via ``ptrace()`` or diff --git a/include/uapi/linux/seccomp.h b/include/uapi/linux/seccomp.h index 6ba18b82a02e..78074254ab98 100644 --- a/include/uapi/linux/seccomp.h +++ b/include/uapi/linux/seccomp.h @@ -115,6 +115,7 @@ struct seccomp_notif_resp { /* valid flags for seccomp_notif_addfd */ #define SECCOMP_ADDFD_FLAG_SETFD (1UL << 0) /* Specify remote fd */ +#define SECCOMP_ADDFD_FLAG_SEND (1UL << 1) /* Addfd and return it, atomically */ /** * struct seccomp_notif_addfd diff --git a/kernel/seccomp.c b/kernel/seccomp.c index 9f58049ac16d..057e17f3215d 100644 --- a/kernel/seccomp.c +++ b/kernel/seccomp.c @@ -107,6 +107,7 @@ struct seccomp_knotif { * installing process should allocate the fd as normal. * @flags: The flags for the new file descriptor. At the moment, only O_CLOEXEC * is allowed. + * @ioctl_flags: The flags used for the seccomp_addfd ioctl. * @ret: The return value of the installing process. It is set to the fd num * upon success (>= 0). * @completion: Indicates that the installing process has completed fd @@ -118,6 +119,7 @@ struct seccomp_kaddfd { struct file *file; int fd; unsigned int flags; + __u32 ioctl_flags; union { bool setfd; @@ -1065,18 +1067,37 @@ static u64 seccomp_next_notify_id(struct seccomp_filter *filter) return filter->notif->next_id++; } -static void seccomp_handle_addfd(struct seccomp_kaddfd *addfd) +static void seccomp_handle_addfd(struct seccomp_kaddfd *addfd, struct seccomp_knotif *n) { + int fd; + /* * Remove the notification, and reset the list pointers, indicating * that it has been handled. */ list_del_init(&addfd->list); if (!addfd->setfd) - addfd->ret = receive_fd(addfd->file, addfd->flags); + fd = receive_fd(addfd->file, addfd->flags); else - addfd->ret = receive_fd_replace(addfd->fd, addfd->file, - addfd->flags); + fd = receive_fd_replace(addfd->fd, addfd->file, addfd->flags); + addfd->ret = fd; + + if (addfd->ioctl_flags & SECCOMP_ADDFD_FLAG_SEND) { + /* If we fail reset and return an error to the notifier */ + if (fd < 0) { + n->state = SECCOMP_NOTIFY_SENT; + } else { + /* Return the FD we just added */ + n->flags = 0; + n->error = 0; + n->val = fd; + } + } + + /* + * Mark the notification as completed. From this point, addfd mem + * might be invalidated and we can't safely read it anymore. + */ complete(&addfd->completion); } @@ -1120,7 +1141,7 @@ static int seccomp_do_user_notification(int this_syscall, struct seccomp_kaddfd, list); /* Check if we were woken up by a addfd message */ if (addfd) - seccomp_handle_addfd(addfd); + seccomp_handle_addfd(addfd, &n); } while (n.state != SECCOMP_NOTIFY_REPLIED); @@ -1581,7 +1602,7 @@ static long seccomp_notify_addfd(struct seccomp_filter *filter, if (addfd.newfd_flags & ~O_CLOEXEC) return -EINVAL; - if (addfd.flags & ~SECCOMP_ADDFD_FLAG_SETFD) + if (addfd.flags & ~(SECCOMP_ADDFD_FLAG_SETFD | SECCOMP_ADDFD_FLAG_SEND)) return -EINVAL; if (addfd.newfd && !(addfd.flags & SECCOMP_ADDFD_FLAG_SETFD)) @@ -1591,6 +1612,7 @@ static long seccomp_notify_addfd(struct seccomp_filter *filter, if (!kaddfd.file) return -EBADF; + kaddfd.ioctl_flags = addfd.flags; kaddfd.flags = addfd.newfd_flags; kaddfd.setfd = addfd.flags & SECCOMP_ADDFD_FLAG_SETFD; kaddfd.fd = addfd.newfd; @@ -1616,6 +1638,23 @@ static long seccomp_notify_addfd(struct seccomp_filter *filter, goto out_unlock; } + if (addfd.flags & SECCOMP_ADDFD_FLAG_SEND) { + /* + * Disallow queuing an atomic addfd + send reply while there are + * some addfd requests still to process. + * + * There is no clear reason to support it and allows us to keep + * the loop on the other side straight-forward. + */ + if (!list_empty(&knotif->addfd)) { + ret = -EBUSY; + goto out_unlock; + } + + /* Allow exactly only one reply */ + knotif->state = SECCOMP_NOTIFY_REPLIED; + } + list_add(&kaddfd.list, &knotif->addfd); complete(&knotif->ready); mutex_unlock(&filter->notify_lock); -- cgit v1.2.3 From 9ba6a1c06279ce499fcf755d8134d679a1f3b4ed Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Thu, 24 Jun 2021 15:09:59 +0100 Subject: io_uring: simplify struct io_uring_sqe layout Flatten struct io_uring_sqe, the last union is exactly 64B, so move them out of union { struct { ... }}, and decrease __pad2 size. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/2e21ef7aed136293d654450bc3088973a8adc730.1624543113.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- include/uapi/linux/io_uring.h | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-) (limited to 'include/uapi') diff --git a/include/uapi/linux/io_uring.h b/include/uapi/linux/io_uring.h index f1f9ac114b51..79126d5cd289 100644 --- a/include/uapi/linux/io_uring.h +++ b/include/uapi/linux/io_uring.h @@ -46,21 +46,17 @@ struct io_uring_sqe { __u32 unlink_flags; }; __u64 user_data; /* data to be passed back at completion time */ + /* pack this to avoid bogus arm OABI complaints */ union { - struct { - /* pack this to avoid bogus arm OABI complaints */ - union { - /* index into fixed buffers, if used */ - __u16 buf_index; - /* for grouped buffer selection */ - __u16 buf_group; - } __attribute__((packed)); - /* personality to use, if used */ - __u16 personality; - __s32 splice_fd_in; - }; - __u64 __pad2[3]; - }; + /* index into fixed buffers, if used */ + __u16 buf_index; + /* for grouped buffer selection */ + __u16 buf_group; + } __attribute__((packed)); + /* personality to use, if used */ + __u16 personality; + __s32 splice_fd_in; + __u64 __pad2[2]; }; enum { -- cgit v1.2.3 From 6497ef8df568afbf5f3e38825a4590ff41611a54 Mon Sep 17 00:00:00 2001 From: Prasanna Kumar Kalever Date: Thu, 29 Apr 2021 15:58:28 +0530 Subject: nbd: provide a way for userspace processes to identify device backends Problem: On reconfigure of device, there is no way to defend if the backend storage is matching with the initial backend storage. Say, if an initial connect request for backend "pool1/image1" got mapped to /dev/nbd0 and the userspace process is terminated. A next reconfigure request within NBD_ATTR_DEAD_CONN_TIMEOUT is allowed to use /dev/nbd0 for a different backend "pool1/image2" For example, an operation like below could be dangerous: $ sudo rbd-nbd map --try-netlink rbd-pool/ext4-image /dev/nbd0 $ sudo blkid /dev/nbd0 /dev/nbd0: UUID="bfc444b4-64b1-418f-8b36-6e0d170cfc04" TYPE="ext4" $ sudo pkill -9 rbd-nbd $ sudo rbd-nbd attach --try-netlink --device /dev/nbd0 rbd-pool/xfs-image /dev/nbd0 $ sudo blkid /dev/nbd0 /dev/nbd0: UUID="d29bf343-6570-4069-a9ea-2fa156ced908" TYPE="xfs" Solution: Provide a way for userspace processes to keep some metadata to identify between the device and the backend, so that when a reconfigure request is made, we can compare and avoid such dangerous operations. With this solution, as part of the initial connect request, backend path can be stored in the sysfs per device config, so that on a reconfigure request it's easy to check if the backend path matches with the initial connect backend path. Please note, ioctl interface to nbd will not have these changes, as there won't be any reconfigure. Signed-off-by: Prasanna Kumar Kalever Reviewed-by: Xiubo Li Reviewed-by: Ming Lei Link: https://lore.kernel.org/r/20210429102828.31248-1-prasanna.kalever@redhat.com Signed-off-by: Jens Axboe --- drivers/block/nbd.c | 60 +++++++++++++++++++++++++++++++++++++++- include/uapi/linux/nbd-netlink.h | 1 + 2 files changed, 60 insertions(+), 1 deletion(-) (limited to 'include/uapi') diff --git a/drivers/block/nbd.c b/drivers/block/nbd.c index 614d82e7fae4..b7d663736d35 100644 --- a/drivers/block/nbd.c +++ b/drivers/block/nbd.c @@ -79,6 +79,7 @@ struct link_dead_args { #define NBD_RT_HAS_CONFIG_REF 4 #define NBD_RT_BOUND 5 #define NBD_RT_DISCONNECT_ON_CLOSE 6 +#define NBD_RT_HAS_BACKEND_FILE 7 #define NBD_DESTROY_ON_DISCONNECT 0 #define NBD_DISCONNECT_REQUESTED 1 @@ -119,6 +120,8 @@ struct nbd_device { struct completion *destroy_complete; unsigned long flags; + + char *backend; }; #define NBD_CMD_REQUEUED 1 @@ -216,6 +219,20 @@ static const struct device_attribute pid_attr = { .show = pid_show, }; +static ssize_t backend_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct gendisk *disk = dev_to_disk(dev); + struct nbd_device *nbd = (struct nbd_device *)disk->private_data; + + return sprintf(buf, "%s\n", nbd->backend ?: ""); +} + +static const struct device_attribute backend_attr = { + .attr = { .name = "backend", .mode = 0444}, + .show = backend_show, +}; + static void nbd_dev_remove(struct nbd_device *nbd) { struct gendisk *disk = nbd->disk; @@ -1211,6 +1228,12 @@ static void nbd_config_put(struct nbd_device *nbd) &config->runtime_flags)) device_remove_file(disk_to_dev(nbd->disk), &pid_attr); nbd->task_recv = NULL; + if (test_and_clear_bit(NBD_RT_HAS_BACKEND_FILE, + &config->runtime_flags)) { + device_remove_file(disk_to_dev(nbd->disk), &backend_attr); + kfree(nbd->backend); + nbd->backend = NULL; + } nbd_clear_sock(nbd); if (config->num_connections) { int i; @@ -1270,7 +1293,7 @@ static int nbd_start_device(struct nbd_device *nbd) error = device_create_file(disk_to_dev(nbd->disk), &pid_attr); if (error) { - dev_err(disk_to_dev(nbd->disk), "device_create_file failed!\n"); + dev_err(disk_to_dev(nbd->disk), "device_create_file failed for pid!\n"); return error; } set_bit(NBD_RT_HAS_PID_FILE, &config->runtime_flags); @@ -1657,6 +1680,7 @@ static int nbd_dev_add(int index) BLK_MQ_F_BLOCKING; nbd->tag_set.driver_data = nbd; nbd->destroy_complete = NULL; + nbd->backend = NULL; err = blk_mq_alloc_tag_set(&nbd->tag_set); if (err) @@ -1743,6 +1767,7 @@ static const struct nla_policy nbd_attr_policy[NBD_ATTR_MAX + 1] = { [NBD_ATTR_SOCKETS] = { .type = NLA_NESTED}, [NBD_ATTR_DEAD_CONN_TIMEOUT] = { .type = NLA_U64 }, [NBD_ATTR_DEVICE_LIST] = { .type = NLA_NESTED}, + [NBD_ATTR_BACKEND_IDENTIFIER] = { .type = NLA_STRING}, }; static const struct nla_policy nbd_sock_policy[NBD_SOCK_MAX + 1] = { @@ -1945,6 +1970,23 @@ again: } } ret = nbd_start_device(nbd); + if (ret) + goto out; + if (info->attrs[NBD_ATTR_BACKEND_IDENTIFIER]) { + nbd->backend = nla_strdup(info->attrs[NBD_ATTR_BACKEND_IDENTIFIER], + GFP_KERNEL); + if (!nbd->backend) { + ret = -ENOMEM; + goto out; + } + } + ret = device_create_file(disk_to_dev(nbd->disk), &backend_attr); + if (ret) { + dev_err(disk_to_dev(nbd->disk), + "device_create_file failed for backend!\n"); + goto out; + } + set_bit(NBD_RT_HAS_BACKEND_FILE, &config->runtime_flags); out: mutex_unlock(&nbd->config_lock); if (!ret) { @@ -2037,6 +2079,22 @@ static int nbd_genl_reconfigure(struct sk_buff *skb, struct genl_info *info) index); return -EINVAL; } + if (nbd->backend) { + if (info->attrs[NBD_ATTR_BACKEND_IDENTIFIER]) { + if (nla_strcmp(info->attrs[NBD_ATTR_BACKEND_IDENTIFIER], + nbd->backend)) { + mutex_unlock(&nbd_index_mutex); + dev_err(nbd_to_dev(nbd), + "backend image doesn't match with %s\n", + nbd->backend); + return -EINVAL; + } + } else { + mutex_unlock(&nbd_index_mutex); + dev_err(nbd_to_dev(nbd), "must specify backend\n"); + return -EINVAL; + } + } if (!refcount_inc_not_zero(&nbd->refs)) { mutex_unlock(&nbd_index_mutex); printk(KERN_ERR "nbd: device at index %d is going down\n", diff --git a/include/uapi/linux/nbd-netlink.h b/include/uapi/linux/nbd-netlink.h index c5d0ef7aa7d5..2d0b90964227 100644 --- a/include/uapi/linux/nbd-netlink.h +++ b/include/uapi/linux/nbd-netlink.h @@ -35,6 +35,7 @@ enum { NBD_ATTR_SOCKETS, NBD_ATTR_DEAD_CONN_TIMEOUT, NBD_ATTR_DEVICE_LIST, + NBD_ATTR_BACKEND_IDENTIFIER, __NBD_ATTR_MAX, }; #define NBD_ATTR_MAX (__NBD_ATTR_MAX - 1) -- cgit v1.2.3 From 964ab0040ff9598783bf37776b5e31b27b50e293 Mon Sep 17 00:00:00 2001 From: Axel Rasmussen Date: Wed, 30 Jun 2021 18:49:27 -0700 Subject: userfaultfd/shmem: advertise shmem minor fault support Now that the feature is fully implemented (the faulting path hooks exist so userspace is notified, and the ioctl to resolve such faults is available), advertise this as a supported feature. Link: https://lkml.kernel.org/r/20210503180737.2487560-6-axelrasmussen@google.com Signed-off-by: Axel Rasmussen Acked-by: Hugh Dickins Acked-by: Peter Xu Cc: Alexander Viro Cc: Andrea Arcangeli Cc: Brian Geffon Cc: "Dr . David Alan Gilbert" Cc: Jerome Glisse Cc: Joe Perches Cc: Kirill A. Shutemov Cc: Lokesh Gidra Cc: Mike Kravetz Cc: Mike Rapoport Cc: Mina Almasry Cc: Oliver Upton Cc: Shaohua Li Cc: Shuah Khan Cc: Stephen Rothwell Cc: Wang Qing Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- Documentation/admin-guide/mm/userfaultfd.rst | 3 ++- fs/userfaultfd.c | 3 ++- include/uapi/linux/userfaultfd.h | 7 ++++++- 3 files changed, 10 insertions(+), 3 deletions(-) (limited to 'include/uapi') diff --git a/Documentation/admin-guide/mm/userfaultfd.rst b/Documentation/admin-guide/mm/userfaultfd.rst index 3aa38e8b8361..6528036093e1 100644 --- a/Documentation/admin-guide/mm/userfaultfd.rst +++ b/Documentation/admin-guide/mm/userfaultfd.rst @@ -77,7 +77,8 @@ events, except page fault notifications, may be generated: - ``UFFD_FEATURE_MINOR_HUGETLBFS`` indicates that the kernel supports ``UFFDIO_REGISTER_MODE_MINOR`` registration for hugetlbfs virtual memory - areas. + areas. ``UFFD_FEATURE_MINOR_SHMEM`` is the analogous feature indicating + support for shmem virtual memory areas. The userland application should set the feature flags it intends to use when invoking the ``UFFDIO_API`` ioctl, to request that those features be diff --git a/fs/userfaultfd.c b/fs/userfaultfd.c index 82ef253d66b6..19ebae443ade 100644 --- a/fs/userfaultfd.c +++ b/fs/userfaultfd.c @@ -1944,7 +1944,8 @@ static int userfaultfd_api(struct userfaultfd_ctx *ctx, /* report all available features and ioctls to userland */ uffdio_api.features = UFFD_API_FEATURES; #ifndef CONFIG_HAVE_ARCH_USERFAULTFD_MINOR - uffdio_api.features &= ~UFFD_FEATURE_MINOR_HUGETLBFS; + uffdio_api.features &= + ~(UFFD_FEATURE_MINOR_HUGETLBFS | UFFD_FEATURE_MINOR_SHMEM); #endif #ifndef CONFIG_HAVE_ARCH_USERFAULTFD_WP uffdio_api.features &= ~UFFD_FEATURE_PAGEFAULT_FLAG_WP; diff --git a/include/uapi/linux/userfaultfd.h b/include/uapi/linux/userfaultfd.h index 650480f41f1d..05b31d60acf6 100644 --- a/include/uapi/linux/userfaultfd.h +++ b/include/uapi/linux/userfaultfd.h @@ -31,7 +31,8 @@ UFFD_FEATURE_MISSING_SHMEM | \ UFFD_FEATURE_SIGBUS | \ UFFD_FEATURE_THREAD_ID | \ - UFFD_FEATURE_MINOR_HUGETLBFS) + UFFD_FEATURE_MINOR_HUGETLBFS | \ + UFFD_FEATURE_MINOR_SHMEM) #define UFFD_API_IOCTLS \ ((__u64)1 << _UFFDIO_REGISTER | \ (__u64)1 << _UFFDIO_UNREGISTER | \ @@ -185,6 +186,9 @@ struct uffdio_api { * UFFD_FEATURE_MINOR_HUGETLBFS indicates that minor faults * can be intercepted (via REGISTER_MODE_MINOR) for * hugetlbfs-backed pages. + * + * UFFD_FEATURE_MINOR_SHMEM indicates the same support as + * UFFD_FEATURE_MINOR_HUGETLBFS, but for shmem-backed pages instead. */ #define UFFD_FEATURE_PAGEFAULT_FLAG_WP (1<<0) #define UFFD_FEATURE_EVENT_FORK (1<<1) @@ -196,6 +200,7 @@ struct uffdio_api { #define UFFD_FEATURE_SIGBUS (1<<7) #define UFFD_FEATURE_THREAD_ID (1<<8) #define UFFD_FEATURE_MINOR_HUGETLBFS (1<<9) +#define UFFD_FEATURE_MINOR_SHMEM (1<<10) __u64 features; __u64 ioctls; -- cgit v1.2.3 From 7858d7bca7fbbbbd5b940d2ec371b2d060b21b84 Mon Sep 17 00:00:00 2001 From: Feng Tang Date: Wed, 30 Jun 2021 18:51:00 -0700 Subject: mm/mempolicy: don't handle MPOL_LOCAL like a fake MPOL_PREFERRED policy MPOL_LOCAL policy has been setup as a real policy, but it is still handled like a faked POL_PREFERRED policy with one internal MPOL_F_LOCAL flag bit set, and there are many places having to judge the real 'prefer' or the 'local' policy, which are quite confusing. In current code, there are 4 cases that MPOL_LOCAL are used: 1. user specifies 'local' policy 2. user specifies 'prefer' policy, but with empty nodemask 3. system 'default' policy is used 4. 'prefer' policy + valid 'preferred' node with MPOL_F_STATIC_NODES flag set, and when it is 'rebind' to a nodemask which doesn't contains the 'preferred' node, it will perform as 'local' policy So make 'local' a real policy instead of a fake 'prefer' one, and kill MPOL_F_LOCAL bit, which can greatly reduce the confusion for code reading. For case 4, the logic of mpol_rebind_preferred() is confusing, as Michal Hocko pointed out: : I do believe that rebinding preferred policy is just bogus and it should : be dropped altogether on the ground that a preference is a mere hint from : userspace where to start the allocation. Unless I am missing something : cpusets will be always authoritative for the final placement. The : preferred node just acts as a starting point and it should be really : preserved when cpusets changes. Otherwise we have a very subtle behavior : corner cases. So dump all the tricky transformation between 'prefer' and 'local', and just record the new nodemask of rebinding. [feng.tang@intel.com: fix a problem in mpol_set_nodemask(), per Michal Hocko] Link: https://lkml.kernel.org/r/1622560492-1294-3-git-send-email-feng.tang@intel.com [feng.tang@intel.com: refine code and comments of mpol_set_nodemask(), per Michal] Link: https://lkml.kernel.org/r/20210603081807.GE56979@shbuild999.sh.intel.com Link: https://lkml.kernel.org/r/1622469956-82897-3-git-send-email-feng.tang@intel.com Signed-off-by: Feng Tang Suggested-by: Michal Hocko Acked-by: Michal Hocko Cc: Andi Kleen Cc: Andrea Arcangeli Cc: Ben Widawsky Cc: Dan Williams Cc: Dave Hansen Cc: David Rientjes Cc: Huang Ying Cc: Mel Gorman Cc: Michal Hocko Cc: Mike Kravetz Cc: Randy Dunlap Cc: Vlastimil Babka Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- include/uapi/linux/mempolicy.h | 1 - mm/mempolicy.c | 136 +++++++++++++++++------------------------ 2 files changed, 56 insertions(+), 81 deletions(-) (limited to 'include/uapi') diff --git a/include/uapi/linux/mempolicy.h b/include/uapi/linux/mempolicy.h index 4832fd0b5642..19a00bc7fe86 100644 --- a/include/uapi/linux/mempolicy.h +++ b/include/uapi/linux/mempolicy.h @@ -60,7 +60,6 @@ enum { * are never OR'ed into the mode in mempolicy API arguments. */ #define MPOL_F_SHARED (1 << 0) /* identify shared policies */ -#define MPOL_F_LOCAL (1 << 1) /* preferred local allocation */ #define MPOL_F_MOF (1 << 3) /* this policy wants migrate on fault */ #define MPOL_F_MORON (1 << 4) /* Migrate On protnone Reference On Node */ diff --git a/mm/mempolicy.c b/mm/mempolicy.c index bd213b900e71..22addae6c4ee 100644 --- a/mm/mempolicy.c +++ b/mm/mempolicy.c @@ -121,8 +121,7 @@ enum zone_type policy_zone = 0; */ static struct mempolicy default_policy = { .refcnt = ATOMIC_INIT(1), /* never free it */ - .mode = MPOL_PREFERRED, - .flags = MPOL_F_LOCAL, + .mode = MPOL_LOCAL, }; static struct mempolicy preferred_node_policy[MAX_NUMNODES]; @@ -200,12 +199,9 @@ static int mpol_new_interleave(struct mempolicy *pol, const nodemask_t *nodes) static int mpol_new_preferred(struct mempolicy *pol, const nodemask_t *nodes) { - if (!nodes) - pol->flags |= MPOL_F_LOCAL; /* local allocation */ - else if (nodes_empty(*nodes)) - return -EINVAL; /* no allowed nodes */ - else - pol->v.preferred_node = first_node(*nodes); + if (nodes_empty(*nodes)) + return -EINVAL; + pol->v.preferred_node = first_node(*nodes); return 0; } @@ -220,8 +216,7 @@ static int mpol_new_bind(struct mempolicy *pol, const nodemask_t *nodes) /* * mpol_set_nodemask is called after mpol_new() to set up the nodemask, if * any, for the new policy. mpol_new() has already validated the nodes - * parameter with respect to the policy mode and flags. But, we need to - * handle an empty nodemask with MPOL_PREFERRED here. + * parameter with respect to the policy mode and flags. * * Must be called holding task's alloc_lock to protect task's mems_allowed * and mempolicy. May also be called holding the mmap_lock for write. @@ -231,33 +226,31 @@ static int mpol_set_nodemask(struct mempolicy *pol, { int ret; - /* if mode is MPOL_DEFAULT, pol is NULL. This is right. */ - if (pol == NULL) + /* + * Default (pol==NULL) resp. local memory policies are not a + * subject of any remapping. They also do not need any special + * constructor. + */ + if (!pol || pol->mode == MPOL_LOCAL) return 0; + /* Check N_MEMORY */ nodes_and(nsc->mask1, cpuset_current_mems_allowed, node_states[N_MEMORY]); VM_BUG_ON(!nodes); - if (pol->mode == MPOL_PREFERRED && nodes_empty(*nodes)) - nodes = NULL; /* explicit local allocation */ - else { - if (pol->flags & MPOL_F_RELATIVE_NODES) - mpol_relative_nodemask(&nsc->mask2, nodes, &nsc->mask1); - else - nodes_and(nsc->mask2, *nodes, nsc->mask1); - if (mpol_store_user_nodemask(pol)) - pol->w.user_nodemask = *nodes; - else - pol->w.cpuset_mems_allowed = - cpuset_current_mems_allowed; - } + if (pol->flags & MPOL_F_RELATIVE_NODES) + mpol_relative_nodemask(&nsc->mask2, nodes, &nsc->mask1); + else + nodes_and(nsc->mask2, *nodes, nsc->mask1); - if (nodes) - ret = mpol_ops[pol->mode].create(pol, &nsc->mask2); + if (mpol_store_user_nodemask(pol)) + pol->w.user_nodemask = *nodes; else - ret = mpol_ops[pol->mode].create(pol, NULL); + pol->w.cpuset_mems_allowed = cpuset_current_mems_allowed; + + ret = mpol_ops[pol->mode].create(pol, &nsc->mask2); return ret; } @@ -290,13 +283,14 @@ static struct mempolicy *mpol_new(unsigned short mode, unsigned short flags, if (((flags & MPOL_F_STATIC_NODES) || (flags & MPOL_F_RELATIVE_NODES))) return ERR_PTR(-EINVAL); + + mode = MPOL_LOCAL; } } else if (mode == MPOL_LOCAL) { if (!nodes_empty(*nodes) || (flags & MPOL_F_STATIC_NODES) || (flags & MPOL_F_RELATIVE_NODES)) return ERR_PTR(-EINVAL); - mode = MPOL_PREFERRED; } else if (nodes_empty(*nodes)) return ERR_PTR(-EINVAL); policy = kmem_cache_alloc(policy_cache, GFP_KERNEL); @@ -344,25 +338,7 @@ static void mpol_rebind_nodemask(struct mempolicy *pol, const nodemask_t *nodes) static void mpol_rebind_preferred(struct mempolicy *pol, const nodemask_t *nodes) { - nodemask_t tmp; - - if (pol->flags & MPOL_F_STATIC_NODES) { - int node = first_node(pol->w.user_nodemask); - - if (node_isset(node, *nodes)) { - pol->v.preferred_node = node; - pol->flags &= ~MPOL_F_LOCAL; - } else - pol->flags |= MPOL_F_LOCAL; - } else if (pol->flags & MPOL_F_RELATIVE_NODES) { - mpol_relative_nodemask(&tmp, &pol->w.user_nodemask, nodes); - pol->v.preferred_node = first_node(tmp); - } else if (!(pol->flags & MPOL_F_LOCAL)) { - pol->v.preferred_node = node_remap(pol->v.preferred_node, - pol->w.cpuset_mems_allowed, - *nodes); - pol->w.cpuset_mems_allowed = *nodes; - } + pol->w.cpuset_mems_allowed = *nodes; } /* @@ -376,7 +352,7 @@ static void mpol_rebind_policy(struct mempolicy *pol, const nodemask_t *newmask) { if (!pol) return; - if (!mpol_store_user_nodemask(pol) && !(pol->flags & MPOL_F_LOCAL) && + if (!mpol_store_user_nodemask(pol) && nodes_equal(pol->w.cpuset_mems_allowed, *newmask)) return; @@ -427,6 +403,9 @@ static const struct mempolicy_operations mpol_ops[MPOL_MAX] = { .create = mpol_new_bind, .rebind = mpol_rebind_nodemask, }, + [MPOL_LOCAL] = { + .rebind = mpol_rebind_default, + }, }; static int migrate_page_add(struct page *page, struct list_head *pagelist, @@ -919,10 +898,12 @@ static void get_policy_nodemask(struct mempolicy *p, nodemask_t *nodes) case MPOL_INTERLEAVE: *nodes = p->v.nodes; break; + case MPOL_LOCAL: + /* return empty node mask for local allocation */ + break; + case MPOL_PREFERRED: - if (!(p->flags & MPOL_F_LOCAL)) - node_set(p->v.preferred_node, *nodes); - /* else return empty node mask for local allocation */ + node_set(p->v.preferred_node, *nodes); break; default: BUG(); @@ -1894,9 +1875,9 @@ nodemask_t *policy_nodemask(gfp_t gfp, struct mempolicy *policy) /* Return the node id preferred by the given mempolicy, or the given id */ static int policy_node(gfp_t gfp, struct mempolicy *policy, int nd) { - if (policy->mode == MPOL_PREFERRED && !(policy->flags & MPOL_F_LOCAL)) + if (policy->mode == MPOL_PREFERRED) { nd = policy->v.preferred_node; - else { + } else { /* * __GFP_THISNODE shouldn't even be used with the bind policy * because we might easily break the expectation to stay on the @@ -1933,14 +1914,11 @@ unsigned int mempolicy_slab_node(void) return node; policy = current->mempolicy; - if (!policy || policy->flags & MPOL_F_LOCAL) + if (!policy) return node; switch (policy->mode) { case MPOL_PREFERRED: - /* - * handled MPOL_F_LOCAL above - */ return policy->v.preferred_node; case MPOL_INTERLEAVE: @@ -1960,6 +1938,8 @@ unsigned int mempolicy_slab_node(void) &policy->v.nodes); return z->zone ? zone_to_nid(z->zone) : node; } + case MPOL_LOCAL: + return node; default: BUG(); @@ -2072,16 +2052,18 @@ bool init_nodemask_of_mempolicy(nodemask_t *mask) mempolicy = current->mempolicy; switch (mempolicy->mode) { case MPOL_PREFERRED: - if (mempolicy->flags & MPOL_F_LOCAL) - nid = numa_node_id(); - else - nid = mempolicy->v.preferred_node; + nid = mempolicy->v.preferred_node; init_nodemask_of_node(mask, nid); break; case MPOL_BIND: case MPOL_INTERLEAVE: - *mask = mempolicy->v.nodes; + *mask = mempolicy->v.nodes; + break; + + case MPOL_LOCAL: + nid = numa_node_id(); + init_nodemask_of_node(mask, nid); break; default: @@ -2188,7 +2170,7 @@ struct page *alloc_pages_vma(gfp_t gfp, int order, struct vm_area_struct *vma, * If the policy is interleave, or does not allow the current * node in its nodemask, we allocate the standard way. */ - if (pol->mode == MPOL_PREFERRED && !(pol->flags & MPOL_F_LOCAL)) + if (pol->mode == MPOL_PREFERRED) hpage_node = pol->v.preferred_node; nmask = policy_nodemask(gfp, pol); @@ -2324,10 +2306,9 @@ bool __mpol_equal(struct mempolicy *a, struct mempolicy *b) case MPOL_INTERLEAVE: return !!nodes_equal(a->v.nodes, b->v.nodes); case MPOL_PREFERRED: - /* a's ->flags is the same as b's */ - if (a->flags & MPOL_F_LOCAL) - return true; return a->v.preferred_node == b->v.preferred_node; + case MPOL_LOCAL: + return true; default: BUG(); return false; @@ -2465,10 +2446,11 @@ int mpol_misplaced(struct page *page, struct vm_area_struct *vma, unsigned long break; case MPOL_PREFERRED: - if (pol->flags & MPOL_F_LOCAL) - polnid = numa_node_id(); - else - polnid = pol->v.preferred_node; + polnid = pol->v.preferred_node; + break; + + case MPOL_LOCAL: + polnid = numa_node_id(); break; case MPOL_BIND: @@ -2835,9 +2817,6 @@ void numa_default_policy(void) * Parse and format mempolicy from/to strings */ -/* - * "local" is implemented internally by MPOL_PREFERRED with MPOL_F_LOCAL flag. - */ static const char * const policy_modes[] = { [MPOL_DEFAULT] = "default", @@ -2915,7 +2894,6 @@ int mpol_parse_str(char *str, struct mempolicy **mpol) */ if (nodelist) goto out; - mode = MPOL_PREFERRED; break; case MPOL_DEFAULT: /* @@ -2959,7 +2937,7 @@ int mpol_parse_str(char *str, struct mempolicy **mpol) else if (nodelist) new->v.preferred_node = first_node(nodes); else - new->flags |= MPOL_F_LOCAL; + new->mode = MPOL_LOCAL; /* * Save nodes for contextualization: this will be used to "clone" @@ -3005,12 +2983,10 @@ void mpol_to_str(char *buffer, int maxlen, struct mempolicy *pol) switch (mode) { case MPOL_DEFAULT: + case MPOL_LOCAL: break; case MPOL_PREFERRED: - if (flags & MPOL_F_LOCAL) - mode = MPOL_LOCAL; - else - node_set(pol->v.preferred_node, nodes); + node_set(pol->v.preferred_node, nodes); break; case MPOL_BIND: case MPOL_INTERLEAVE: -- cgit v1.2.3 From 4ca9b3859dac14bbef0c27d00667bb5b10917adb Mon Sep 17 00:00:00 2001 From: David Hildenbrand Date: Wed, 30 Jun 2021 18:52:28 -0700 Subject: mm/madvise: introduce MADV_POPULATE_(READ|WRITE) to prefault page tables I. Background: Sparse Memory Mappings When we manage sparse memory mappings dynamically in user space - also sometimes involving MAP_NORESERVE - we want to dynamically populate/ discard memory inside such a sparse memory region. Example users are hypervisors (especially implementing memory ballooning or similar technologies like virtio-mem) and memory allocators. In addition, we want to fail in a nice way (instead of generating SIGBUS) if populating does not succeed because we are out of backend memory (which can happen easily with file-based mappings, especially tmpfs and hugetlbfs). While MADV_DONTNEED, MADV_REMOVE and FALLOC_FL_PUNCH_HOLE allow for reliably discarding memory for most mapping types, there is no generic approach to populate page tables and preallocate memory. Although mmap() supports MAP_POPULATE, it is not applicable to the concept of sparse memory mappings, where we want to populate/discard dynamically and avoid expensive/problematic remappings. In addition, we never actually report errors during the final populate phase - it is best-effort only. fallocate() can be used to preallocate file-based memory and fail in a safe way. However, it cannot really be used for any private mappings on anonymous files via memfd due to COW semantics. In addition, fallocate() does not actually populate page tables, so we still always get pagefaults on first access - which is sometimes undesired (i.e., real-time workloads) and requires real prefaulting of page tables, not just a preallocation of backend storage. There might be interesting use cases for sparse memory regions along with mlockall(MCL_ONFAULT) which fallocate() cannot satisfy as it does not prefault page tables. II. On preallcoation/prefaulting from user space Because we don't have a proper interface, what applications (like QEMU and databases) end up doing is touching (i.e., reading+writing one byte to not overwrite existing data) all individual pages. However, that approach 1) Can result in wear on storage backing, because we end up reading/writing each page; this is especially a problem for dax/pmem. 2) Can result in mmap_sem contention when prefaulting via multiple threads. 3) Requires expensive signal handling, especially to catch SIGBUS in case of hugetlbfs/shmem/file-backed memory. For example, this is problematic in hypervisors like QEMU where SIGBUS handlers might already be used by other subsystems concurrently to e.g, handle hardware errors. "Simply" doing preallocation concurrently from other thread is not that easy. III. On MADV_WILLNEED Extending MADV_WILLNEED is not an option because 1. It would change the semantics: "Expect access in the near future." and "might be a good idea to read some pages" vs. "Definitely populate/ preallocate all memory and definitely fail on errors.". 2. Existing users (like virtio-balloon in QEMU when deflating the balloon) don't want populate/prealloc semantics. They treat this rather as a hint to give a little performance boost without too much overhead - and don't expect that a lot of memory might get consumed or a lot of time might be spent. IV. MADV_POPULATE_READ and MADV_POPULATE_WRITE Let's introduce MADV_POPULATE_READ and MADV_POPULATE_WRITE, inspired by MAP_POPULATE, with the following semantics: 1. MADV_POPULATE_READ can be used to prefault page tables just like manually reading each individual page. This will not break any COW mappings. The shared zero page might get mapped and no backend storage might get preallocated -- allocation might be deferred to write-fault time. Especially shared file mappings require an explicit fallocate() upfront to actually preallocate backend memory (blocks in the file system) in case the file might have holes. 2. If MADV_POPULATE_READ succeeds, all page tables have been populated (prefaulted) readable once. 3. MADV_POPULATE_WRITE can be used to preallocate backend memory and prefault page tables just like manually writing (or reading+writing) each individual page. This will break any COW mappings -- e.g., the shared zeropage is never populated. 4. If MADV_POPULATE_WRITE succeeds, all page tables have been populated (prefaulted) writable once. 5. MADV_POPULATE_READ and MADV_POPULATE_WRITE cannot be applied to special mappings marked with VM_PFNMAP and VM_IO. Also, proper access permissions (e.g., PROT_READ, PROT_WRITE) are required. If any such mapping is encountered, madvise() fails with -EINVAL. 6. If MADV_POPULATE_READ or MADV_POPULATE_WRITE fails, some page tables might have been populated. 7. MADV_POPULATE_READ and MADV_POPULATE_WRITE will return -EHWPOISON when encountering a HW poisoned page in the range. 8. Similar to MAP_POPULATE, MADV_POPULATE_READ and MADV_POPULATE_WRITE cannot protect from the OOM (Out Of Memory) handler killing the process. While the use case for MADV_POPULATE_WRITE is fairly obvious (i.e., preallocate memory and prefault page tables for VMs), one issue is that whenever we prefault pages writable, the pages have to be marked dirty, because the CPU could dirty them any time. while not a real problem for hugetlbfs or dax/pmem, it can be a problem for shared file mappings: each page will be marked dirty and has to be written back later when evicting. MADV_POPULATE_READ allows for optimizing this scenario: Pre-read a whole mapping from backend storage without marking it dirty, such that eviction won't have to write it back. As discussed above, shared file mappings might require an explciit fallocate() upfront to achieve preallcoation+prepopulation. Although sparse memory mappings are the primary use case, this will also be useful for other preallocate/prefault use cases where MAP_POPULATE is not desired or the semantics of MAP_POPULATE are not sufficient: as one example, QEMU users can trigger preallocation/prefaulting of guest RAM after the mapping was created -- and don't want errors to be silently suppressed. Looking at the history, MADV_POPULATE was already proposed in 2013 [1], however, the main motivation back than was performance improvements -- which should also still be the case. V. Single-threaded performance comparison I did a short experiment, prefaulting page tables on completely *empty mappings/files* and repeated the experiment 10 times. The results correspond to the shortest execution time. In general, the performance benefit for huge pages is negligible with small mappings. V.1: Private mappings POPULATE_READ and POPULATE_WRITE is fastest. Note that Reading/POPULATE_READ will populate the shared zeropage where applicable -- which result in short population times. The fastest way to allocate backend storage (here: swap or huge pages) and prefault page tables is POPULATE_WRITE. V.2: Shared mappings fallocate() is fastest, however, doesn't prefault page tables. POPULATE_WRITE is faster than simple writes and read/writes. POPULATE_READ is faster than simple reads. Without a fd, the fastest way to allocate backend storage and prefault page tables is POPULATE_WRITE. With an fd, the fastest way is usually FALLOCATE+POPULATE_READ or FALLOCATE+POPULATE_WRITE respectively; one exception are actual files: FALLOCATE+Read is slightly faster than FALLOCATE+POPULATE_READ. The fastest way to allocate backend storage prefault page tables is FALLOCATE+POPULATE_WRITE -- except when dealing with actual files; then, FALLOCATE+POPULATE_READ is fastest and won't directly mark all pages as dirty. v.3: Detailed results ================================================== 2 MiB MAP_PRIVATE: ************************************************** Anon 4 KiB : Read : 0.119 ms Anon 4 KiB : Write : 0.222 ms Anon 4 KiB : Read/Write : 0.380 ms Anon 4 KiB : POPULATE_READ : 0.060 ms Anon 4 KiB : POPULATE_WRITE : 0.158 ms Memfd 4 KiB : Read : 0.034 ms Memfd 4 KiB : Write : 0.310 ms Memfd 4 KiB : Read/Write : 0.362 ms Memfd 4 KiB : POPULATE_READ : 0.039 ms Memfd 4 KiB : POPULATE_WRITE : 0.229 ms Memfd 2 MiB : Read : 0.030 ms Memfd 2 MiB : Write : 0.030 ms Memfd 2 MiB : Read/Write : 0.030 ms Memfd 2 MiB : POPULATE_READ : 0.030 ms Memfd 2 MiB : POPULATE_WRITE : 0.030 ms tmpfs : Read : 0.033 ms tmpfs : Write : 0.313 ms tmpfs : Read/Write : 0.406 ms tmpfs : POPULATE_READ : 0.039 ms tmpfs : POPULATE_WRITE : 0.285 ms file : Read : 0.033 ms file : Write : 0.351 ms file : Read/Write : 0.408 ms file : POPULATE_READ : 0.039 ms file : POPULATE_WRITE : 0.290 ms hugetlbfs : Read : 0.030 ms hugetlbfs : Write : 0.030 ms hugetlbfs : Read/Write : 0.030 ms hugetlbfs : POPULATE_READ : 0.030 ms hugetlbfs : POPULATE_WRITE : 0.030 ms ************************************************** 4096 MiB MAP_PRIVATE: ************************************************** Anon 4 KiB : Read : 237.940 ms Anon 4 KiB : Write : 708.409 ms Anon 4 KiB : Read/Write : 1054.041 ms Anon 4 KiB : POPULATE_READ : 124.310 ms Anon 4 KiB : POPULATE_WRITE : 572.582 ms Memfd 4 KiB : Read : 136.928 ms Memfd 4 KiB : Write : 963.898 ms Memfd 4 KiB : Read/Write : 1106.561 ms Memfd 4 KiB : POPULATE_READ : 78.450 ms Memfd 4 KiB : POPULATE_WRITE : 805.881 ms Memfd 2 MiB : Read : 357.116 ms Memfd 2 MiB : Write : 357.210 ms Memfd 2 MiB : Read/Write : 357.606 ms Memfd 2 MiB : POPULATE_READ : 356.094 ms Memfd 2 MiB : POPULATE_WRITE : 356.937 ms tmpfs : Read : 137.536 ms tmpfs : Write : 954.362 ms tmpfs : Read/Write : 1105.954 ms tmpfs : POPULATE_READ : 80.289 ms tmpfs : POPULATE_WRITE : 822.826 ms file : Read : 137.874 ms file : Write : 987.025 ms file : Read/Write : 1107.439 ms file : POPULATE_READ : 80.413 ms file : POPULATE_WRITE : 857.622 ms hugetlbfs : Read : 355.607 ms hugetlbfs : Write : 355.729 ms hugetlbfs : Read/Write : 356.127 ms hugetlbfs : POPULATE_READ : 354.585 ms hugetlbfs : POPULATE_WRITE : 355.138 ms ************************************************** 2 MiB MAP_SHARED: ************************************************** Anon 4 KiB : Read : 0.394 ms Anon 4 KiB : Write : 0.348 ms Anon 4 KiB : Read/Write : 0.400 ms Anon 4 KiB : POPULATE_READ : 0.326 ms Anon 4 KiB : POPULATE_WRITE : 0.273 ms Anon 2 MiB : Read : 0.030 ms Anon 2 MiB : Write : 0.030 ms Anon 2 MiB : Read/Write : 0.030 ms Anon 2 MiB : POPULATE_READ : 0.030 ms Anon 2 MiB : POPULATE_WRITE : 0.030 ms Memfd 4 KiB : Read : 0.412 ms Memfd 4 KiB : Write : 0.372 ms Memfd 4 KiB : Read/Write : 0.419 ms Memfd 4 KiB : POPULATE_READ : 0.343 ms Memfd 4 KiB : POPULATE_WRITE : 0.288 ms Memfd 4 KiB : FALLOCATE : 0.137 ms Memfd 4 KiB : FALLOCATE+Read : 0.446 ms Memfd 4 KiB : FALLOCATE+Write : 0.330 ms Memfd 4 KiB : FALLOCATE+Read/Write : 0.454 ms Memfd 4 KiB : FALLOCATE+POPULATE_READ : 0.379 ms Memfd 4 KiB : FALLOCATE+POPULATE_WRITE : 0.268 ms Memfd 2 MiB : Read : 0.030 ms Memfd 2 MiB : Write : 0.030 ms Memfd 2 MiB : Read/Write : 0.030 ms Memfd 2 MiB : POPULATE_READ : 0.030 ms Memfd 2 MiB : POPULATE_WRITE : 0.030 ms Memfd 2 MiB : FALLOCATE : 0.030 ms Memfd 2 MiB : FALLOCATE+Read : 0.031 ms Memfd 2 MiB : FALLOCATE+Write : 0.031 ms Memfd 2 MiB : FALLOCATE+Read/Write : 0.031 ms Memfd 2 MiB : FALLOCATE+POPULATE_READ : 0.030 ms Memfd 2 MiB : FALLOCATE+POPULATE_WRITE : 0.030 ms tmpfs : Read : 0.416 ms tmpfs : Write : 0.369 ms tmpfs : Read/Write : 0.425 ms tmpfs : POPULATE_READ : 0.346 ms tmpfs : POPULATE_WRITE : 0.295 ms tmpfs : FALLOCATE : 0.139 ms tmpfs : FALLOCATE+Read : 0.447 ms tmpfs : FALLOCATE+Write : 0.333 ms tmpfs : FALLOCATE+Read/Write : 0.454 ms tmpfs : FALLOCATE+POPULATE_READ : 0.380 ms tmpfs : FALLOCATE+POPULATE_WRITE : 0.272 ms file : Read : 0.191 ms file : Write : 0.511 ms file : Read/Write : 0.524 ms file : POPULATE_READ : 0.196 ms file : POPULATE_WRITE : 0.434 ms file : FALLOCATE : 0.004 ms file : FALLOCATE+Read : 0.197 ms file : FALLOCATE+Write : 0.554 ms file : FALLOCATE+Read/Write : 0.480 ms file : FALLOCATE+POPULATE_READ : 0.201 ms file : FALLOCATE+POPULATE_WRITE : 0.381 ms hugetlbfs : Read : 0.030 ms hugetlbfs : Write : 0.030 ms hugetlbfs : Read/Write : 0.030 ms hugetlbfs : POPULATE_READ : 0.030 ms hugetlbfs : POPULATE_WRITE : 0.030 ms hugetlbfs : FALLOCATE : 0.030 ms hugetlbfs : FALLOCATE+Read : 0.031 ms hugetlbfs : FALLOCATE+Write : 0.031 ms hugetlbfs : FALLOCATE+Read/Write : 0.030 ms hugetlbfs : FALLOCATE+POPULATE_READ : 0.030 ms hugetlbfs : FALLOCATE+POPULATE_WRITE : 0.030 ms ************************************************** 4096 MiB MAP_SHARED: ************************************************** Anon 4 KiB : Read : 1053.090 ms Anon 4 KiB : Write : 913.642 ms Anon 4 KiB : Read/Write : 1060.350 ms Anon 4 KiB : POPULATE_READ : 893.691 ms Anon 4 KiB : POPULATE_WRITE : 782.885 ms Anon 2 MiB : Read : 358.553 ms Anon 2 MiB : Write : 358.419 ms Anon 2 MiB : Read/Write : 357.992 ms Anon 2 MiB : POPULATE_READ : 357.533 ms Anon 2 MiB : POPULATE_WRITE : 357.808 ms Memfd 4 KiB : Read : 1078.144 ms Memfd 4 KiB : Write : 942.036 ms Memfd 4 KiB : Read/Write : 1100.391 ms Memfd 4 KiB : POPULATE_READ : 925.829 ms Memfd 4 KiB : POPULATE_WRITE : 804.394 ms Memfd 4 KiB : FALLOCATE : 304.632 ms Memfd 4 KiB : FALLOCATE+Read : 1163.359 ms Memfd 4 KiB : FALLOCATE+Write : 933.186 ms Memfd 4 KiB : FALLOCATE+Read/Write : 1187.304 ms Memfd 4 KiB : FALLOCATE+POPULATE_READ : 1013.660 ms Memfd 4 KiB : FALLOCATE+POPULATE_WRITE : 794.560 ms Memfd 2 MiB : Read : 358.131 ms Memfd 2 MiB : Write : 358.099 ms Memfd 2 MiB : Read/Write : 358.250 ms Memfd 2 MiB : POPULATE_READ : 357.563 ms Memfd 2 MiB : POPULATE_WRITE : 357.334 ms Memfd 2 MiB : FALLOCATE : 356.735 ms Memfd 2 MiB : FALLOCATE+Read : 358.152 ms Memfd 2 MiB : FALLOCATE+Write : 358.331 ms Memfd 2 MiB : FALLOCATE+Read/Write : 358.018 ms Memfd 2 MiB : FALLOCATE+POPULATE_READ : 357.286 ms Memfd 2 MiB : FALLOCATE+POPULATE_WRITE : 357.523 ms tmpfs : Read : 1087.265 ms tmpfs : Write : 950.840 ms tmpfs : Read/Write : 1107.567 ms tmpfs : POPULATE_READ : 922.605 ms tmpfs : POPULATE_WRITE : 810.094 ms tmpfs : FALLOCATE : 306.320 ms tmpfs : FALLOCATE+Read : 1169.796 ms tmpfs : FALLOCATE+Write : 933.730 ms tmpfs : FALLOCATE+Read/Write : 1191.610 ms tmpfs : FALLOCATE+POPULATE_READ : 1020.474 ms tmpfs : FALLOCATE+POPULATE_WRITE : 798.945 ms file : Read : 654.101 ms file : Write : 1259.142 ms file : Read/Write : 1289.509 ms file : POPULATE_READ : 661.642 ms file : POPULATE_WRITE : 1106.816 ms file : FALLOCATE : 1.864 ms file : FALLOCATE+Read : 656.328 ms file : FALLOCATE+Write : 1153.300 ms file : FALLOCATE+Read/Write : 1180.613 ms file : FALLOCATE+POPULATE_READ : 668.347 ms file : FALLOCATE+POPULATE_WRITE : 996.143 ms hugetlbfs : Read : 357.245 ms hugetlbfs : Write : 357.413 ms hugetlbfs : Read/Write : 357.120 ms hugetlbfs : POPULATE_READ : 356.321 ms hugetlbfs : POPULATE_WRITE : 356.693 ms hugetlbfs : FALLOCATE : 355.927 ms hugetlbfs : FALLOCATE+Read : 357.074 ms hugetlbfs : FALLOCATE+Write : 357.120 ms hugetlbfs : FALLOCATE+Read/Write : 356.983 ms hugetlbfs : FALLOCATE+POPULATE_READ : 356.413 ms hugetlbfs : FALLOCATE+POPULATE_WRITE : 356.266 ms ************************************************** [1] https://lkml.org/lkml/2013/6/27/698 [akpm@linux-foundation.org: coding style fixes] Link: https://lkml.kernel.org/r/20210419135443.12822-3-david@redhat.com Signed-off-by: David Hildenbrand Cc: Arnd Bergmann Cc: Michal Hocko Cc: Oscar Salvador Cc: Matthew Wilcox (Oracle) Cc: Andrea Arcangeli Cc: Minchan Kim Cc: Jann Horn Cc: Jason Gunthorpe Cc: Dave Hansen Cc: Hugh Dickins Cc: Rik van Riel Cc: Michael S. Tsirkin Cc: Kirill A. Shutemov Cc: Vlastimil Babka Cc: Richard Henderson Cc: Ivan Kokshaysky Cc: Matt Turner Cc: Thomas Bogendoerfer Cc: "James E.J. Bottomley" Cc: Helge Deller Cc: Chris Zankel Cc: Max Filippov Cc: Mike Kravetz Cc: Peter Xu Cc: Rolf Eike Beer Cc: Ram Pai Cc: Shuah Khan Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- arch/alpha/include/uapi/asm/mman.h | 3 ++ arch/mips/include/uapi/asm/mman.h | 3 ++ arch/parisc/include/uapi/asm/mman.h | 3 ++ arch/xtensa/include/uapi/asm/mman.h | 3 ++ include/uapi/asm-generic/mman-common.h | 3 ++ mm/gup.c | 58 ++++++++++++++++++++++++++++++ mm/internal.h | 3 ++ mm/madvise.c | 66 ++++++++++++++++++++++++++++++++++ 8 files changed, 142 insertions(+) (limited to 'include/uapi') diff --git a/arch/alpha/include/uapi/asm/mman.h b/arch/alpha/include/uapi/asm/mman.h index a18ec7f63888..56b4ee5a6c9e 100644 --- a/arch/alpha/include/uapi/asm/mman.h +++ b/arch/alpha/include/uapi/asm/mman.h @@ -71,6 +71,9 @@ #define MADV_COLD 20 /* deactivate these pages */ #define MADV_PAGEOUT 21 /* reclaim these pages */ +#define MADV_POPULATE_READ 22 /* populate (prefault) page tables readable */ +#define MADV_POPULATE_WRITE 23 /* populate (prefault) page tables writable */ + /* compatibility flags */ #define MAP_FILE 0 diff --git a/arch/mips/include/uapi/asm/mman.h b/arch/mips/include/uapi/asm/mman.h index 57dc2ac4f8bd..40b210c65a5a 100644 --- a/arch/mips/include/uapi/asm/mman.h +++ b/arch/mips/include/uapi/asm/mman.h @@ -98,6 +98,9 @@ #define MADV_COLD 20 /* deactivate these pages */ #define MADV_PAGEOUT 21 /* reclaim these pages */ +#define MADV_POPULATE_READ 22 /* populate (prefault) page tables readable */ +#define MADV_POPULATE_WRITE 23 /* populate (prefault) page tables writable */ + /* compatibility flags */ #define MAP_FILE 0 diff --git a/arch/parisc/include/uapi/asm/mman.h b/arch/parisc/include/uapi/asm/mman.h index ab78cba446ed..9e3c010c0f61 100644 --- a/arch/parisc/include/uapi/asm/mman.h +++ b/arch/parisc/include/uapi/asm/mman.h @@ -52,6 +52,9 @@ #define MADV_COLD 20 /* deactivate these pages */ #define MADV_PAGEOUT 21 /* reclaim these pages */ +#define MADV_POPULATE_READ 22 /* populate (prefault) page tables readable */ +#define MADV_POPULATE_WRITE 23 /* populate (prefault) page tables writable */ + #define MADV_MERGEABLE 65 /* KSM may merge identical pages */ #define MADV_UNMERGEABLE 66 /* KSM may not merge identical pages */ diff --git a/arch/xtensa/include/uapi/asm/mman.h b/arch/xtensa/include/uapi/asm/mman.h index e5e643752947..b3a22095371b 100644 --- a/arch/xtensa/include/uapi/asm/mman.h +++ b/arch/xtensa/include/uapi/asm/mman.h @@ -106,6 +106,9 @@ #define MADV_COLD 20 /* deactivate these pages */ #define MADV_PAGEOUT 21 /* reclaim these pages */ +#define MADV_POPULATE_READ 22 /* populate (prefault) page tables readable */ +#define MADV_POPULATE_WRITE 23 /* populate (prefault) page tables writable */ + /* compatibility flags */ #define MAP_FILE 0 diff --git a/include/uapi/asm-generic/mman-common.h b/include/uapi/asm-generic/mman-common.h index f94f65d429be..1567a3294c3d 100644 --- a/include/uapi/asm-generic/mman-common.h +++ b/include/uapi/asm-generic/mman-common.h @@ -72,6 +72,9 @@ #define MADV_COLD 20 /* deactivate these pages */ #define MADV_PAGEOUT 21 /* reclaim these pages */ +#define MADV_POPULATE_READ 22 /* populate (prefault) page tables readable */ +#define MADV_POPULATE_WRITE 23 /* populate (prefault) page tables writable */ + /* compatibility flags */ #define MAP_FILE 0 diff --git a/mm/gup.c b/mm/gup.c index 8651309f8ec3..728d996767cb 100644 --- a/mm/gup.c +++ b/mm/gup.c @@ -1500,6 +1500,64 @@ long populate_vma_page_range(struct vm_area_struct *vma, NULL, NULL, locked); } +/* + * faultin_vma_page_range() - populate (prefault) page tables inside the + * given VMA range readable/writable + * + * This takes care of mlocking the pages, too, if VM_LOCKED is set. + * + * @vma: target vma + * @start: start address + * @end: end address + * @write: whether to prefault readable or writable + * @locked: whether the mmap_lock is still held + * + * Returns either number of processed pages in the vma, or a negative error + * code on error (see __get_user_pages()). + * + * vma->vm_mm->mmap_lock must be held. The range must be page-aligned and + * covered by the VMA. + * + * If @locked is NULL, it may be held for read or write and will be unperturbed. + * + * If @locked is non-NULL, it must held for read only and may be released. If + * it's released, *@locked will be set to 0. + */ +long faultin_vma_page_range(struct vm_area_struct *vma, unsigned long start, + unsigned long end, bool write, int *locked) +{ + struct mm_struct *mm = vma->vm_mm; + unsigned long nr_pages = (end - start) / PAGE_SIZE; + int gup_flags; + + VM_BUG_ON(!PAGE_ALIGNED(start)); + VM_BUG_ON(!PAGE_ALIGNED(end)); + VM_BUG_ON_VMA(start < vma->vm_start, vma); + VM_BUG_ON_VMA(end > vma->vm_end, vma); + mmap_assert_locked(mm); + + /* + * FOLL_TOUCH: Mark page accessed and thereby young; will also mark + * the page dirty with FOLL_WRITE -- which doesn't make a + * difference with !FOLL_FORCE, because the page is writable + * in the page table. + * FOLL_HWPOISON: Return -EHWPOISON instead of -EFAULT when we hit + * a poisoned page. + * FOLL_POPULATE: Always populate memory with VM_LOCKONFAULT. + * !FOLL_FORCE: Require proper access permissions. + */ + gup_flags = FOLL_TOUCH | FOLL_POPULATE | FOLL_MLOCK | FOLL_HWPOISON; + if (write) + gup_flags |= FOLL_WRITE; + + /* + * See check_vma_flags(): Will return -EFAULT on incompatible mappings + * or with insufficient permissions. + */ + return __get_user_pages(mm, start, nr_pages, gup_flags, + NULL, NULL, locked); +} + /* * __mm_populate - populate and/or mlock pages within a range of address space. * diff --git a/mm/internal.h b/mm/internal.h index d9c2b33cd2ab..f38bc5d75ab7 100644 --- a/mm/internal.h +++ b/mm/internal.h @@ -345,6 +345,9 @@ void __vma_unlink_list(struct mm_struct *mm, struct vm_area_struct *vma); #ifdef CONFIG_MMU extern long populate_vma_page_range(struct vm_area_struct *vma, unsigned long start, unsigned long end, int *locked); +extern long faultin_vma_page_range(struct vm_area_struct *vma, + unsigned long start, unsigned long end, + bool write, int *locked); extern void munlock_vma_pages_range(struct vm_area_struct *vma, unsigned long start, unsigned long end); static inline void munlock_vma_pages_all(struct vm_area_struct *vma) diff --git a/mm/madvise.c b/mm/madvise.c index 63e489e5bfdb..6d3d348b17f4 100644 --- a/mm/madvise.c +++ b/mm/madvise.c @@ -53,6 +53,8 @@ static int madvise_need_mmap_write(int behavior) case MADV_COLD: case MADV_PAGEOUT: case MADV_FREE: + case MADV_POPULATE_READ: + case MADV_POPULATE_WRITE: return 0; default: /* be safe, default to 1. list exceptions explicitly */ @@ -822,6 +824,61 @@ static long madvise_dontneed_free(struct vm_area_struct *vma, return -EINVAL; } +static long madvise_populate(struct vm_area_struct *vma, + struct vm_area_struct **prev, + unsigned long start, unsigned long end, + int behavior) +{ + const bool write = behavior == MADV_POPULATE_WRITE; + struct mm_struct *mm = vma->vm_mm; + unsigned long tmp_end; + int locked = 1; + long pages; + + *prev = vma; + + while (start < end) { + /* + * We might have temporarily dropped the lock. For example, + * our VMA might have been split. + */ + if (!vma || start >= vma->vm_end) { + vma = find_vma(mm, start); + if (!vma || start < vma->vm_start) + return -ENOMEM; + } + + tmp_end = min_t(unsigned long, end, vma->vm_end); + /* Populate (prefault) page tables readable/writable. */ + pages = faultin_vma_page_range(vma, start, tmp_end, write, + &locked); + if (!locked) { + mmap_read_lock(mm); + locked = 1; + *prev = NULL; + vma = NULL; + } + if (pages < 0) { + switch (pages) { + case -EINTR: + return -EINTR; + case -EFAULT: /* Incompatible mappings / permissions. */ + return -EINVAL; + case -EHWPOISON: + return -EHWPOISON; + default: + pr_warn_once("%s: unhandled return value: %ld\n", + __func__, pages); + fallthrough; + case -ENOMEM: + return -ENOMEM; + } + } + start += pages * PAGE_SIZE; + } + return 0; +} + /* * Application wants to free up the pages and associated backing store. * This is effectively punching a hole into the middle of a file. @@ -935,6 +992,9 @@ madvise_vma(struct vm_area_struct *vma, struct vm_area_struct **prev, case MADV_FREE: case MADV_DONTNEED: return madvise_dontneed_free(vma, prev, start, end, behavior); + case MADV_POPULATE_READ: + case MADV_POPULATE_WRITE: + return madvise_populate(vma, prev, start, end, behavior); default: return madvise_behavior(vma, prev, start, end, behavior); } @@ -955,6 +1015,8 @@ madvise_behavior_valid(int behavior) case MADV_FREE: case MADV_COLD: case MADV_PAGEOUT: + case MADV_POPULATE_READ: + case MADV_POPULATE_WRITE: #ifdef CONFIG_KSM case MADV_MERGEABLE: case MADV_UNMERGEABLE: @@ -1042,6 +1104,10 @@ process_madvise_behavior_valid(int behavior) * easily if memory pressure happens. * MADV_PAGEOUT - the application is not expected to use this memory soon, * page out the pages in this range immediately. + * MADV_POPULATE_READ - populate (prefault) page tables readable by + * triggering read faults if required + * MADV_POPULATE_WRITE - populate (prefault) page tables writable by + * triggering write faults if required * * return values: * zero - success -- cgit v1.2.3 From d61914ea6adabde9126b0bed64a7a3a42249435e Mon Sep 17 00:00:00 2001 From: Zhu Lingshan Date: Mon, 10 May 2021 16:10:14 +0800 Subject: virtio: update virtio id table, add transitional ids This commit updates virtio id table by adding transitional device ids Signed-off-by: Zhu Lingshan Link: https://lore.kernel.org/r/20210510081015.4212-2-lingshan.zhu@intel.com Signed-off-by: Michael S. Tsirkin --- include/uapi/linux/virtio_ids.h | 12 ++++++++++++ 1 file changed, 12 insertions(+) (limited to 'include/uapi') diff --git a/include/uapi/linux/virtio_ids.h b/include/uapi/linux/virtio_ids.h index 4fe842c3a3a9..70a8057ad4bb 100644 --- a/include/uapi/linux/virtio_ids.h +++ b/include/uapi/linux/virtio_ids.h @@ -57,4 +57,16 @@ #define VIRTIO_ID_MAC80211_HWSIM 29 /* virtio mac80211-hwsim */ #define VIRTIO_ID_BT 40 /* virtio bluetooth */ +/* + * Virtio Transitional IDs + */ + +#define VIRTIO_TRANS_ID_NET 1000 /* transitional virtio net */ +#define VIRTIO_TRANS_ID_BLOCK 1001 /* transitional virtio block */ +#define VIRTIO_TRANS_ID_BALLOON 1002 /* transitional virtio balloon */ +#define VIRTIO_TRANS_ID_CONSOLE 1003 /* transitional virtio console */ +#define VIRTIO_TRANS_ID_SCSI 1004 /* transitional virtio SCSI */ +#define VIRTIO_TRANS_ID_RNG 1005 /* transitional virtio rng */ +#define VIRTIO_TRANS_ID_9P 1009 /* transitional virtio 9p console */ + #endif /* _LINUX_VIRTIO_IDS_H */ -- cgit v1.2.3 From 347269c113f10fbe893f11dd3ae5f44aa15d3111 Mon Sep 17 00:00:00 2001 From: Krzysztof Wilczyński Date: Sat, 3 Jul 2021 15:13:02 +0000 Subject: PCI: Fix kernel-doc formatting MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fix kernel-doc formatting throughout drivers/pci and related include files. No change to functionality intended. Check for warnings: $ find include drivers/pci -type f -path "*pci*.[ch]" | xargs scripts/kernel-doc -none [bhelgaas: squashed to one commit] Link: https://lore.kernel.org/r/20210509030237.368540-1-kw@linux.com Link: https://lore.kernel.org/r/20210703151306.1922450-1-kw@linux.com Link: https://lore.kernel.org/r/20210703151306.1922450-2-kw@linux.com Link: https://lore.kernel.org/r/20210703151306.1922450-3-kw@linux.com Link: https://lore.kernel.org/r/20210703151306.1922450-4-kw@linux.com Link: https://lore.kernel.org/r/20210703151306.1922450-5-kw@linux.com Signed-off-by: Krzysztof Wilczyński Signed-off-by: Bjorn Helgaas --- drivers/pci/controller/cadence/pcie-cadence.h | 7 +++++-- drivers/pci/controller/pci-xgene.c | 2 +- drivers/pci/controller/pcie-iproc-msi.c | 4 ++-- drivers/pci/controller/pcie-iproc.c | 24 +++++++++++------------- drivers/pci/controller/pcie-iproc.h | 16 +++++++++++----- drivers/pci/hotplug/cpqphp_core.c | 7 ++++--- drivers/pci/hotplug/cpqphp_ctrl.c | 2 +- drivers/pci/hotplug/pciehp.h | 3 +++ drivers/pci/pci.h | 4 ++-- include/linux/pci-ep-cfs.h | 2 +- include/linux/pci-epc.h | 5 ++++- include/linux/pci-epf.h | 5 ++++- include/linux/pci_hotplug.h | 2 ++ include/uapi/linux/pcitest.h | 2 +- 14 files changed, 52 insertions(+), 33 deletions(-) (limited to 'include/uapi') diff --git a/drivers/pci/controller/cadence/pcie-cadence.h b/drivers/pci/controller/cadence/pcie-cadence.h index 254d2570f8c9..30db2d68c17a 100644 --- a/drivers/pci/controller/cadence/pcie-cadence.h +++ b/drivers/pci/controller/cadence/pcie-cadence.h @@ -263,9 +263,12 @@ struct cdns_pcie_ops { * struct cdns_pcie - private data for Cadence PCIe controller drivers * @reg_base: IO mapped register base * @mem_res: start/end offsets in the physical system memory to map PCI accesses + * @dev: PCIe controller * @is_rc: tell whether the PCIe controller mode is Root Complex or Endpoint. - * @bus: In Root Complex mode, the bus number - * @ops: Platform specific ops to control various inputs from Cadence PCIe + * @phy_count: number of supported PHY devices + * @phy: list of pointers to specific PHY control blocks + * @link: list of pointers to corresponding device link representations + * @ops: Platform-specific ops to control various inputs from Cadence PCIe * wrapper */ struct cdns_pcie { diff --git a/drivers/pci/controller/pci-xgene.c b/drivers/pci/controller/pci-xgene.c index 7f503dd4ff81..8dc5140dd80d 100644 --- a/drivers/pci/controller/pci-xgene.c +++ b/drivers/pci/controller/pci-xgene.c @@ -1,5 +1,5 @@ // SPDX-License-Identifier: GPL-2.0+ -/** +/* * APM X-Gene PCIe Driver * * Copyright (c) 2014 Applied Micro Circuits Corporation. diff --git a/drivers/pci/controller/pcie-iproc-msi.c b/drivers/pci/controller/pcie-iproc-msi.c index eede4e8f3f75..21d28dc0b901 100644 --- a/drivers/pci/controller/pcie-iproc-msi.c +++ b/drivers/pci/controller/pcie-iproc-msi.c @@ -49,7 +49,7 @@ enum iproc_msi_reg { struct iproc_msi; /** - * iProc MSI group + * struct iproc_msi_grp - iProc MSI group * * One MSI group is allocated per GIC interrupt, serviced by one iProc MSI * event queue. @@ -65,7 +65,7 @@ struct iproc_msi_grp { }; /** - * iProc event queue based MSI + * struct iproc_msi - iProc event queue based MSI * * Only meant to be used on platforms without MSI support integrated into the * GIC. diff --git a/drivers/pci/controller/pcie-iproc.c b/drivers/pci/controller/pcie-iproc.c index 02e52f698eeb..30ac5fbefbbf 100644 --- a/drivers/pci/controller/pcie-iproc.c +++ b/drivers/pci/controller/pcie-iproc.c @@ -89,8 +89,8 @@ #define IPROC_PCIE_REG_INVALID 0xffff /** - * iProc PCIe outbound mapping controller specific parameters - * + * struct iproc_pcie_ob_map - iProc PCIe outbound mapping controller-specific + * parameters * @window_sizes: list of supported outbound mapping window sizes in MB * @nr_sizes: number of supported outbound mapping window sizes */ @@ -136,22 +136,20 @@ static const struct iproc_pcie_ob_map paxb_v2_ob_map[] = { }; /** - * iProc PCIe inbound mapping type + * enum iproc_pcie_ib_map_type - iProc PCIe inbound mapping type + * @IPROC_PCIE_IB_MAP_MEM: DDR memory + * @IPROC_PCIE_IB_MAP_IO: device I/O memory + * @IPROC_PCIE_IB_MAP_INVALID: invalid or unused */ enum iproc_pcie_ib_map_type { - /* for DDR memory */ IPROC_PCIE_IB_MAP_MEM = 0, - - /* for device I/O memory */ IPROC_PCIE_IB_MAP_IO, - - /* invalid or unused */ IPROC_PCIE_IB_MAP_INVALID }; /** - * iProc PCIe inbound mapping controller specific parameters - * + * struct iproc_pcie_ib_map - iProc PCIe inbound mapping controller-specific + * parameters * @type: inbound mapping region type * @size_unit: inbound mapping region size unit, could be SZ_1K, SZ_1M, or * SZ_1G @@ -437,7 +435,7 @@ static inline void iproc_pcie_write_reg(struct iproc_pcie *pcie, writel(val, pcie->base + offset); } -/** +/* * APB error forwarding can be disabled during access of configuration * registers of the endpoint device, to prevent unsupported requests * (typically seen during enumeration with multi-function devices) from @@ -619,7 +617,7 @@ static int iproc_pcie_config_read(struct pci_bus *bus, unsigned int devfn, return PCIBIOS_SUCCESSFUL; } -/** +/* * Note access to the configuration registers are protected at the higher layer * by 'pci_lock' in drivers/pci/access.c */ @@ -897,7 +895,7 @@ static inline int iproc_pcie_ob_write(struct iproc_pcie *pcie, int window_idx, return 0; } -/** +/* * Some iProc SoCs require the SW to configure the outbound address mapping * * Outbound address translation: diff --git a/drivers/pci/controller/pcie-iproc.h b/drivers/pci/controller/pcie-iproc.h index c2676e442f55..dcca315897c8 100644 --- a/drivers/pci/controller/pcie-iproc.h +++ b/drivers/pci/controller/pcie-iproc.h @@ -7,7 +7,13 @@ #define _PCIE_IPROC_H /** - * iProc PCIe interface type + * enum iproc_pcie_type - iProc PCIe interface type + * @IPROC_PCIE_PAXB_BCMA: BCMA-based host controllers + * @IPROC_PCIE_PAXB: PAXB-based host controllers for + * NS, NSP, Cygnus, NS2, and Pegasus SOCs + * @IPROC_PCIE_PAXB_V2: PAXB-based host controllers for Stingray SoCs + * @IPROC_PCIE_PAXC: PAXC-based host controllers + * @IPROC_PCIE_PAXC_V2: PAXC-based host controllers (second generation) * * PAXB is the wrapper used in root complex that can be connected to an * external endpoint device. @@ -24,7 +30,7 @@ enum iproc_pcie_type { }; /** - * iProc PCIe outbound mapping + * struct iproc_pcie_ob - iProc PCIe outbound mapping * @axi_offset: offset from the AXI address to the internal address used by * the iProc PCIe core * @nr_windows: total number of supported outbound mapping windows @@ -35,7 +41,7 @@ struct iproc_pcie_ob { }; /** - * iProc PCIe inbound mapping + * struct iproc_pcie_ib - iProc PCIe inbound mapping * @nr_regions: total number of supported inbound mapping regions */ struct iproc_pcie_ib { @@ -47,13 +53,13 @@ struct iproc_pcie_ib_map; struct iproc_msi; /** - * iProc PCIe device - * + * struct iproc_pcie - iProc PCIe device * @dev: pointer to device data structure * @type: iProc PCIe interface type * @reg_offsets: register offsets * @base: PCIe host controller I/O register base * @base_addr: PCIe host controller register base physical address + * @mem: host bridge memory window resource * @phy: optional PHY device that controls the Serdes * @map_irq: function callback to map interrupts * @ep_is_internal: indicates an internal emulated endpoint device is connected diff --git a/drivers/pci/hotplug/cpqphp_core.c b/drivers/pci/hotplug/cpqphp_core.c index b8aacb41a83c..f99a7927e5a8 100644 --- a/drivers/pci/hotplug/cpqphp_core.c +++ b/drivers/pci/hotplug/cpqphp_core.c @@ -296,9 +296,10 @@ static int ctrl_slot_cleanup(struct controller *ctrl) * * Won't work for more than one PCI-PCI bridge in a slot. * - * @bus_num - bus number of PCI device - * @dev_num - device number of PCI device - * @slot - Pointer to u8 where slot number will be returned + * @bus: pointer to the PCI bus structure + * @bus_num: bus number of PCI device + * @dev_num: device number of PCI device + * @slot: Pointer to u8 where slot number will be returned * * Output: SUCCESS or FAILURE */ diff --git a/drivers/pci/hotplug/cpqphp_ctrl.c b/drivers/pci/hotplug/cpqphp_ctrl.c index 68de958a9be8..1b26ca0b3701 100644 --- a/drivers/pci/hotplug/cpqphp_ctrl.c +++ b/drivers/pci/hotplug/cpqphp_ctrl.c @@ -1877,7 +1877,7 @@ static void interrupt_event_handler(struct controller *ctrl) /** * cpqhp_pushbutton_thread - handle pushbutton events - * @slot: target slot (struct) + * @t: pointer to struct timer_list which holds all timer-related callbacks * * Scheduled procedure to handle blocking stuff for the pushbuttons. * Handles all pending events and exits. diff --git a/drivers/pci/hotplug/pciehp.h b/drivers/pci/hotplug/pciehp.h index 4fd200d8b0a9..d4a930881054 100644 --- a/drivers/pci/hotplug/pciehp.h +++ b/drivers/pci/hotplug/pciehp.h @@ -47,6 +47,9 @@ extern int pciehp_poll_time; * struct controller - PCIe hotplug controller * @pcie: pointer to the controller's PCIe port service device * @slot_cap: cached copy of the Slot Capabilities register + * @inband_presence_disabled: In-Band Presence Detect Disable supported by + * controller and disabled per spec recommendation (PCIe r5.0, appendix I + * implementation note) * @slot_ctrl: cached copy of the Slot Control register * @ctrl_lock: serializes writes to the Slot Control register * @cmd_started: jiffies when the Slot Control register was last written; diff --git a/drivers/pci/pci.h b/drivers/pci/pci.h index 37c913bbc6e1..8b385eaf2043 100644 --- a/drivers/pci/pci.h +++ b/drivers/pci/pci.h @@ -324,8 +324,8 @@ struct pci_sriov { /** * pci_dev_set_io_state - Set the new error state if possible. * - * @dev - pci device to set new error_state - * @new - the state we want dev to be in + * @dev: PCI device to set new error_state + * @new: the state we want dev to be in * * Must be called with device_lock held. * diff --git a/include/linux/pci-ep-cfs.h b/include/linux/pci-ep-cfs.h index 662881335c7e..3e2140d7e31d 100644 --- a/include/linux/pci-ep-cfs.h +++ b/include/linux/pci-ep-cfs.h @@ -1,5 +1,5 @@ /* SPDX-License-Identifier: GPL-2.0+ */ -/** +/* * PCI Endpoint ConfigFS header file * * Copyright (C) 2017 Texas Instruments diff --git a/include/linux/pci-epc.h b/include/linux/pci-epc.h index b82c9b100e97..50a649d33e68 100644 --- a/include/linux/pci-epc.h +++ b/include/linux/pci-epc.h @@ -1,5 +1,5 @@ /* SPDX-License-Identifier: GPL-2.0 */ -/** +/* * PCI Endpoint *Controller* (EPC) header file * * Copyright (C) 2017 Texas Instruments @@ -58,6 +58,7 @@ pci_epc_interface_string(enum pci_epc_interface_type type) * @map_msi_irq: ops to map physical address to MSI address and return MSI data * @start: ops to start the PCI link * @stop: ops to stop the PCI link + * @get_features: ops to get the features supported by the EPC * @owner: the module owner containing the ops */ struct pci_epc_ops { @@ -150,6 +151,8 @@ struct pci_epc { /** * struct pci_epc_features - features supported by a EPC device per function * @linkup_notifier: indicate if the EPC device can notify EPF driver on link up + * @core_init_notifier: indicate cores that can notify about their availability + * for initialization * @msi_capable: indicate if the endpoint function has MSI capability * @msix_capable: indicate if the endpoint function has MSI-X capability * @reserved_bar: bitmap to indicate reserved BAR unavailable to function driver diff --git a/include/linux/pci-epf.h b/include/linux/pci-epf.h index 6833e2160ef1..2debc27ba95e 100644 --- a/include/linux/pci-epf.h +++ b/include/linux/pci-epf.h @@ -1,5 +1,5 @@ /* SPDX-License-Identifier: GPL-2.0 */ -/** +/* * PCI Endpoint *Function* (EPF) header file * * Copyright (C) 2017 Texas Instruments @@ -102,6 +102,8 @@ struct pci_epf_driver { * @phys_addr: physical address that should be mapped to the BAR * @addr: virtual address corresponding to the @phys_addr * @size: the size of the address space present in BAR + * @barno: BAR number + * @flags: flags that are set for the BAR */ struct pci_epf_bar { dma_addr_t phys_addr; @@ -118,6 +120,7 @@ struct pci_epf_bar { * @header: represents standard configuration header * @bar: represents the BAR of EPF device * @msi_interrupts: number of MSI interrupts required by this function + * @msix_interrupts: number of MSI-X interrupts required by this function * @func_no: unique function number within this endpoint device * @epc: the EPC device to which this EPF device is bound * @driver: the EPF driver to which this EPF device is bound diff --git a/include/linux/pci_hotplug.h b/include/linux/pci_hotplug.h index b482e42d7153..2dac431d94ac 100644 --- a/include/linux/pci_hotplug.h +++ b/include/linux/pci_hotplug.h @@ -50,6 +50,8 @@ struct hotplug_slot_ops { /** * struct hotplug_slot - used to register a physical slot with the hotplug pci core * @ops: pointer to the &struct hotplug_slot_ops to be used for this slot + * @slot_list: internal list used to track hotplug PCI slots + * @pci_slot: represents a physical slot * @owner: The module owner of this structure * @mod_name: The module name (KBUILD_MODNAME) of this structure */ diff --git a/include/uapi/linux/pcitest.h b/include/uapi/linux/pcitest.h index c3ab4c826297..f9c1af8d141b 100644 --- a/include/uapi/linux/pcitest.h +++ b/include/uapi/linux/pcitest.h @@ -1,5 +1,5 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -/** +/* * pcitest.h - PCI test uapi defines * * Copyright (C) 2017 Texas Instruments -- cgit v1.2.3 From 1507f51255c9ff07d75909a84e7c0d7f3c4b2f49 Mon Sep 17 00:00:00 2001 From: Mike Rapoport Date: Wed, 7 Jul 2021 18:08:03 -0700 Subject: mm: introduce memfd_secret system call to create "secret" memory areas Introduce "memfd_secret" system call with the ability to create memory areas visible only in the context of the owning process and not mapped not only to other processes but in the kernel page tables as well. The secretmem feature is off by default and the user must explicitly enable it at the boot time. Once secretmem is enabled, the user will be able to create a file descriptor using the memfd_secret() system call. The memory areas created by mmap() calls from this file descriptor will be unmapped from the kernel direct map and they will be only mapped in the page table of the processes that have access to the file descriptor. Secretmem is designed to provide the following protections: * Enhanced protection (in conjunction with all the other in-kernel attack prevention systems) against ROP attacks. Seceretmem makes "simple" ROP insufficient to perform exfiltration, which increases the required complexity of the attack. Along with other protections like the kernel stack size limit and address space layout randomization which make finding gadgets is really hard, absence of any in-kernel primitive for accessing secret memory means the one gadget ROP attack can't work. Since the only way to access secret memory is to reconstruct the missing mapping entry, the attacker has to recover the physical page and insert a PTE pointing to it in the kernel and then retrieve the contents. That takes at least three gadgets which is a level of difficulty beyond most standard attacks. * Prevent cross-process secret userspace memory exposures. Once the secret memory is allocated, the user can't accidentally pass it into the kernel to be transmitted somewhere. The secreremem pages cannot be accessed via the direct map and they are disallowed in GUP. * Harden against exploited kernel flaws. In order to access secretmem, a kernel-side attack would need to either walk the page tables and create new ones, or spawn a new privileged uiserspace process to perform secrets exfiltration using ptrace. The file descriptor based memory has several advantages over the "traditional" mm interfaces, such as mlock(), mprotect(), madvise(). File descriptor approach allows explicit and controlled sharing of the memory areas, it allows to seal the operations. Besides, file descriptor based memory paves the way for VMMs to remove the secret memory range from the userspace hipervisor process, for instance QEMU. Andy Lutomirski says: "Getting fd-backed memory into a guest will take some possibly major work in the kernel, but getting vma-backed memory into a guest without mapping it in the host user address space seems much, much worse." memfd_secret() is made a dedicated system call rather than an extension to memfd_create() because it's purpose is to allow the user to create more secure memory mappings rather than to simply allow file based access to the memory. Nowadays a new system call cost is negligible while it is way simpler for userspace to deal with a clear-cut system calls than with a multiplexer or an overloaded syscall. Moreover, the initial implementation of memfd_secret() is completely distinct from memfd_create() so there is no much sense in overloading memfd_create() to begin with. If there will be a need for code sharing between these implementation it can be easily achieved without a need to adjust user visible APIs. The secret memory remains accessible in the process context using uaccess primitives, but it is not exposed to the kernel otherwise; secret memory areas are removed from the direct map and functions in the follow_page()/get_user_page() family will refuse to return a page that belongs to the secret memory area. Once there will be a use case that will require exposing secretmem to the kernel it will be an opt-in request in the system call flags so that user would have to decide what data can be exposed to the kernel. Removing of the pages from the direct map may cause its fragmentation on architectures that use large pages to map the physical memory which affects the system performance. However, the original Kconfig text for CONFIG_DIRECT_GBPAGES said that gigabyte pages in the direct map "... can improve the kernel's performance a tiny bit ..." (commit 00d1c5e05736 ("x86: add gbpages switches")) and the recent report [1] showed that "... although 1G mappings are a good default choice, there is no compelling evidence that it must be the only choice". Hence, it is sufficient to have secretmem disabled by default with the ability of a system administrator to enable it at boot time. Pages in the secretmem regions are unevictable and unmovable to avoid accidental exposure of the sensitive data via swap or during page migration. Since the secretmem mappings are locked in memory they cannot exceed RLIMIT_MEMLOCK. Since these mappings are already locked independently from mlock(), an attempt to mlock()/munlock() secretmem range would fail and mlockall()/munlockall() will ignore secretmem mappings. However, unlike mlock()ed memory, secretmem currently behaves more like long-term GUP: secretmem mappings are unmovable mappings directly consumed by user space. With default limits, there is no excessive use of secretmem and it poses no real problem in combination with ZONE_MOVABLE/CMA, but in the future this should be addressed to allow balanced use of large amounts of secretmem along with ZONE_MOVABLE/CMA. A page that was a part of the secret memory area is cleared when it is freed to ensure the data is not exposed to the next user of that page. The following example demonstrates creation of a secret mapping (error handling is omitted): fd = memfd_secret(0); ftruncate(fd, MAP_SIZE); ptr = mmap(NULL, MAP_SIZE, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); [1] https://lore.kernel.org/linux-mm/213b4567-46ce-f116-9cdf-bbd0c884eb3c@linux.intel.com/ [akpm@linux-foundation.org: suppress Kconfig whine] Link: https://lkml.kernel.org/r/20210518072034.31572-5-rppt@kernel.org Signed-off-by: Mike Rapoport Acked-by: Hagen Paul Pfeifer Acked-by: James Bottomley Cc: Alexander Viro Cc: Andy Lutomirski Cc: Arnd Bergmann Cc: Borislav Petkov Cc: Catalin Marinas Cc: Christopher Lameter Cc: Dan Williams Cc: Dave Hansen Cc: Elena Reshetova Cc: "H. Peter Anvin" Cc: Ingo Molnar Cc: James Bottomley Cc: "Kirill A. Shutemov" Cc: Matthew Wilcox Cc: Mark Rutland Cc: Michael Kerrisk Cc: Palmer Dabbelt Cc: Palmer Dabbelt Cc: Paul Walmsley Cc: Peter Zijlstra Cc: Rick Edgecombe Cc: Roman Gushchin Cc: Shakeel Butt Cc: Shuah Khan Cc: Thomas Gleixner Cc: Tycho Andersen Cc: Will Deacon Cc: David Hildenbrand Cc: kernel test robot Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- include/linux/secretmem.h | 48 +++++++++ include/uapi/linux/magic.h | 1 + kernel/sys_ni.c | 2 + mm/Kconfig | 4 + mm/Makefile | 1 + mm/gup.c | 12 +++ mm/mlock.c | 3 +- mm/secretmem.c | 239 +++++++++++++++++++++++++++++++++++++++++++++ 8 files changed, 309 insertions(+), 1 deletion(-) create mode 100644 include/linux/secretmem.h create mode 100644 mm/secretmem.c (limited to 'include/uapi') diff --git a/include/linux/secretmem.h b/include/linux/secretmem.h new file mode 100644 index 000000000000..e617b4afcc62 --- /dev/null +++ b/include/linux/secretmem.h @@ -0,0 +1,48 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +#ifndef _LINUX_SECRETMEM_H +#define _LINUX_SECRETMEM_H + +#ifdef CONFIG_SECRETMEM + +extern const struct address_space_operations secretmem_aops; + +static inline bool page_is_secretmem(struct page *page) +{ + struct address_space *mapping; + + /* + * Using page_mapping() is quite slow because of the actual call + * instruction and repeated compound_head(page) inside the + * page_mapping() function. + * We know that secretmem pages are not compound and LRU so we can + * save a couple of cycles here. + */ + if (PageCompound(page) || !PageLRU(page)) + return false; + + mapping = (struct address_space *) + ((unsigned long)page->mapping & ~PAGE_MAPPING_FLAGS); + + if (mapping != page->mapping) + return false; + + return mapping->a_ops == &secretmem_aops; +} + +bool vma_is_secretmem(struct vm_area_struct *vma); + +#else + +static inline bool vma_is_secretmem(struct vm_area_struct *vma) +{ + return false; +} + +static inline bool page_is_secretmem(struct page *page) +{ + return false; +} + +#endif /* CONFIG_SECRETMEM */ + +#endif /* _LINUX_SECRETMEM_H */ diff --git a/include/uapi/linux/magic.h b/include/uapi/linux/magic.h index f3956fc11de6..35687dcb1a42 100644 --- a/include/uapi/linux/magic.h +++ b/include/uapi/linux/magic.h @@ -97,5 +97,6 @@ #define DEVMEM_MAGIC 0x454d444d /* "DMEM" */ #define Z3FOLD_MAGIC 0x33 #define PPC_CMM_MAGIC 0xc7571590 +#define SECRETMEM_MAGIC 0x5345434d /* "SECM" */ #endif /* __LINUX_MAGIC_H__ */ diff --git a/kernel/sys_ni.c b/kernel/sys_ni.c index dad4d994641e..30971b1dd4a9 100644 --- a/kernel/sys_ni.c +++ b/kernel/sys_ni.c @@ -358,6 +358,8 @@ COND_SYSCALL(pkey_mprotect); COND_SYSCALL(pkey_alloc); COND_SYSCALL(pkey_free); +/* memfd_secret */ +COND_SYSCALL(memfd_secret); /* * Architecture specific weak syscall entries. diff --git a/mm/Kconfig b/mm/Kconfig index a02498c0e13d..40a9bfcd5062 100644 --- a/mm/Kconfig +++ b/mm/Kconfig @@ -885,4 +885,8 @@ config KMAP_LOCAL # struct io_mapping based helper. Selected by drivers that need them config IO_MAPPING bool + +config SECRETMEM + def_bool ARCH_HAS_SET_DIRECT_MAP && !EMBEDDED + endmenu diff --git a/mm/Makefile b/mm/Makefile index 74b47c354682..e3436741d539 100644 --- a/mm/Makefile +++ b/mm/Makefile @@ -113,6 +113,7 @@ obj-$(CONFIG_CMA) += cma.o obj-$(CONFIG_MEMORY_BALLOON) += balloon_compaction.o obj-$(CONFIG_PAGE_EXTENSION) += page_ext.o obj-$(CONFIG_CMA_DEBUGFS) += cma_debug.o +obj-$(CONFIG_SECRETMEM) += secretmem.o obj-$(CONFIG_CMA_SYSFS) += cma_sysfs.o obj-$(CONFIG_USERFAULTFD) += userfaultfd.o obj-$(CONFIG_IDLE_PAGE_TRACKING) += page_idle.o diff --git a/mm/gup.c b/mm/gup.c index 728d996767cb..42b8b1fa6521 100644 --- a/mm/gup.c +++ b/mm/gup.c @@ -10,6 +10,7 @@ #include #include #include +#include #include #include @@ -855,6 +856,9 @@ struct page *follow_page(struct vm_area_struct *vma, unsigned long address, struct follow_page_context ctx = { NULL }; struct page *page; + if (vma_is_secretmem(vma)) + return NULL; + page = follow_page_mask(vma, address, foll_flags, &ctx); if (ctx.pgmap) put_dev_pagemap(ctx.pgmap); @@ -988,6 +992,9 @@ static int check_vma_flags(struct vm_area_struct *vma, unsigned long gup_flags) if ((gup_flags & FOLL_LONGTERM) && vma_is_fsdax(vma)) return -EOPNOTSUPP; + if (vma_is_secretmem(vma)) + return -EFAULT; + if (write) { if (!(vm_flags & VM_WRITE)) { if (!(gup_flags & FOLL_FORCE)) @@ -2170,6 +2177,11 @@ static int gup_pte_range(pmd_t pmd, unsigned long addr, unsigned long end, if (!head) goto pte_unmap; + if (unlikely(page_is_secretmem(page))) { + put_compound_head(head, 1, flags); + goto pte_unmap; + } + if (unlikely(pte_val(pte) != pte_val(*ptep))) { put_compound_head(head, 1, flags); goto pte_unmap; diff --git a/mm/mlock.c b/mm/mlock.c index 0d639bf48794..16d2ee160d43 100644 --- a/mm/mlock.c +++ b/mm/mlock.c @@ -23,6 +23,7 @@ #include #include #include +#include #include "internal.h" @@ -503,7 +504,7 @@ static int mlock_fixup(struct vm_area_struct *vma, struct vm_area_struct **prev, if (newflags == vma->vm_flags || (vma->vm_flags & VM_SPECIAL) || is_vm_hugetlb_page(vma) || vma == get_gate_vma(current->mm) || - vma_is_dax(vma)) + vma_is_dax(vma) || vma_is_secretmem(vma)) /* don't set VM_LOCKED or VM_LOCKONFAULT and don't count */ goto out; diff --git a/mm/secretmem.c b/mm/secretmem.c new file mode 100644 index 000000000000..972cd1bbc3cc --- /dev/null +++ b/mm/secretmem.c @@ -0,0 +1,239 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright IBM Corporation, 2021 + * + * Author: Mike Rapoport + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include + +#include "internal.h" + +#undef pr_fmt +#define pr_fmt(fmt) "secretmem: " fmt + +/* + * Define mode and flag masks to allow validation of the system call + * parameters. + */ +#define SECRETMEM_MODE_MASK (0x0) +#define SECRETMEM_FLAGS_MASK SECRETMEM_MODE_MASK + +static bool secretmem_enable __ro_after_init; +module_param_named(enable, secretmem_enable, bool, 0400); +MODULE_PARM_DESC(secretmem_enable, + "Enable secretmem and memfd_secret(2) system call"); + +static vm_fault_t secretmem_fault(struct vm_fault *vmf) +{ + struct address_space *mapping = vmf->vma->vm_file->f_mapping; + struct inode *inode = file_inode(vmf->vma->vm_file); + pgoff_t offset = vmf->pgoff; + gfp_t gfp = vmf->gfp_mask; + unsigned long addr; + struct page *page; + int err; + + if (((loff_t)vmf->pgoff << PAGE_SHIFT) >= i_size_read(inode)) + return vmf_error(-EINVAL); + +retry: + page = find_lock_page(mapping, offset); + if (!page) { + page = alloc_page(gfp | __GFP_ZERO); + if (!page) + return VM_FAULT_OOM; + + err = set_direct_map_invalid_noflush(page); + if (err) { + put_page(page); + return vmf_error(err); + } + + __SetPageUptodate(page); + err = add_to_page_cache_lru(page, mapping, offset, gfp); + if (unlikely(err)) { + put_page(page); + /* + * If a split of large page was required, it + * already happened when we marked the page invalid + * which guarantees that this call won't fail + */ + set_direct_map_default_noflush(page); + if (err == -EEXIST) + goto retry; + + return vmf_error(err); + } + + addr = (unsigned long)page_address(page); + flush_tlb_kernel_range(addr, addr + PAGE_SIZE); + } + + vmf->page = page; + return VM_FAULT_LOCKED; +} + +static const struct vm_operations_struct secretmem_vm_ops = { + .fault = secretmem_fault, +}; + +static int secretmem_mmap(struct file *file, struct vm_area_struct *vma) +{ + unsigned long len = vma->vm_end - vma->vm_start; + + if ((vma->vm_flags & (VM_SHARED | VM_MAYSHARE)) == 0) + return -EINVAL; + + if (mlock_future_check(vma->vm_mm, vma->vm_flags | VM_LOCKED, len)) + return -EAGAIN; + + vma->vm_flags |= VM_LOCKED | VM_DONTDUMP; + vma->vm_ops = &secretmem_vm_ops; + + return 0; +} + +bool vma_is_secretmem(struct vm_area_struct *vma) +{ + return vma->vm_ops == &secretmem_vm_ops; +} + +static const struct file_operations secretmem_fops = { + .mmap = secretmem_mmap, +}; + +static bool secretmem_isolate_page(struct page *page, isolate_mode_t mode) +{ + return false; +} + +static int secretmem_migratepage(struct address_space *mapping, + struct page *newpage, struct page *page, + enum migrate_mode mode) +{ + return -EBUSY; +} + +static void secretmem_freepage(struct page *page) +{ + set_direct_map_default_noflush(page); + clear_highpage(page); +} + +const struct address_space_operations secretmem_aops = { + .freepage = secretmem_freepage, + .migratepage = secretmem_migratepage, + .isolate_page = secretmem_isolate_page, +}; + +static struct vfsmount *secretmem_mnt; + +static struct file *secretmem_file_create(unsigned long flags) +{ + struct file *file = ERR_PTR(-ENOMEM); + struct inode *inode; + + inode = alloc_anon_inode(secretmem_mnt->mnt_sb); + if (IS_ERR(inode)) + return ERR_CAST(inode); + + file = alloc_file_pseudo(inode, secretmem_mnt, "secretmem", + O_RDWR, &secretmem_fops); + if (IS_ERR(file)) + goto err_free_inode; + + mapping_set_gfp_mask(inode->i_mapping, GFP_HIGHUSER); + mapping_set_unevictable(inode->i_mapping); + + inode->i_mapping->a_ops = &secretmem_aops; + + /* pretend we are a normal file with zero size */ + inode->i_mode |= S_IFREG; + inode->i_size = 0; + + return file; + +err_free_inode: + iput(inode); + return file; +} + +SYSCALL_DEFINE1(memfd_secret, unsigned int, flags) +{ + struct file *file; + int fd, err; + + /* make sure local flags do not confict with global fcntl.h */ + BUILD_BUG_ON(SECRETMEM_FLAGS_MASK & O_CLOEXEC); + + if (!secretmem_enable) + return -ENOSYS; + + if (flags & ~(SECRETMEM_FLAGS_MASK | O_CLOEXEC)) + return -EINVAL; + + fd = get_unused_fd_flags(flags & O_CLOEXEC); + if (fd < 0) + return fd; + + file = secretmem_file_create(flags); + if (IS_ERR(file)) { + err = PTR_ERR(file); + goto err_put_fd; + } + + file->f_flags |= O_LARGEFILE; + + fd_install(fd, file); + return fd; + +err_put_fd: + put_unused_fd(fd); + return err; +} + +static int secretmem_init_fs_context(struct fs_context *fc) +{ + return init_pseudo(fc, SECRETMEM_MAGIC) ? 0 : -ENOMEM; +} + +static struct file_system_type secretmem_fs = { + .name = "secretmem", + .init_fs_context = secretmem_init_fs_context, + .kill_sb = kill_anon_super, +}; + +static int secretmem_init(void) +{ + int ret = 0; + + if (!secretmem_enable) + return ret; + + secretmem_mnt = kern_mount(&secretmem_fs); + if (IS_ERR(secretmem_mnt)) + ret = PTR_ERR(secretmem_mnt); + + /* prevent secretmem mappings from ever getting PROT_EXEC */ + secretmem_mnt->mnt_flags |= MNT_NOEXEC; + + return ret; +} +fs_initcall(secretmem_init); -- cgit v1.2.3 From 7bb7f2ac24a028b20fca466b9633847b289b156a Mon Sep 17 00:00:00 2001 From: Mike Rapoport Date: Wed, 7 Jul 2021 18:08:11 -0700 Subject: arch, mm: wire up memfd_secret system call where relevant Wire up memfd_secret system call on architectures that define ARCH_HAS_SET_DIRECT_MAP, namely arm64, risc-v and x86. Link: https://lkml.kernel.org/r/20210518072034.31572-7-rppt@kernel.org Signed-off-by: Mike Rapoport Acked-by: Palmer Dabbelt Acked-by: Arnd Bergmann Acked-by: Catalin Marinas Acked-by: David Hildenbrand Acked-by: James Bottomley Cc: Alexander Viro Cc: Andy Lutomirski Cc: Borislav Petkov Cc: Christopher Lameter Cc: Dan Williams Cc: Dave Hansen Cc: David Hildenbrand Cc: Elena Reshetova Cc: Hagen Paul Pfeifer Cc: "H. Peter Anvin" Cc: Ingo Molnar Cc: James Bottomley Cc: "Kirill A. Shutemov" Cc: Mark Rutland Cc: Matthew Wilcox Cc: Michael Kerrisk Cc: Palmer Dabbelt Cc: Paul Walmsley Cc: Peter Zijlstra Cc: Rick Edgecombe Cc: Roman Gushchin Cc: Shakeel Butt Cc: Shuah Khan Cc: Thomas Gleixner Cc: Tycho Andersen Cc: Will Deacon Cc: kernel test robot Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- arch/arm64/include/uapi/asm/unistd.h | 1 + arch/riscv/include/asm/unistd.h | 1 + arch/x86/entry/syscalls/syscall_32.tbl | 1 + arch/x86/entry/syscalls/syscall_64.tbl | 1 + include/linux/syscalls.h | 1 + include/uapi/asm-generic/unistd.h | 7 ++++++- scripts/checksyscalls.sh | 4 ++++ 7 files changed, 15 insertions(+), 1 deletion(-) (limited to 'include/uapi') diff --git a/arch/arm64/include/uapi/asm/unistd.h b/arch/arm64/include/uapi/asm/unistd.h index f83a70e07df8..ce2ee8f1e361 100644 --- a/arch/arm64/include/uapi/asm/unistd.h +++ b/arch/arm64/include/uapi/asm/unistd.h @@ -20,5 +20,6 @@ #define __ARCH_WANT_SET_GET_RLIMIT #define __ARCH_WANT_TIME32_SYSCALLS #define __ARCH_WANT_SYS_CLONE3 +#define __ARCH_WANT_MEMFD_SECRET #include diff --git a/arch/riscv/include/asm/unistd.h b/arch/riscv/include/asm/unistd.h index 977ee6181dab..6c316093a1e5 100644 --- a/arch/riscv/include/asm/unistd.h +++ b/arch/riscv/include/asm/unistd.h @@ -9,6 +9,7 @@ */ #define __ARCH_WANT_SYS_CLONE +#define __ARCH_WANT_MEMFD_SECRET #include diff --git a/arch/x86/entry/syscalls/syscall_32.tbl b/arch/x86/entry/syscalls/syscall_32.tbl index fba2f615119a..ce763a12311c 100644 --- a/arch/x86/entry/syscalls/syscall_32.tbl +++ b/arch/x86/entry/syscalls/syscall_32.tbl @@ -451,3 +451,4 @@ 444 i386 landlock_create_ruleset sys_landlock_create_ruleset 445 i386 landlock_add_rule sys_landlock_add_rule 446 i386 landlock_restrict_self sys_landlock_restrict_self +447 i386 memfd_secret sys_memfd_secret diff --git a/arch/x86/entry/syscalls/syscall_64.tbl b/arch/x86/entry/syscalls/syscall_64.tbl index af973e400053..f6b57799c1ea 100644 --- a/arch/x86/entry/syscalls/syscall_64.tbl +++ b/arch/x86/entry/syscalls/syscall_64.tbl @@ -368,6 +368,7 @@ 444 common landlock_create_ruleset sys_landlock_create_ruleset 445 common landlock_add_rule sys_landlock_add_rule 446 common landlock_restrict_self sys_landlock_restrict_self +447 common memfd_secret sys_memfd_secret # # Due to a historical design error, certain syscalls are numbered differently diff --git a/include/linux/syscalls.h b/include/linux/syscalls.h index 586128d5c3b8..69c9a7010081 100644 --- a/include/linux/syscalls.h +++ b/include/linux/syscalls.h @@ -1050,6 +1050,7 @@ asmlinkage long sys_landlock_create_ruleset(const struct landlock_ruleset_attr _ asmlinkage long sys_landlock_add_rule(int ruleset_fd, enum landlock_rule_type rule_type, const void __user *rule_attr, __u32 flags); asmlinkage long sys_landlock_restrict_self(int ruleset_fd, __u32 flags); +asmlinkage long sys_memfd_secret(unsigned int flags); /* * Architecture-specific system calls diff --git a/include/uapi/asm-generic/unistd.h b/include/uapi/asm-generic/unistd.h index f211961ce1da..a9d6fcd95f42 100644 --- a/include/uapi/asm-generic/unistd.h +++ b/include/uapi/asm-generic/unistd.h @@ -873,8 +873,13 @@ __SYSCALL(__NR_landlock_add_rule, sys_landlock_add_rule) #define __NR_landlock_restrict_self 446 __SYSCALL(__NR_landlock_restrict_self, sys_landlock_restrict_self) +#ifdef __ARCH_WANT_MEMFD_SECRET +#define __NR_memfd_secret 447 +__SYSCALL(__NR_memfd_secret, sys_memfd_secret) +#endif + #undef __NR_syscalls -#define __NR_syscalls 447 +#define __NR_syscalls 448 /* * 32 bit systems traditionally used different diff --git a/scripts/checksyscalls.sh b/scripts/checksyscalls.sh index a18b47695f55..b7609958ee36 100755 --- a/scripts/checksyscalls.sh +++ b/scripts/checksyscalls.sh @@ -40,6 +40,10 @@ cat << EOF #define __IGNORE_setrlimit /* setrlimit */ #endif +#ifndef __ARCH_WANT_MEMFD_SECRET +#define __IGNORE_memfd_secret +#endif + /* Missing flags argument */ #define __IGNORE_renameat /* renameat2 */ -- cgit v1.2.3