diff options
| -rw-r--r-- | drivers/misc/fastrpc.c | 194 | ||||
| -rw-r--r-- | include/uapi/misc/fastrpc.h | 29 |
2 files changed, 215 insertions, 8 deletions
diff --git a/drivers/misc/fastrpc.c b/drivers/misc/fastrpc.c index d42b3e9e2a07..93aa6556bd47 100644 --- a/drivers/misc/fastrpc.c +++ b/drivers/misc/fastrpc.c @@ -25,6 +25,8 @@ #include <linux/bitfield.h> #include <linux/bits.h> #include <linux/bitops.h> +#include <linux/compiler.h> +#include <linux/iopoll.h> #define ADSP_DOMAIN_ID (0) #define MDSP_DOMAIN_ID (1) @@ -39,7 +41,16 @@ #define FASTRPC_CTX_MAX (256) #define FASTRPC_INIT_HANDLE 1 #define FASTRPC_DSP_UTILITIES_HANDLE 2 +/* + * Maximum handle value for static handles. + * Static handles are pre-defined, fixed numeric values statically assigned + * in the IDL file or FastRPC framework. + */ +#define FASTRPC_MAX_STATIC_HANDLE (20) #define FASTRPC_CTXID_MASK GENMASK(15, 8) +/* Sequence number occupies bits 63:16 of the ctxid / message context */ +#define FASTRPC_CTXID_SEQ_SHIFT 16 +#define FASTRPC_CTXID_SEQ_MASK GENMASK_ULL(63, 16) #define INIT_FILELEN_MAX (2 * 1024 * 1024) #define INIT_FILE_NAMELEN_MAX (128) #define FASTRPC_DEVICE_NAME "fastrpc" @@ -107,6 +118,12 @@ #define miscdev_to_fdevice(d) container_of(d, struct fastrpc_device, miscdev) +/* Poll response number from remote processor for call completion */ +#define FASTRPC_POLL_RESPONSE (0xdecaf) + +/* Polling mode timeout limit */ +#define FASTRPC_POLL_MAX_TIMEOUT_US (10000) + struct fastrpc_phy_page { dma_addr_t addr; /* dma address */ u64 size; /* size of contiguous region */ @@ -237,8 +254,14 @@ struct fastrpc_invoke_ctx { u32 sc; u64 *fdlist; u32 *crc; + /* Poll memory that DSP updates */ + u32 *poll_addr; u64 ctxid; u64 msg_sz; + /* work done status flag */ + bool is_work_done; + /* process updates poll memory instead of glink response */ + bool is_polled; struct kref refcount; struct list_head node; /* list of ctxs */ struct completion work; @@ -264,6 +287,7 @@ struct fastrpc_soc_data { u32 sid_pos; u32 dma_addr_bits_cdsp; u32 dma_addr_bits_default; + bool poll_mode_supported; }; struct fastrpc_channel_ctx { @@ -286,6 +310,9 @@ struct fastrpc_channel_ctx { struct list_head invoke_interrupted_mmaps; bool secure; bool unsigned_support; + bool poll_mode_supported; + /* Per-channel sequence counter; incremented on every context allocation */ + atomic_t ctx_seq; u64 dma_mask; const struct fastrpc_soc_data *soc_data; }; @@ -309,6 +336,8 @@ struct fastrpc_user { int client_id; int pd; bool is_secure_dev; + /* Flags poll mode state */ + bool poll_mode; /* Lock for lists */ spinlock_t lock; /* lock for allocations */ @@ -706,7 +735,9 @@ static struct fastrpc_invoke_ctx *fastrpc_context_alloc( spin_unlock_irqrestore(&cctx->lock, flags); goto err_idr; } - ctx->ctxid = FIELD_PREP(FASTRPC_CTXID_MASK, ret); + ctx->ctxid = FIELD_PREP(FASTRPC_CTXID_MASK, ret) | + FIELD_PREP(FASTRPC_CTXID_SEQ_MASK, + (u64)atomic_inc_return(&cctx->ctx_seq)); spin_unlock_irqrestore(&cctx->lock, flags); kref_init(&ctx->refcount); @@ -971,7 +1002,8 @@ static int fastrpc_get_meta_size(struct fastrpc_invoke_ctx *ctx) sizeof(struct fastrpc_invoke_buf) + sizeof(struct fastrpc_phy_page)) * ctx->nscalars + sizeof(u64) * FASTRPC_MAX_FDLIST + - sizeof(u32) * FASTRPC_MAX_CRCLIST; + sizeof(u32) * FASTRPC_MAX_CRCLIST + + sizeof(u32); return size; } @@ -1067,6 +1099,9 @@ static int fastrpc_get_args(u32 kernel, struct fastrpc_invoke_ctx *ctx) list = fastrpc_invoke_buf_start(rpra, ctx->nscalars); pages = fastrpc_phy_page_start(list, ctx->nscalars); ctx->fdlist = (u64 *)(pages + ctx->nscalars); + ctx->poll_addr = (u32 *)((uintptr_t)ctx->fdlist + sizeof(u64) * FASTRPC_MAX_FDLIST + + sizeof(u32) * FASTRPC_MAX_CRCLIST); + args = (uintptr_t)ctx->buf->virt + metalen; rlen = pkt_size - metalen; ctx->rpra = rpra; @@ -1236,6 +1271,71 @@ static int fastrpc_invoke_send(struct fastrpc_session_ctx *sctx, } +static u32 fastrpc_read_poll_addr(struct fastrpc_invoke_ctx *ctx) +{ + dma_rmb(); + return READ_ONCE(*ctx->poll_addr); +} + +static int poll_for_remote_response(struct fastrpc_invoke_ctx *ctx) +{ + u32 val; + int ret; + + /* + * Poll until DSP writes FASTRPC_POLL_RESPONSE into *ctx->poll_addr + * or until another path marks the work done. + */ + ret = read_poll_timeout_atomic(fastrpc_read_poll_addr, val, + (val == FASTRPC_POLL_RESPONSE) || ctx->is_work_done, 1, + FASTRPC_POLL_MAX_TIMEOUT_US, false, ctx); + + if (!ret && val == FASTRPC_POLL_RESPONSE) { + /* + * DSP writes FASTRPC_POLL_RESPONSE to signal successful + * completion via the poll path. + */ + ctx->is_work_done = true; + ctx->retval = 0; + } + + if (ret == -ETIMEDOUT) + ret = -EIO; + + return ret; +} + +static inline int fastrpc_wait_for_response(struct fastrpc_invoke_ctx *ctx, + u32 kernel) +{ + int err = 0; + + if (kernel) { + if (!wait_for_completion_timeout(&ctx->work, 10 * HZ)) + err = -ETIMEDOUT; + } else { + err = wait_for_completion_interruptible(&ctx->work); + } + + return err; +} + +static int fastrpc_wait_for_completion(struct fastrpc_invoke_ctx *ctx, + u32 kernel) +{ + int err; + + if (ctx->is_polled) { + err = poll_for_remote_response(ctx); + if (!err) + return 0; + /* If polling timed out or failed, move to normal response mode */ + ctx->is_polled = false; + } + + return fastrpc_wait_for_response(ctx, kernel); +} + static int fastrpc_internal_invoke(struct fastrpc_user *fl, u32 kernel, u32 handle, u32 sc, struct fastrpc_invoke_args *args) @@ -1271,13 +1371,14 @@ static int fastrpc_internal_invoke(struct fastrpc_user *fl, u32 kernel, if (err) goto bail; - if (kernel) { - if (!wait_for_completion_timeout(&ctx->work, 10 * HZ)) - err = -ETIMEDOUT; - } else { - err = wait_for_completion_interruptible(&ctx->work); - } + /* + * Set message context as polled if the call is for a user PD + * dynamic module and user has enabled poll mode. + */ + if (handle > FASTRPC_MAX_STATIC_HANDLE && fl->pd == USER_PD && fl->poll_mode) + ctx->is_polled = true; + err = fastrpc_wait_for_completion(ctx, kernel); if (err) goto bail; @@ -1841,6 +1942,35 @@ done: return 0; } +static int fastrpc_set_option(struct fastrpc_user *fl, char __user *argp) +{ + struct fastrpc_ioctl_set_option opt = {0}; + int i; + + if (copy_from_user(&opt, argp, sizeof(opt))) + return -EFAULT; + + for (i = 0; i < ARRAY_SIZE(opt.reserved); i++) { + if (opt.reserved[i] != 0) + return -EINVAL; + } + + if (opt.request_id != FASTRPC_POLL_MODE) + return -EINVAL; + + if (!fl->cctx->poll_mode_supported) + return -EOPNOTSUPP; + + if (opt.value == FASTRPC_POLL_MODE_ENABLE) + fl->poll_mode = true; + else if (opt.value == FASTRPC_POLL_MODE_DISABLE) + fl->poll_mode = false; + else + return -EINVAL; + + return 0; +} + static int fastrpc_get_dsp_info(struct fastrpc_user *fl, char __user *argp) { struct fastrpc_ioctl_capability cap = {0}; @@ -2196,6 +2326,9 @@ static long fastrpc_device_ioctl(struct file *file, unsigned int cmd, case FASTRPC_IOCTL_MEM_UNMAP: err = fastrpc_req_mem_unmap(fl, argp); break; + case FASTRPC_IOCTL_SET_OPTION: + err = fastrpc_set_option(fl, argp); + break; case FASTRPC_IOCTL_GET_DSP_INFO: err = fastrpc_get_dsp_info(fl, argp); break; @@ -2352,6 +2485,7 @@ static const struct fastrpc_soc_data kaanapali_soc_data = { .sid_pos = 56, .dma_addr_bits_cdsp = 34, .dma_addr_bits_default = 32, + .poll_mode_supported = true, }; static const struct fastrpc_soc_data default_soc_data = { @@ -2360,6 +2494,29 @@ static const struct fastrpc_soc_data default_soc_data = { .dma_addr_bits_default = 32, }; +/* + * Exception list for older platforms that use default_soc_data but whose + * DSP firmware supports FastRPC polling mode. + * + * NOTE: This list is intentionally closed. + * Do NOT add new platforms here. New SoCs must advertise polling mode + * support via their soc_data. + */ + +static const struct of_device_id fastrpc_poll_supported_machines[] __maybe_unused = { + { .compatible = "qcom,milos" }, + { .compatible = "qcom,qcs8300" }, + { .compatible = "qcom,sa8775p" }, + { .compatible = "qcom,sar2130p" }, + { .compatible = "qcom,sm8450" }, + { .compatible = "qcom,sm8550" }, + { .compatible = "qcom,sm8650" }, + { .compatible = "qcom,sm8750" }, + { .compatible = "qcom,x1e80100" }, + { .compatible = "qcom,x1p42100" }, + {}, +}; + static int fastrpc_rpmsg_probe(struct rpmsg_device *rpdev) { struct device *rdev = &rpdev->dev; @@ -2426,6 +2583,8 @@ static int fastrpc_rpmsg_probe(struct rpmsg_device *rpdev) secure_dsp = !(of_property_read_bool(rdev->of_node, "qcom,non-secure-domain")); data->secure = secure_dsp; data->soc_data = soc_data; + data->poll_mode_supported = soc_data->poll_mode_supported || + of_machine_get_match(fastrpc_poll_supported_machines); switch (domain_id) { case ADSP_DOMAIN_ID: @@ -2455,6 +2614,7 @@ static int fastrpc_rpmsg_probe(struct rpmsg_device *rpdev) } kref_init(&data->refcount); + atomic_set(&data->ctx_seq, 0); rdev->dma_mask = &data->dma_mask; dma_set_mask_and_coherent(rdev, DMA_BIT_MASK(32)); @@ -2552,7 +2712,25 @@ static int fastrpc_rpmsg_callback(struct rpmsg_device *rpdev, void *data, return -ENOENT; } + /* + * Validate the sequence number embedded in the upper bits of the + * context ID. Under high concurrency the IDR slot can be recycled + * for a new request before a late (or duplicate) glink COMPLETE + * response for the previous request arrives. Without this check the + * driver would call complete() on the wrong context, waking a thread + * whose buffers are still being accessed by the DSP. + */ + if (FIELD_GET(FASTRPC_CTXID_SEQ_MASK, rsp->ctx) != + FIELD_GET(FASTRPC_CTXID_SEQ_MASK, ctx->ctxid)) { + dev_dbg(&rpdev->dev, + "Stale glink response ctx 0x%llx (expected seq 0x%llx), dropping\n", + rsp->ctx, + FIELD_GET(FASTRPC_CTXID_SEQ_MASK, ctx->ctxid)); + return 0; + } + ctx->retval = rsp->retval; + ctx->is_work_done = true; complete(&ctx->work); /* diff --git a/include/uapi/misc/fastrpc.h b/include/uapi/misc/fastrpc.h index c6e2925f47e6..ba1ea5ed426c 100644 --- a/include/uapi/misc/fastrpc.h +++ b/include/uapi/misc/fastrpc.h @@ -16,6 +16,7 @@ #define FASTRPC_IOCTL_INIT_CREATE_STATIC _IOWR('R', 9, struct fastrpc_init_create_static) #define FASTRPC_IOCTL_MEM_MAP _IOWR('R', 10, struct fastrpc_mem_map) #define FASTRPC_IOCTL_MEM_UNMAP _IOWR('R', 11, struct fastrpc_mem_unmap) +#define FASTRPC_IOCTL_SET_OPTION _IOWR('R', 12, struct fastrpc_ioctl_set_option) #define FASTRPC_IOCTL_GET_DSP_INFO _IOWR('R', 13, struct fastrpc_ioctl_capability) /** @@ -67,6 +68,28 @@ enum fastrpc_proc_attr { /* Fastrpc attribute for memory protection of buffers */ #define FASTRPC_ATTR_SECUREMAP (1) +/** + * FASTRPC_POLL_MODE - Enable/disable poll mode for FastRPC invocations + * + * Poll mode is an optimization that allows the CPU to poll shared memory + * for completion instead of waiting for an interrupt-based response. + * This reduces latency for fast-completing operations. + * + * Restrictions: + * - Only supported for USER_PD (User Protection Domain) + * - Only applies to dynamic modules (handle > 20) + * - Static modules always use interrupt-based completion + * + * Values: + * - 0: Disable poll mode (use interrupt-based completion) + * - 1: Enable poll mode (poll shared memory for completion) + */ +#define FASTRPC_POLL_MODE (1) + +/* Values for FASTRPC_POLL_MODE request */ +#define FASTRPC_POLL_MODE_DISABLE 0 +#define FASTRPC_POLL_MODE_ENABLE 1 + struct fastrpc_invoke_args { __u64 ptr; __u64 length; @@ -133,6 +156,12 @@ struct fastrpc_mem_unmap { __s32 reserved[5]; }; +struct fastrpc_ioctl_set_option { + __u32 request_id; /* Request type (e.g., FASTRPC_POLL_MODE) */ + __u32 value; /* Request-specific value */ + __s32 reserved[6]; +}; + struct fastrpc_ioctl_capability { __u32 unused; /* deprecated, ignored by the kernel */ __u32 attribute_id; |