From ec59ce97169e22ed56ef1dfdb76f4f7b3b874229 Mon Sep 17 00:00:00 2001 From: Kai Vehmanen Date: Wed, 4 Nov 2026 16:45:26 +0200 Subject: [PATCH 1/8] schedule: zephyr_ll: grant LL lock access from a kernel-only copy zephyr_ll_locks[] is in a user-accessible partition, so user threads can read the lock handles for the lock syscalls. user_ll_grant_access() runs in kernel context, but took the handle to grant from that same array. A user thread could overwrite an entry and make the kernel grant it access to an arbitrary kernel object. This is reachable whenever grants are done while user threads already run, e.g. when a secondary core and its IPC user thread are brought up, and with cross-core streams also for the locks of other cores. Keep a kernel-only copy of the lock handles and use it for the grants. Signed-off-by: Kai Vehmanen --- src/include/sof/schedule/ll_schedule_domain.h | 2 ++ src/schedule/zephyr_ll.c | 13 +++++++++++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/src/include/sof/schedule/ll_schedule_domain.h b/src/include/sof/schedule/ll_schedule_domain.h index c369242640af..3860f5c96bf7 100644 --- a/src/include/sof/schedule/ll_schedule_domain.h +++ b/src/include/sof/schedule/ll_schedule_domain.h @@ -229,6 +229,8 @@ void zephyr_ll_user_resources_init(void); /** * @brief Grant thread access to per-core Low-Latency mutexes. + * + * Kernel context only. * @param[in,out] thread Target thread. * @param[in] core Target core ID. */ diff --git a/src/schedule/zephyr_ll.c b/src/schedule/zephyr_ll.c index 3edebbca5b20..4c4fd7059c30 100644 --- a/src/schedule/zephyr_ll.c +++ b/src/schedule/zephyr_ll.c @@ -56,6 +56,13 @@ struct zephyr_ll_pdata { */ static APP_SYSUSER_BSS struct k_mutex *zephyr_ll_locks[CONFIG_CORE_COUNT]; +/* + * Kernel-only copy of zephyr_ll_locks[]. That array is writable by user + * threads, so kernel code granting access to the locks must use this copy, + * otherwise a user thread could make it grant access to any kernel object. + */ +static struct k_mutex *zephyr_ll_klocks[CONFIG_CORE_COUNT]; + /* * Shadow ownership tracking for the per-core LL mutex, used only to * power user_ll_assert_locked(). The kernel k_mutex object is not @@ -703,6 +710,7 @@ static void zephyr_ll_scheduler_free(void *data, uint32_t flags) #if CONFIG_SOF_USERSPACE_LL zephyr_ll_locks[sch->core] = NULL; + zephyr_ll_klocks[sch->core] = NULL; k_object_free(sch->lock); #endif @@ -776,9 +784,9 @@ struct task *zephyr_ll_task_alloc(void) void user_ll_grant_access(struct k_thread *thread, int core) { - assert(core < CONFIG_CORE_COUNT && zephyr_ll_locks[core] != NULL); + assert(core < CONFIG_CORE_COUNT && zephyr_ll_klocks[core] != NULL); - k_thread_access_grant(thread, zephyr_ll_locks[core]); + k_thread_access_grant(thread, zephyr_ll_klocks[core]); } /** @@ -908,6 +916,7 @@ __cold int zephyr_ll_scheduler_init(struct ll_schedule_domain *domain) } assert(core < CONFIG_CORE_COUNT && zephyr_ll_locks[core] == NULL); zephyr_ll_locks[core] = sch->lock; + zephyr_ll_klocks[core] = sch->lock; k_mutex_init(sch->lock); tr_dbg(&ll_tr, "ll-scheduler init done, sch %p sch->lock %p", sch, sch->lock); From ded6ae7c1907757193785851944a7ae3e3fa4d2c Mon Sep 17 00:00:00 2001 From: Kai Vehmanen Date: Thu, 29 Oct 2026 20:10:48 +0200 Subject: [PATCH 2/8] idc: add idc_cmd_user() for processing IDC messages in user space Pass the message payload to the component IDC message handlers as an argument, instead of fetching it from the per-core IDC payload area, which is kernel memory. Add idc_cmd_user() to let user-space threads (with CONFIG_SOF_USERSPACE_LL) process component messages, sent from another core, with a payload in user-accessible memory. Only the messages, which can originate from the user-space IPC and LL threads, are supported. LL modules on a core different from their pipeline core aren't supported with user-space LL, so fail the prepare instead of trying to set up an LL task for them. Signed-off-by: Kai Vehmanen --- src/idc/idc.c | 88 +++++++++++++++++++++++++-------------- zephyr/include/rtos/idc.h | 5 +++ 2 files changed, 62 insertions(+), 31 deletions(-) diff --git a/src/idc/idc.c b/src/idc/idc.c index 8fdecb8ba173..5a6745719db2 100644 --- a/src/idc/idc.c +++ b/src/idc/idc.c @@ -84,51 +84,39 @@ static void idc_ipc(void) ipc_cmd(ipc->comp_data); } -static int idc_ipc4_bind(uint32_t comp_id) +static int idc_ipc4_bind(uint32_t comp_id, void *payload) { struct ipc_comp_dev *ipc_dev; - struct idc_payload *payload; - struct bind_info *bu; + struct bind_info *bu = payload; ipc_dev = ipc_get_comp_by_id(ipc_get(), comp_id); if (!ipc_dev) return -ENODEV; - payload = idc_payload_get(*idc_get(), cpu_get_id()); - bu = (struct bind_info *)payload; - return comp_bind(ipc_dev->cd, bu); } -static int idc_ipc4_unbind(uint32_t comp_id) +static int idc_ipc4_unbind(uint32_t comp_id, void *payload) { struct ipc_comp_dev *ipc_dev; - struct idc_payload *payload; - struct bind_info *bu; + struct bind_info *bu = payload; ipc_dev = ipc_get_comp_by_id(ipc_get(), comp_id); if (!ipc_dev) return -ENODEV; - payload = idc_payload_get(*idc_get(), cpu_get_id()); - bu = (struct bind_info *)payload; - return comp_unbind(ipc_dev->cd, bu); } -static int idc_get_attribute(uint32_t comp_id) +static int idc_get_attribute(uint32_t comp_id, void *payload) { struct ipc_comp_dev *ipc_dev; - struct idc_payload *idc_payload; - struct get_attribute_remote_payload *get_attr_payload; + struct get_attribute_remote_payload *get_attr_payload = payload; ipc_dev = ipc_get_comp_by_id(ipc_get(), comp_id); if (!ipc_dev) return -ENODEV; - idc_payload = idc_payload_get(*idc_get(), cpu_get_id()); - get_attr_payload = (struct get_attribute_remote_payload *)idc_payload; - return comp_get_attribute(ipc_dev->cd, get_attr_payload->type, get_attr_payload->value); } @@ -137,14 +125,11 @@ static int idc_get_attribute(uint32_t comp_id) * \param[in] comp_id Component id to have params set. * \return Error code. */ -static int idc_params(uint32_t comp_id) +static int idc_params(uint32_t comp_id, void *payload) { struct ipc *ipc = ipc_get(); struct ipc_comp_dev *ipc_dev; - struct idc *idc = *idc_get(); - struct idc_payload *payload = idc_payload_get(idc, cpu_get_id()); - struct sof_ipc_stream_params *params = - (struct sof_ipc_stream_params *)payload; + struct sof_ipc_stream_params *params = payload; int ret; ipc_dev = ipc_get_comp_by_id(ipc, comp_id); @@ -185,6 +170,12 @@ static int idc_prepare(uint32_t comp_id) /* we're running LL on different core, so allocate our own task */ if (!dev->task && dev->ipc_config.proc_domain == COMP_PROCESSING_DOMAIN_LL) { +#ifdef CONFIG_SOF_USERSPACE_LL + /* user-space LL modules must run on their pipeline core */ + tr_err(&idc_tr, "comp %#x: LL module on a remote core", comp_id); + ret = -EINVAL; + goto out; +#endif /* allocate task for shared component */ dev->task = sof_heap_alloc(dev->drv->user_heap, SOF_MEM_FLAG_USER, sizeof(*dev->task), 0); @@ -218,12 +209,10 @@ static int idc_prepare(uint32_t comp_id) * \param[in] comp_id Component id to be triggered. * \return Error code. */ -static int idc_trigger(uint32_t comp_id) +static int idc_trigger(uint32_t comp_id, void *payload) { struct ipc *ipc = ipc_get(); struct ipc_comp_dev *ipc_dev; - struct idc *idc = *idc_get(); - struct idc_payload *payload = idc_payload_get(idc, cpu_get_id()); uint32_t cmd = *(uint32_t *)payload; ipc_dev = ipc_get_comp_by_id(ipc, comp_id); @@ -356,6 +345,7 @@ static void idc_secondary_core_crashed(const uint32_t header) */ void idc_cmd(struct idc_msg *msg) { + void *payload = idc_payload_get(*idc_get(), cpu_get_id()); uint32_t type = iTS(msg->header); int ret = 0; @@ -377,25 +367,25 @@ void idc_cmd(struct idc_msg *msg) dbg_path_hot_stop_watching(); break; case iTS(IDC_MSG_BIND): - ret = idc_ipc4_bind(msg->extension); + ret = idc_ipc4_bind(msg->extension, payload); break; case iTS(IDC_MSG_UNBIND): - ret = idc_ipc4_unbind(msg->extension); + ret = idc_ipc4_unbind(msg->extension, payload); break; case iTS(IDC_MSG_GET_ATTRIBUTE): - ret = idc_get_attribute(msg->extension); + ret = idc_get_attribute(msg->extension, payload); break; case iTS(IDC_MSG_FREE): ret = idc_comp_free(msg->extension); break; case iTS(IDC_MSG_PARAMS): - ret = idc_params(msg->extension); + ret = idc_params(msg->extension, payload); break; case iTS(IDC_MSG_PREPARE): ret = idc_prepare(msg->extension); break; case iTS(IDC_MSG_TRIGGER): - ret = idc_trigger(msg->extension); + ret = idc_trigger(msg->extension, payload); break; case iTS(IDC_MSG_RESET): ret = idc_reset(msg->extension); @@ -421,6 +411,42 @@ void idc_cmd(struct idc_msg *msg) idc_msg_status_set(ret, cpu_get_id()); } +#ifdef CONFIG_SOF_USERSPACE_LL +/** + * \brief Executes a component IDC message in a user-space thread. + * \param[in] msg IDC message, forwarded by a user thread on another core. + * \param[in,out] payload Message payload. + * \return Error code. + * + * Only the component messages, that can be sent from the user-space IPC + * and LL threads, are supported. + */ +int idc_cmd_user(const struct idc_msg *msg, void *payload) +{ + switch (iTS(msg->header)) { + case iTS(IDC_MSG_BIND): + return idc_ipc4_bind(msg->extension, payload); + case iTS(IDC_MSG_UNBIND): + return idc_ipc4_unbind(msg->extension, payload); + case iTS(IDC_MSG_GET_ATTRIBUTE): + return idc_get_attribute(msg->extension, payload); + case iTS(IDC_MSG_FREE): + return idc_comp_free(msg->extension); + case iTS(IDC_MSG_PARAMS): + return idc_params(msg->extension, payload); + case iTS(IDC_MSG_PREPARE): + return idc_prepare(msg->extension); + case iTS(IDC_MSG_TRIGGER): + return idc_trigger(msg->extension, payload); + case iTS(IDC_MSG_RESET): + return idc_reset(msg->extension); + default: + tr_err(&idc_tr, "unsupported user msg->header = %#x", msg->header); + return -EINVAL; + } +} +#endif /* CONFIG_SOF_USERSPACE_LL */ + /* Runs on each CPU */ __cold int idc_init(void) { diff --git a/zephyr/include/rtos/idc.h b/zephyr/include/rtos/idc.h index 2afe69d8e186..f47025ff5bf6 100644 --- a/zephyr/include/rtos/idc.h +++ b/zephyr/include/rtos/idc.h @@ -202,6 +202,11 @@ enum task_state idc_do_cmd(void *data); void idc_cmd(struct idc_msg *msg); +#ifdef CONFIG_SOF_USERSPACE_LL +int idc_cmd_user(const struct idc_msg *msg, void *payload); +int ipc_user_xcore_send(const struct idc_msg *msg); +#endif + int idc_msg_status_get(uint32_t core); int idc_init(void); From 4cc180350a9dbb7e9d6121bdcfffd58aea99cd85 Mon Sep 17 00:00:00 2001 From: Kai Vehmanen Date: Tue, 3 Nov 2026 19:22:33 +0200 Subject: [PATCH 3/8] ipc: pass cross-core messages between user-space IPC threads With CONFIG_SOF_USERSPACE_LL, component operations on a module on another core (cross-core bind, remote get_attribute, free, params, prepare, trigger, reset) are called from user-space IPC and LL threads. They use idc_send_msg(), which relies on kernel objects and data and faults in user mode. Add a cross-core request slot per core to struct ipc_user. When called in user context, idc_send_msg() now copies the message and its payload to the slot of the target core and wakes the user-space IPC thread of that core, which processes it with idc_cmd_user() and signals completion. The IPC4 bind / unbind message, referenced by the bind payload, usually is on the sender's stack, so it's copied as well. The user-space IPC thread of a core now has two independent producers, the kernel IPC handler (IPC_USER_EVENT_CMD) and user threads on other cores (IPC_USER_EVENT_XCORE). Post the events with k_event_post() instead of k_event_set(), which would clear a pending request of the other producer. An LL thread holds the LL scheduler lock of its core for the whole LL cycle, also when it calls a remote module, e.g. from a timer-driven pipeline trigger task or when a failed copy resets the pipeline. Waiting for the target IPC thread with the lock held could deadlock, if that thread needs the lock, e.g. for a bind locking both cores, or needs the LL lock of its own core, whose LL thread may in turn wait for a request to this core. Release all holds of an LL thread on its LL lock while it waits for a cross-core request, and re-take them afterwards, like the kernel-mode IDC path does implicitly. Track the LL lock owner and depth also without CONFIG_ASSERT for this. Grant all user-space IPC and LL threads access to the request slots and LL scheduler locks of all initialized cores. The grants are done from kernel-only copies of the thread pointers, so keep one for the LL audio threads as well. Signed-off-by: Kai Vehmanen --- src/idc/zephyr_idc.c | 17 ++- src/include/sof/ipc/common.h | 27 ++++ src/include/sof/schedule/ll_schedule.h | 18 +++ src/ipc/ipc-common.c | 181 ++++++++++++++++++++++++- src/schedule/zephyr_ll.c | 43 ++++-- 5 files changed, 270 insertions(+), 16 deletions(-) diff --git a/src/idc/zephyr_idc.c b/src/idc/zephyr_idc.c index bba2d32a5ffd..576d18f4243f 100644 --- a/src/idc/zephyr_idc.c +++ b/src/idc/zephyr_idc.c @@ -127,7 +127,7 @@ static struct zephyr_idc_msg idc_work[CONFIG_CORE_COUNT * 2]; /* Protect the above array */ static K_MUTEX_DEFINE(idc_mutex); -int idc_send_msg(struct idc_msg *msg, uint32_t mode) +static int idc_send_msg_kernel(struct idc_msg *msg, uint32_t mode) { struct idc *idc = *idc_get(); struct idc_payload *payload = idc_payload_get(idc, msg->core); @@ -200,6 +200,21 @@ int idc_send_msg(struct idc_msg *msg, uint32_t mode) return ret; } +int idc_send_msg(struct idc_msg *msg, uint32_t mode) +{ +#ifdef CONFIG_SOF_USERSPACE_LL + /* + * IDC uses kernel objects and data. User-space threads instead pass + * the message to the user-space IPC thread of the target core. Such + * messages are always processed synchronously. + */ + if (k_is_user_context()) + return ipc_user_xcore_send(msg); +#endif + + return idc_send_msg_kernel(msg, mode); +} + __cold void idc_init_thread(void) { char thread_name[] = "idc_p4wq0"; diff --git a/src/include/sof/ipc/common.h b/src/include/sof/ipc/common.h index 696cb553661d..cdf4fd44c764 100644 --- a/src/include/sof/ipc/common.h +++ b/src/include/sof/ipc/common.h @@ -62,6 +62,31 @@ extern struct tr_ctx ipc_tr; #define IPC_DSP_CORE_COUNT 1 #endif +/** @brief Maximum payload size of a cross-core request between user threads */ +#define IPC_USER_XCORE_PAYLOAD_SIZE 128 + +/** + * @brief Cross-core request slot of a user-space IPC thread. + * + * User-space threads can't use IDC. Instead, they pass IDC messages for + * another core to the user-space IPC thread of that core via this slot. + */ +struct ipc_user_xcore { + /** @brief Serializes senders to this core */ + struct k_sem *lock; + /** @brief Signalled by the target core when the request is done */ + struct k_sem *done; + /** @brief IDC message header and extension */ + uint32_t header; + uint32_t extension; + /** @brief Result from the target core */ + int result; + /** @brief Copy of IPC4 bind / unbind message, referenced by the payload */ + uint32_t ipc4_bind_data[2]; + /** @brief Copy of the IDC message payload */ + uint8_t payload[IPC_USER_XCORE_PAYLOAD_SIZE] __aligned(8); +}; + struct ipc_user { struct k_thread *thread[IPC_DSP_CORE_COUNT]; struct k_sem *sem; @@ -96,6 +121,8 @@ struct ipc_user { * Size verified by BUILD_ASSERT in handler-user.c. */ uint8_t init_drv_data[160] __aligned(4); + /** @brief Cross-core request slots, one per target core */ + struct ipc_user_xcore xcore[IPC_DSP_CORE_COUNT]; }; struct ipc { diff --git a/src/include/sof/schedule/ll_schedule.h b/src/include/sof/schedule/ll_schedule.h index c49bf9f0f131..e3c9bc40c3c1 100644 --- a/src/include/sof/schedule/ll_schedule.h +++ b/src/include/sof/schedule/ll_schedule.h @@ -58,6 +58,24 @@ void user_ll_lock_sched(int core); */ void user_ll_unlock_sched(int core); +/** + * @brief Release all holds of the calling thread on a per-core user + * Low-Latency scheduling lock. + * + * Must be paired with user_ll_relock_sched(). + * @param[in] core Target core ID. + * @return Number of holds released, 0 if the calling thread doesn't hold + * the lock. + */ +unsigned int user_ll_unlock_sched_all(int core); + +/** + * @brief Re-take holds released by user_ll_unlock_sched_all(). + * @param[in] core Target core ID. + * @param[in] depth Value returned by user_ll_unlock_sched_all(). + */ +void user_ll_relock_sched(int core, unsigned int depth); + #ifdef CONFIG_ASSERT /** * @brief Assert that the calling context holds the user Low-Latency lock for a core. diff --git a/src/ipc/ipc-common.c b/src/ipc/ipc-common.c index bbc4a9b8a7d3..8f4b6322be4b 100644 --- a/src/ipc/ipc-common.c +++ b/src/ipc/ipc-common.c @@ -32,6 +32,9 @@ #include #include #include +#if CONFIG_IPC_MAJOR_4 +#include +#endif #include #include #include @@ -423,6 +426,15 @@ void ipc_schedule_process(struct ipc *ipc) #define IPC_USER_EVENT_CMD BIT(0) #define IPC_USER_EVENT_STOP BIT(1) +#define IPC_USER_EVENT_XCORE BIT(2) + +BUILD_ASSERT(IPC_USER_XCORE_PAYLOAD_SIZE >= IDC_MAX_PAYLOAD_SIZE, + "cross-core request payload smaller than IDC payload"); +#if CONFIG_IPC_MAJOR_4 +BUILD_ASSERT(sizeof(((struct ipc_user_xcore *)0)->ipc4_bind_data) == + sizeof(struct ipc4_module_bind_unbind), + "cross-core request bind data size mismatch"); +#endif SOF_DEFINE_REG_UUID(sec_core_init); @@ -438,8 +450,13 @@ static struct ipc_user *ipc_user_kptr; static struct k_sem *ipc_user_ksem; static struct k_event *ipc_user_kevent[CONFIG_CORE_COUNT]; static struct k_thread *ipc_user_kthread[CONFIG_CORE_COUNT]; +static struct k_thread *ipc_user_kaudio_thread[CONFIG_CORE_COUNT]; static bool ipc_user_kinit_needed[CONFIG_CORE_COUNT]; +/* Cross-core request slot semaphores, static so they're never freed */ +static struct k_sem ipc_user_xcore_lock[CONFIG_CORE_COUNT]; +static struct k_sem ipc_user_xcore_done[CONFIG_CORE_COUNT]; + /** * @brief Forward an IPC command to the user-space thread. * @@ -474,8 +491,11 @@ int ipc_user_forward_cmd(uint32_t primary, uint32_t extension, unsigned int core ipc->task_mask |= IPC_TASK_IN_THREAD; k_spin_unlock(&ipc->lock, key); - /* Wake the user thread */ - k_event_set(ipc_user_kevent[core], IPC_USER_EVENT_CMD); + /* + * Wake the user thread. Post, not set: a cross-core request + * (IPC_USER_EVENT_XCORE) may be pending on the same event. + */ + k_event_post(ipc_user_kevent[core], IPC_USER_EVENT_CMD); /* Wait for user thread to complete */ ret = k_sem_take(ipc_user_ksem, K_MSEC(100)); @@ -493,6 +513,86 @@ int ipc_user_forward_cmd(uint32_t primary, uint32_t extension, unsigned int core return !ret ? pdata->result : ret; } +/** + * @brief Pass an IDC message to the user-space IPC thread of another core. + * + * Called from user-space threads (IPC or LL) instead of idc_send_msg(), + * which is kernel-only. Blocks until the target core has processed the + * message and returns its result. + * + * @param msg IDC message + * @return Result from the target core, or negative error code + */ +int ipc_user_xcore_send(const struct idc_msg *msg) +{ + struct ipc_user *ipc_user = ipc_get()->ipc_user_pdata; + struct ipc_user_xcore *xc; + unsigned int ll_depth = 0; + int ll_core; + int ret; + + if (msg->core >= CONFIG_CORE_COUNT || msg->size > sizeof(xc->payload) || + !ipc_user->event[msg->core]) + return -EINVAL; + + xc = &ipc_user->xcore[msg->core]; + + /* + * An LL thread holds the LL scheduler lock of its core for the whole + * LL cycle, e.g. when it triggers or resets a remote module. Release + * it while waiting: the target IPC thread may need it (e.g. for a + * bind, locking both cores), or need the LL lock of its own core, + * whose LL thread may in turn wait for a request to this core. + */ + for (ll_core = 0; ll_core < CONFIG_CORE_COUNT; ll_core++) { + if (ipc_user->audio_thread[ll_core] == k_current_get()) { + ll_depth = user_ll_unlock_sched_all(ll_core); + break; + } + } + + k_sem_take(xc->lock, K_FOREVER); + + xc->header = msg->header; + xc->extension = msg->extension; + if (msg->size) { + ret = memcpy_s(xc->payload, sizeof(xc->payload), msg->payload, msg->size); + if (ret < 0) + goto out; + } + +#if CONFIG_IPC_MAJOR_4 + /* + * The bind / unbind message usually is on the sender's stack, which + * the target thread can't access. Pass a copy of it. + */ + if (iTS(msg->header) == iTS(IDC_MSG_BIND) || iTS(msg->header) == iTS(IDC_MSG_UNBIND)) { + struct bind_info *bind_data = (struct bind_info *)xc->payload; + + if (bind_data->ipc4_data) { + ret = memcpy_s(xc->ipc4_bind_data, sizeof(xc->ipc4_bind_data), + bind_data->ipc4_data, sizeof(*bind_data->ipc4_data)); + if (ret < 0) + goto out; + bind_data->ipc4_data = + (const struct ipc4_module_bind_unbind *)xc->ipc4_bind_data; + } + } +#endif + + /* post, not set: don't clear a pending IPC_USER_EVENT_CMD */ + k_event_post(ipc_user->event[msg->core], IPC_USER_EVENT_XCORE); + k_sem_take(xc->done, K_FOREVER); + ret = xc->result; +out: + k_sem_give(xc->lock); + + if (ll_depth) + user_ll_relock_sched(ll_core, ll_depth); + + return ret; +} + /** * @brief Protocol-specific dispatch of a forwarded IPC command. * @@ -524,11 +624,24 @@ static void ipc_user_thread_fn(void *p1, void *p2, void *p3) for (;;) { uint32_t mask = k_event_wait_safe(ipc_user->event[core], - IPC_USER_EVENT_CMD | IPC_USER_EVENT_STOP, + IPC_USER_EVENT_CMD | IPC_USER_EVENT_STOP | + IPC_USER_EVENT_XCORE, false, K_FOREVER); LOG_DBG("IPC user wake, mask %u", mask); + if (mask & IPC_USER_EVENT_XCORE) { + struct ipc_user_xcore *xc = &ipc_user->xcore[core]; + const struct idc_msg msg = { + .header = xc->header, + .extension = xc->extension, + .core = core, + }; + + xc->result = idc_cmd_user(&msg, xc->payload); + k_sem_give(xc->done); + } + if (mask & IPC_USER_EVENT_CMD) { ipc_user->result = ipc_user_thread_dispatch(ipc_user); @@ -605,6 +718,55 @@ __cold static int ipc_user_init_thread(struct ipc_user *ipc_user, unsigned int c return ret; } +/* + * Grant the threads of the newly initialized core and of the other already + * initialized cores access to each other's cross-core request slots and LL + * scheduler locks. Uses only the kernel-only copies of the thread state. + */ +__cold static void ipc_user_xcore_grant(unsigned int core) +{ + struct k_thread *new_threads[] = { + ipc_user_kthread[core], ipc_user_kaudio_thread[core], + }; + unsigned int c, i; + + assert_can_be_cold(); + + for (c = 0; c < CONFIG_CORE_COUNT; c++) { + struct k_thread *threads[] = { + ipc_user_kthread[c], ipc_user_kaudio_thread[c], + }; + + /* + * Grant the slot semaphores of all cores. Note: braces needed, + * k_thread_access_grant() expands to multiple statements. + */ + for (i = 0; i < ARRAY_SIZE(new_threads); i++) { + if (new_threads[i]) { + k_thread_access_grant(new_threads[i], &ipc_user_xcore_lock[c], + &ipc_user_xcore_done[c]); + } + } + + if (!ipc_user_kthread[c]) + continue; + + for (i = 0; i < ARRAY_SIZE(threads); i++) { + if (!threads[i]) + continue; + k_thread_access_grant(threads[i], ipc_user_kevent[core]); + user_ll_grant_access(threads[i], core); + } + + for (i = 0; i < ARRAY_SIZE(new_threads); i++) { + if (!new_threads[i]) + continue; + k_thread_access_grant(new_threads[i], ipc_user_kevent[c]); + user_ll_grant_access(new_threads[i], c); + } + } +} + __cold int ipc_user_init_secondary(unsigned int core) { struct ipc_user *ipc_user = ipc_user_kptr; @@ -640,7 +802,9 @@ __cold int ipc_user_init_secondary(unsigned int core) } k_thread_access_grant(ipc_user_kthread[core], audio_thread); + ipc_user_kaudio_thread[core] = audio_thread; ipc_user->audio_thread[core] = audio_thread; + ipc_user_xcore_grant(core); return 0; } @@ -740,6 +904,15 @@ __cold static void ipc_user_init(void) k_sem_init(ipc_user_ksem, 0, 1); + for (core = 0; core < CONFIG_CORE_COUNT; core++) { + struct ipc_user_xcore *xc = &ipc_user->xcore[core]; + + xc->lock = &ipc_user_xcore_lock[core]; + xc->done = &ipc_user_xcore_done[core]; + k_sem_init(xc->lock, 1, 1); + k_sem_init(xc->done, 0, 1); + } + ret = ipc_user_init_thread(ipc_user, PLATFORM_PRIMARY_CORE_ID); if (ret < 0) { LOG_ERR("user IPC thread initialization failed"); @@ -776,7 +949,9 @@ __cold static void ipc_user_init(void) * k_thread_access_grant(audio_thread, dai_mutex) from user context. */ k_thread_access_grant(ipc_user_kthread[PLATFORM_PRIMARY_CORE_ID], audio_thread); + ipc_user_kaudio_thread[PLATFORM_PRIMARY_CORE_ID] = audio_thread; ipc_user->audio_thread[PLATFORM_PRIMARY_CORE_ID] = audio_thread; + ipc_user_xcore_grant(PLATFORM_PRIMARY_CORE_ID); k_thread_start(ipc_user_kthread[PLATFORM_PRIMARY_CORE_ID]); diff --git a/src/schedule/zephyr_ll.c b/src/schedule/zephyr_ll.c index 4c4fd7059c30..f1de577f49de 100644 --- a/src/schedule/zephyr_ll.c +++ b/src/schedule/zephyr_ll.c @@ -64,16 +64,15 @@ static APP_SYSUSER_BSS struct k_mutex *zephyr_ll_locks[CONFIG_CORE_COUNT]; static struct k_mutex *zephyr_ll_klocks[CONFIG_CORE_COUNT]; /* - * Shadow ownership tracking for the per-core LL mutex, used only to - * power user_ll_assert_locked(). The kernel k_mutex object is not - * readable from user-space threads, so its owner cannot be inspected - * directly; instead record the owner and recursion depth here, in a - * user-accessible partition. Updated on every lock/unlock of the LL - * mutex, by both the LL thread (zephyr_ll_lock()) and IPC handlers - * (user_ll_lock_sched()). Only ever mutated by the current lock holder - * while the lock is held, so no extra synchronization is required. + * Shadow ownership tracking for the per-core LL mutex, used by + * user_ll_assert_locked() and user_ll_unlock_sched_all(). The kernel + * k_mutex object is not readable from user-space threads, so its owner + * cannot be inspected directly; instead record the owner and recursion + * depth here, in a user-accessible partition. Updated on every lock/unlock + * of the LL mutex, by both the LL thread (zephyr_ll_lock()) and IPC + * handlers (user_ll_lock_sched()). Only ever mutated by the current lock + * holder while the lock is held, so no extra synchronization is required. */ -#ifdef CONFIG_ASSERT static APP_SYSUSER_BSS k_tid_t zephyr_ll_lock_owner[CONFIG_CORE_COUNT]; static APP_SYSUSER_BSS uint32_t zephyr_ll_lock_depth[CONFIG_CORE_COUNT]; @@ -90,14 +89,12 @@ static inline void zephyr_ll_lock_releasing(int core) zephyr_ll_lock_owner[core] = NULL; } +#ifdef CONFIG_ASSERT void user_ll_assert_locked(int core) { assert(core < CONFIG_CORE_COUNT && zephyr_ll_lock_owner[core] == k_current_get()); } -#else /* CONFIG_ASSERT */ -static inline void zephyr_ll_lock_acquired(int core) { (void)core; } -static inline void zephyr_ll_lock_releasing(int core) { (void)core; } #endif /* CONFIG_ASSERT */ #endif /* CONFIG_SOF_USERSPACE_LL */ @@ -815,6 +812,28 @@ void user_ll_unlock_sched(int core) assert(!ret); } +unsigned int user_ll_unlock_sched_all(int core) +{ + unsigned int depth, i; + + assert(core >= 0 && core < CONFIG_CORE_COUNT); + + if (zephyr_ll_lock_owner[core] != k_current_get()) + return 0; + + depth = zephyr_ll_lock_depth[core]; + for (i = 0; i < depth; i++) + user_ll_unlock_sched(core); + + return depth; +} + +void user_ll_relock_sched(int core, unsigned int depth) +{ + while (depth--) + user_ll_lock_sched(core); +} + #endif /* CONFIG_SOF_USERSPACE_LL */ int zephyr_ll_task_init(struct task *task, From df39af850f9f92f0e5ab0839c5c6ceceba124d6c Mon Sep 17 00:00:00 2001 From: Kai Vehmanen Date: Thu, 29 Oct 2026 20:10:48 +0200 Subject: [PATCH 4/8] audio: component: allocate remote attribute buffer from LL user heap comp_ipc4_get_attribute_remote() allocates the buffer for the remote core to fill with rzalloc(), which takes a spinlock and fails in user mode. With CONFIG_SOF_USERSPACE_LL it's called from user-space threads and the buffer is filled by the user-space IPC thread of the remote core, so allocate it from the coherent LL user heap instead. Signed-off-by: Kai Vehmanen --- src/include/sof/audio/component_ext.h | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/src/include/sof/audio/component_ext.h b/src/include/sof/audio/component_ext.h index 3a4c3ce8826e..913e69c62c9c 100644 --- a/src/include/sof/audio/component_ext.h +++ b/src/include/sof/audio/component_ext.h @@ -18,8 +18,9 @@ #include #include -/* TODO: the cross-core helpers below send IDC messages, which is kernel only */ +/* in user-space LL builds idc_send_msg() forwards to the remote user IPC thread */ #include +#include #include /** \addtogroup component_infra_api @@ -299,9 +300,18 @@ static inline int comp_ipc4_get_attribute_remote(struct comp_dev *dev, uint32_t if (type != COMP_ATTR_BASE_CONFIG) return -EINVAL; +#ifdef CONFIG_SOF_USERSPACE_LL + /* must be writable by the user-space thread on the remote core */ + base_cfg = sof_heap_alloc(zephyr_ll_user_heap(), SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, + sizeof(*base_cfg), 0); + if (!base_cfg) + return -ENOMEM; + memset(base_cfg, 0, sizeof(*base_cfg)); +#else base_cfg = rzalloc(SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, sizeof(*base_cfg)); if (!base_cfg) return -ENOMEM; +#endif payload.type = type; payload.value = base_cfg; @@ -312,7 +322,11 @@ static inline int comp_ipc4_get_attribute_remote(struct comp_dev *dev, uint32_t memcpy_s(value, sizeof(struct ipc4_base_module_cfg), base_cfg, sizeof(struct ipc4_base_module_cfg)); +#ifdef CONFIG_SOF_USERSPACE_LL + sof_heap_free(zephyr_ll_user_heap(), base_cfg); +#else rfree(base_cfg); +#endif return ret; } #endif /* CONFIG_IPC_MAJOR_4 */ From 5f74a5310dc13cd3bd222639cb63ebf592fbc4da Mon Sep 17 00:00:00 2001 From: Kai Vehmanen Date: Thu, 29 Oct 2026 20:10:48 +0200 Subject: [PATCH 5/8] ipc4: helper: lock the pipeline core when freeing pipeline modules With CONFIG_SOF_USERSPACE_LL, ipc_pipeline_module_free() takes the LL scheduler lock of the core of the first module in the pipeline. That is supposed to be the pipeline core, but a module can be on another core (e.g. an offloaded DP module). Freeing such a remote module is done on its core, where ipc_comp_free() takes the LL lock of that core, which would deadlock, if it's the lock taken here. Pass the pipeline core to ipc_pipeline_module_free() and lock that. Signed-off-by: Kai Vehmanen --- src/ipc/ipc4/helper.c | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/ipc/ipc4/helper.c b/src/ipc/ipc4/helper.c index d5a0e095b3f4..62c0b3d46e4d 100644 --- a/src/ipc/ipc4/helper.c +++ b/src/ipc/ipc4/helper.c @@ -589,14 +589,11 @@ __cold static inline int ipc_comp_free_remote(struct comp_dev *dev) return idc_send_msg(&msg, IDC_BLOCKING); } -__cold static int ipc_pipeline_module_free(uint32_t pipeline_id) +__cold static int ipc_pipeline_module_free(uint32_t pipeline_id, uint32_t ppl_core) { struct ipc *ipc = ipc_get(); struct ipc_comp_dev *icd; int ret; -#ifdef CONFIG_SOF_USERSPACE_LL - int ppl_core; -#endif assert_can_be_cold(); @@ -605,7 +602,11 @@ __cold static int ipc_pipeline_module_free(uint32_t pipeline_id) return IPC4_SUCCESS; #ifdef CONFIG_SOF_USERSPACE_LL - ppl_core = icd->core; + /* + * Lock the pipeline core's LL scheduler. Not the core of the first + * module, which can be a remote one: freeing a remote module takes + * the LL lock of its core on that core. + */ user_ll_lock_sched(ppl_core); #endif @@ -672,7 +673,7 @@ __cold int ipc_pipeline_free(struct ipc *ipc, uint32_t comp_id) if (!cpu_is_me(ipc_pipe->core)) return ipc4_process_on_core(ipc_pipe->core, false); - ret = ipc_pipeline_module_free(ipc_pipe->pipeline->pipeline_id); + ret = ipc_pipeline_module_free(ipc_pipe->pipeline->pipeline_id, ipc_pipe->core); if (ret != IPC4_SUCCESS) { tr_err(&ipc_tr, "module free () failed"); return ret; From 0d60b3646f8592288bce94cfaf72520ff312193b Mon Sep 17 00:00:00 2001 From: Kai Vehmanen Date: Thu, 29 Oct 2026 20:10:48 +0200 Subject: [PATCH 6/8] ipc4: helper: support cross-core binds with user-space LL Cross-core binding was rejected at build time with CONFIG_SOF_USERSPACE_LL. Implement the LL blocking for it: The user-space LL thread holds its scheduler lock for the whole LL cycle, so taking the LL lock of a core both blocks its next cycle and waits for the current one to finish. ll_block() / ll_unblock() now take the two modules and lock the cores of both modules and of their pipelines, in ascending order to avoid deadlocks. ll_wait_finished_on_core() doesn't need to send an IDC anymore. pipeline_connect() / pipeline_disconnect() asserted the LL lock of the buffer core. Cross-core buffers always belong to the primary core, which isn't locked for e.g. a bind between cores 1 and 2, nor when ipc_pipeline_module_free() disconnects a DP module on core 0 in a pipeline on core 1. The modified lists are walked by the LL thread of the pipeline core, so assert the lock of that core instead. struct pipeline is in cached memory of the pipeline core, so its core field can be stale on other cores, e.g. on the primary core, which processes cross-core binds. Record the pipeline core in the coherent comp_dev, in the new ipc_config.ppl_core, and read it with comp_ppl_core(). The remote parts of the bind (get_attribute, bind, unbind) go through idc_send_msg(), which forwards them to the user-space IPC thread of the remote core. The domain block / unblock in zephyr_domain.c uses a mutex and a condition variable, embedded in user memory, and isn't used with user-space LL, so don't build it then. Signed-off-by: Kai Vehmanen --- src/audio/pipeline/pipeline-graph.c | 9 +- src/audio/pipeline/static_pipeline_loader.c | 3 + src/include/sof/audio/component.h | 1 + src/include/sof/audio/component_ext.h | 17 ++++ src/ipc/ipc4/helper.c | 98 ++++++++++++++++----- src/schedule/zephyr_domain.c | 14 +-- 6 files changed, 112 insertions(+), 30 deletions(-) diff --git a/src/audio/pipeline/pipeline-graph.c b/src/audio/pipeline/pipeline-graph.c index 27d56e12af7f..2c78d8920df7 100644 --- a/src/audio/pipeline/pipeline-graph.c +++ b/src/audio/pipeline/pipeline-graph.c @@ -285,9 +285,12 @@ static void buffer_set_comp(struct comp_buffer *buffer, struct comp_dev *comp, * lock via user_ll_lock_sched()/ll_block() while modifying pipeline * connections, which provides mutual exclusion with the LL thread. No * additional lock is taken here; instead assert that the lock is held. + * That's the lock of the pipeline core, whose LL thread walks the buffer + * lists of the pipeline modules. Not of the buffer core: cross-core + * buffers belong to the primary core. */ #define PPL_LOCK_DECLARE -#define PPL_LOCK(x) user_ll_assert_locked(x) +#define PPL_LOCK(comp) user_ll_assert_locked(comp_ppl_core(comp)) #define PPL_UNLOCK() #else /* @@ -314,7 +317,7 @@ __cold int pipeline_connect(struct comp_dev *comp, struct comp_buffer *buffer, else comp_info(comp, "connect buffer %d as source", buf_get_id(buffer)); - PPL_LOCK(buffer->core); + PPL_LOCK(comp); comp_list = comp_buffer_list(comp, dir); ret = buffer_attach(buffer, comp_list, dir); @@ -343,7 +346,7 @@ __cold void pipeline_disconnect(struct comp_dev *comp, struct comp_buffer *buffe else comp_dbg(comp, "disconnect buffer %d as source", buf_get_id(buffer)); - PPL_LOCK(buffer->core); + PPL_LOCK(comp); comp_list = comp_buffer_list(comp, dir); buffer_detach(buffer, comp_list, dir); diff --git a/src/audio/pipeline/static_pipeline_loader.c b/src/audio/pipeline/static_pipeline_loader.c index dcd7ae88eb0f..7d1a29afd4e9 100644 --- a/src/audio/pipeline/static_pipeline_loader.c +++ b/src/audio/pipeline/static_pipeline_loader.c @@ -564,6 +564,9 @@ static int sof_static_topology_init_comps(const struct sof_static_topology *topo .id = cdesc->id, .pipeline_id = cdesc->pipeline_id, .core = comp_core, +#if CONFIG_IPC_MAJOR_4 + .ppl_core = comp_core, +#endif .proc_domain = COMP_PROCESSING_DOMAIN_LL, .frame_fmt = cdesc->caps.default_fmt, .type = drv_type, diff --git a/src/include/sof/audio/component.h b/src/include/sof/audio/component.h index e8e549913906..e879888053b0 100644 --- a/src/include/sof/audio/component.h +++ b/src/include/sof/audio/component.h @@ -571,6 +571,7 @@ struct comp_ipc_config { #if CONFIG_IPC_MAJOR_4 bool ipc_extended_init; /**< True if extended init is included in IPC payload */ uint32_t ipc_config_size; /**< Size of config received by IPC */ + uint32_t ppl_core; /**< Core of the parent pipeline */ #endif }; diff --git a/src/include/sof/audio/component_ext.h b/src/include/sof/audio/component_ext.h index 913e69c62c9c..c0496650df2a 100644 --- a/src/include/sof/audio/component_ext.h +++ b/src/include/sof/audio/component_ext.h @@ -515,6 +515,23 @@ static inline void comp_make_shared(struct comp_dev *dev) dev->is_shared = true; } +/** + * \brief Get the core of the parent pipeline of a component. + * + * Use this instead of dev->pipeline->core on other cores than the pipeline + * core: struct pipeline isn't coherent between cores, struct comp_dev is. + * \param[in] dev Component device. + * \return Pipeline core, or the component core if it has no pipeline. + */ +static inline uint32_t comp_ppl_core(const struct comp_dev *dev) +{ +#if CONFIG_IPC_MAJOR_4 + return dev->ipc_config.ppl_core; +#else + return dev->ipc_config.core; +#endif +} + /** * \brief Get the global component driver list. * \return Pointer to the component driver list. diff --git a/src/ipc/ipc4/helper.c b/src/ipc/ipc4/helper.c index 62c0b3d46e4d..d85fc71d644b 100644 --- a/src/ipc/ipc4/helper.c +++ b/src/ipc/ipc4/helper.c @@ -132,6 +132,7 @@ __cold static int ipc4_comp_new_config(struct comp_ipc_config *ipc_config, ipc_config->id = comp_id; ipc_config->pipeline_id = module_init->extension.r.ppl_instance_id; ipc_config->core = module_init->extension.r.core_id; + ipc_config->ppl_core = ipc_config->core; ipc_config->ipc_config_size = module_init->extension.r.param_block_size * sizeof(uint32_t); ipc_config->ipc_extended_init = module_init->extension.r.extended_init; if (ipc_config->ipc_config_size > MAILBOX_HOSTBOX_SIZE) { @@ -177,6 +178,8 @@ __cold static int ipc4_comp_new_config(struct comp_ipc_config *ipc_config, (uint32_t)ipc_config->pipeline_id, (uint32_t)ppl->core); return -EINVAL; } + + ipc_config->ppl_core = ppl->core; } return 0; @@ -735,19 +738,69 @@ __cold static struct comp_buffer *ipc4_create_buffer(struct comp_dev *src, bool */ #if CONFIG_SOF_USERSPACE_LL -#error "CONFIG_SOF_USERSPACE_LL not compatible with cross-core streams" +/* + * Cores, whose LL threads can access the two modules or the buffer between + * them: the cores of the modules and of their pipelines. The pipeline core + * differs from the module core for DP modules on another core. + */ +__cold static uint32_t ll_block_cores(struct comp_dev *src, struct comp_dev *dst) +{ + assert_can_be_cold(); + + return BIT(src->ipc_config.core) | BIT(dst->ipc_config.core) | + BIT(comp_ppl_core(src)) | BIT(comp_ppl_core(dst)); +} + +/* + * The LL thread holds its scheduler lock for the whole LL cycle, so taking + * the lock both blocks the next cycle and waits for the current one to + * finish. Take the locks in ascending core order to avoid deadlocks. + */ +__cold static void ll_lock_cores(uint32_t cores) +{ + int core; + + assert_can_be_cold(); + + for (core = 0; core < CONFIG_CORE_COUNT; core++) + if (cores & BIT(core)) + user_ll_lock_sched(core); +} + +__cold static void ll_unlock_cores(uint32_t cores) +{ + int core; + + assert_can_be_cold(); + + for (core = CONFIG_CORE_COUNT - 1; core >= 0; core--) + if (cores & BIT(core)) + user_ll_unlock_sched(core); +} + +#define ll_block(src, dst, flags) \ + do { \ + ll_lock_cores(ll_block_cores(src, dst)); \ + ARG_UNUSED(flags); \ + } while (0) + +#define ll_unblock(src, dst, flags) \ + do { \ + ll_unlock_cores(ll_block_cores(src, dst)); \ + ARG_UNUSED(flags); \ + } while (0) #else -#define ll_block(src_core, dst_core, flags) \ +#define ll_block(src, dst, flags) \ do { \ - if (src_core != dst_core) \ + if ((src)->ipc_config.core != (dst)->ipc_config.core) \ domain_block(sof_get()->platform_timer_domain); \ else \ irq_local_disable(flags); \ } while (0) -#define ll_unblock(src_core, dst_core, flags) \ +#define ll_unblock(src, dst, flags) \ do { \ - if (src_core != dst_core) \ + if ((src)->ipc_config.core != (dst)->ipc_config.core) \ domain_unblock(sof_get()->platform_timer_domain); \ else \ irq_local_enable(flags); \ @@ -759,6 +812,10 @@ __cold static struct comp_buffer *ipc4_create_buffer(struct comp_dev *src, bool */ __cold static int ll_wait_finished_on_core(struct comp_dev *dev) { +#if CONFIG_SOF_USERSPACE_LL + /* ll_block() has already waited for the LL cycle to finish */ + return 0; +#else /* To make sure (blocked) LL has finished its current cycle, it is * enough to send any blocking IDC to the core. Since IDC task has lower * priority then LL thread and cannot preempt it, execution of IDC task @@ -782,25 +839,26 @@ __cold static int ll_wait_finished_on_core(struct comp_dev *dev) } return 0; +#endif } #else /* CONFIG_CROSS_CORE_STREAM */ #if CONFIG_SOF_USERSPACE_LL -/* note: cross-core streams are disabled so src_core==dst_core */ -#define ll_block(src_core, dst_core, flags) \ +/* note: cross-core streams are disabled so src and dst are on the same core */ +#define ll_block(src, dst, flags) \ do { \ - user_ll_lock_sched(src_core); \ + user_ll_lock_sched((src)->ipc_config.core); \ ARG_UNUSED(flags); \ } while(0) -#define ll_unblock(src_core, dst_core, flags) \ +#define ll_unblock(src, dst, flags) \ do { \ - user_ll_unlock_sched(src_core); \ + user_ll_unlock_sched((src)->ipc_config.core); \ ARG_UNUSED(flags); \ } while(0) #else -#define ll_block(src_core, dst_core, flags) irq_local_disable(flags) -#define ll_unblock(src_core, dst_core, flags) irq_local_enable(flags) +#define ll_block(src, dst, flags) irq_local_disable(flags) +#define ll_unblock(src, dst, flags) irq_local_enable(flags) #endif #endif /* CONFIG_CROSS_CORE_STREAM */ @@ -1000,7 +1058,7 @@ __cold int ipc4_comp_connect(struct ipc *ipc, const struct ipc4_module_bind_unbi * blocked on corresponding core(s) to prevent IPC or IDC task getting preempted which * could result in buffers being only half connected when a pipeline task gets executed. */ - ll_block(source->ipc_config.core, sink->ipc_config.core, flags); + ll_block(source, sink, flags); #if CONFIG_CROSS_CORE_STREAM if (cross_core_bind) { @@ -1054,7 +1112,7 @@ __cold int ipc4_comp_connect(struct ipc *ipc, const struct ipc4_module_bind_unbi source->direction_set = true; } - ll_unblock(source->ipc_config.core, sink->ipc_config.core, flags); + ll_unblock(source, sink, flags); return IPC4_SUCCESS; @@ -1068,7 +1126,7 @@ __cold int ipc4_comp_connect(struct ipc *ipc, const struct ipc4_module_bind_unbi e_sink_connect: pipeline_disconnect(source, buffer, PPL_CONN_DIR_COMP_TO_BUFFER); free: - ll_unblock(source->ipc_config.core, sink->ipc_config.core, flags); + ll_unblock(source, sink, flags); buffer_free(buffer); return IPC4_INVALID_RESOURCE_STATE; } @@ -1139,24 +1197,24 @@ __cold int ipc4_comp_disconnect(struct ipc *ipc, const struct ipc4_module_bind_u * IPC or IDC task getting preempted which could result in buffers being only half connected * when a pipeline task gets executed. */ - ll_block(src->ipc_config.core, sink->ipc_config.core, flags); + ll_block(src, sink, flags); if (cross_core_unbind) { #if CONFIG_CROSS_CORE_STREAM /* Make sure LL has finished on both cores */ if (!cpu_is_me(src->ipc_config.core)) if (ll_wait_finished_on_core(src) < 0) { - ll_unblock(src->ipc_config.core, sink->ipc_config.core, flags); + ll_unblock(src, sink, flags); return IPC4_FAILURE; } if (!cpu_is_me(sink->ipc_config.core)) if (ll_wait_finished_on_core(sink) < 0) { - ll_unblock(src->ipc_config.core, sink->ipc_config.core, flags); + ll_unblock(src, sink, flags); return IPC4_FAILURE; } #else tr_err(&ipc_tr, "Cross-core binding is disabled"); - ll_unblock(src->ipc_config.core, sink->ipc_config.core, flags); + ll_unblock(src, sink, flags); return IPC4_FAILURE; #endif /* CONFIG_CROSS_CORE_STREAM */ } @@ -1173,7 +1231,7 @@ __cold int ipc4_comp_disconnect(struct ipc *ipc, const struct ipc4_module_bind_u unbind_data.source = audio_buffer_get_source(&buffer->audio_buffer); ret1 = comp_unbind(sink, &unbind_data); - ll_unblock(src->ipc_config.core, sink->ipc_config.core, flags); + ll_unblock(src, sink, flags); buffer_free(buffer); diff --git a/src/schedule/zephyr_domain.c b/src/schedule/zephyr_domain.c index 5ceee36fa98a..6c8ed5c66b80 100644 --- a/src/schedule/zephyr_domain.c +++ b/src/schedule/zephyr_domain.c @@ -63,7 +63,7 @@ struct zephyr_domain { #endif struct zephyr_domain_thread domain_thread[CONFIG_CORE_COUNT]; struct ll_schedule_domain *ll_domain; -#if CONFIG_CROSS_CORE_STREAM +#if CONFIG_CROSS_CORE_STREAM && !CONFIG_SOF_USERSPACE_LL atomic_t block; struct k_mutex block_mutex; struct k_condvar block_condvar; @@ -106,7 +106,7 @@ static void zephyr_domain_thread_fn(void *p1, void *p2, void *p3) cycles0 = k_cycle_get_32(); #endif -#if CONFIG_CROSS_CORE_STREAM +#if CONFIG_CROSS_CORE_STREAM && !CONFIG_SOF_USERSPACE_LL /* * If zephyr_domain->block is set -- block LL scheduler from starting its * next cycle. @@ -124,7 +124,7 @@ static void zephyr_domain_thread_fn(void *p1, void *p2, void *p3) &zephyr_domain->block_mutex, K_FOREVER); k_mutex_unlock(&zephyr_domain->block_mutex); } -#endif /* CONFIG_CROSS_CORE_STREAM */ +#endif /* CONFIG_CROSS_CORE_STREAM && !CONFIG_SOF_USERSPACE_LL */ if (dt->handler) dt->handler(dt->arg); @@ -539,7 +539,7 @@ struct k_thread *zephyr_ll_domain_thread(void) #endif /* CONFIG_SOF_USERSPACE_LL */ -#if CONFIG_CROSS_CORE_STREAM +#if CONFIG_CROSS_CORE_STREAM && !CONFIG_SOF_USERSPACE_LL static void zephyr_domain_block(struct ll_schedule_domain *domain) { struct zephyr_domain *zephyr_domain = ll_sch_domain_get_pdata(domain); @@ -562,7 +562,7 @@ static void zephyr_domain_unblock(struct ll_schedule_domain *domain) k_condvar_broadcast(&zephyr_domain->block_condvar); k_mutex_unlock(&zephyr_domain->block_mutex); } -#endif /* CONFIG_CROSS_CORE_STREAM */ +#endif /* CONFIG_CROSS_CORE_STREAM && !CONFIG_SOF_USERSPACE_LL */ APP_TASK_DATA static const struct ll_schedule_domain_ops zephyr_domain_ops = { #ifdef CONFIG_SOF_USERSPACE_LL @@ -574,7 +574,7 @@ APP_TASK_DATA static const struct ll_schedule_domain_ops zephyr_domain_ops = { .domain_register = zephyr_domain_register, .domain_unregister = zephyr_domain_unregister, #endif -#if CONFIG_CROSS_CORE_STREAM +#if CONFIG_CROSS_CORE_STREAM && !CONFIG_SOF_USERSPACE_LL .domain_block = zephyr_domain_block, .domain_unblock = zephyr_domain_unblock, #endif @@ -625,7 +625,7 @@ struct ll_schedule_domain *zephyr_domain_init(int clk) zephyr_domain->timer = &zephyr_domain->timer_obj; #endif -#if CONFIG_CROSS_CORE_STREAM +#if CONFIG_CROSS_CORE_STREAM && !CONFIG_SOF_USERSPACE_LL atomic_set(&zephyr_domain->block, 0); k_mutex_init(&zephyr_domain->block_mutex); k_condvar_init(&zephyr_domain->block_condvar); From a1b2369a20847567211cde17d2d027df5c1538fe Mon Sep 17 00:00:00 2001 From: Kai Vehmanen Date: Mon, 2 Nov 2026 13:57:42 +0200 Subject: [PATCH 7/8] ipc4: allow DP modules on another core with user-space LL With CONFIG_CROSS_CORE_STREAM and CONFIG_SOF_USERSPACE_LL, a DP module on a different core than its pipeline can now be bound and operated across cores. With user-space LL, keep rejecting only LL modules on another core than their pipeline. Such an LL module would need its own LL task on that core, which is only set up by the kernel-mode IDC handlers. comp_buffer_connect() marked a component shared only when the buffer is on another core. Cross-core buffers always belong to the primary core, so e.g. a DP module on core 0 in a pipeline on core 1 was never marked shared, and its operations, called on the pipeline core, were run there instead of on core 0. Also mark a component shared, if it's on another core than its pipeline. Signed-off-by: Kai Vehmanen --- src/ipc/ipc-helper.c | 10 ++++++++++ src/ipc/ipc4/helper.c | 11 ++++++++--- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/src/ipc/ipc-helper.c b/src/ipc/ipc-helper.c index 3480f113a4a2..ddba0b918bb9 100644 --- a/src/ipc/ipc-helper.c +++ b/src/ipc/ipc-helper.c @@ -221,6 +221,16 @@ __cold int comp_buffer_connect(struct comp_dev *comp, uint32_t comp_core, comp_make_shared(comp); } + /* + * A component on another core than its pipeline (e.g. a DP module) + * must be shared too, so that its operations, called on the pipeline + * core, are passed to its own core. The buffer check above misses it, + * if the component is on the primary core, which owns all cross-core + * buffers. + */ + if (comp_ppl_core(comp) != comp_core && !comp->is_shared) + comp_make_shared(comp); + return pipeline_connect(comp, buffer, dir); } diff --git a/src/ipc/ipc4/helper.c b/src/ipc/ipc4/helper.c index d85fc71d644b..01fe70702227 100644 --- a/src/ipc/ipc4/helper.c +++ b/src/ipc/ipc4/helper.c @@ -169,11 +169,16 @@ __cold static int ipc4_comp_new_config(struct comp_ipc_config *ipc_config, /* Without cross-core streams a stream is processed entirely on a * single core, so a module cannot live on another core than its - * pipeline. + * pipeline. With user-space LL, only DP modules can run on another + * core: an LL module would need an own LL task on that core, which + * is set up by the IDC handlers in kernel mode only. */ - if (!IS_ENABLED(CONFIG_CROSS_CORE_STREAM) && ppl->core != ipc_config->core) { + if (ppl->core != ipc_config->core && + (!IS_ENABLED(CONFIG_CROSS_CORE_STREAM) || + (IS_ENABLED(CONFIG_SOF_USERSPACE_LL) && + ipc_config->proc_domain != COMP_PROCESSING_DOMAIN_DP))) { tr_err(&ipc_tr, - "comp 0x%x: core %u != pipeline %u core %u, cross-core disabled", + "comp 0x%x: core %u != pipeline %u core %u, not supported", comp_id, (uint32_t)ipc_config->core, (uint32_t)ipc_config->pipeline_id, (uint32_t)ppl->core); return -EINVAL; From 6d2890d20df1608c4b88eed81d5e6fd6a3b5a893 Mon Sep 17 00:00:00 2001 From: Kai Vehmanen Date: Thu, 29 Oct 2026 20:10:48 +0200 Subject: [PATCH 8/8] app: ll_userspace_overlay: enable cross-core streams Cross-core streams are now supported with user-space LL. Stop disabling CONFIG_CROSS_CORE_STREAM in the LL user-space overlay. This allows e.g. the compressed playback topology, which runs the decoder (DP) on core 1, bound to the host copier on core 0. The overlay is used by all ACE3.x and newer targets (e.g. PTL, WCL, NVL) built with os_linux_overlay.conf. Tested on PTL. Signed-off-by: Kai Vehmanen --- app/ll_userspace_overlay.conf | 1 - 1 file changed, 1 deletion(-) diff --git a/app/ll_userspace_overlay.conf b/app/ll_userspace_overlay.conf index 79ebcff70252..df7835e24b54 100644 --- a/app/ll_userspace_overlay.conf +++ b/app/ll_userspace_overlay.conf @@ -41,7 +41,6 @@ CONFIG_SOF_TELEMETRY_IO_PERFORMANCE_MEASUREMENTS=n CONFIG_SOF_BOOT_TEST_ALLOWED=n # misc features not yet compatible with user-space -CONFIG_CROSS_CORE_STREAM=n CONFIG_INTEL_ADSP_MIC_PRIVACY=n CONFIG_XRUN_NOTIFICATIONS_ENABLE=n