diff --git a/app/ll_userspace_overlay.conf b/app/ll_userspace_overlay.conf index 79ebcff70252..df7835e24b54 100644 --- a/app/ll_userspace_overlay.conf +++ b/app/ll_userspace_overlay.conf @@ -41,7 +41,6 @@ CONFIG_SOF_TELEMETRY_IO_PERFORMANCE_MEASUREMENTS=n CONFIG_SOF_BOOT_TEST_ALLOWED=n # misc features not yet compatible with user-space -CONFIG_CROSS_CORE_STREAM=n CONFIG_INTEL_ADSP_MIC_PRIVACY=n CONFIG_XRUN_NOTIFICATIONS_ENABLE=n diff --git a/src/audio/pipeline/pipeline-graph.c b/src/audio/pipeline/pipeline-graph.c index 27d56e12af7f..2c78d8920df7 100644 --- a/src/audio/pipeline/pipeline-graph.c +++ b/src/audio/pipeline/pipeline-graph.c @@ -285,9 +285,12 @@ static void buffer_set_comp(struct comp_buffer *buffer, struct comp_dev *comp, * lock via user_ll_lock_sched()/ll_block() while modifying pipeline * connections, which provides mutual exclusion with the LL thread. No * additional lock is taken here; instead assert that the lock is held. + * That's the lock of the pipeline core, whose LL thread walks the buffer + * lists of the pipeline modules. Not of the buffer core: cross-core + * buffers belong to the primary core. */ #define PPL_LOCK_DECLARE -#define PPL_LOCK(x) user_ll_assert_locked(x) +#define PPL_LOCK(comp) user_ll_assert_locked(comp_ppl_core(comp)) #define PPL_UNLOCK() #else /* @@ -314,7 +317,7 @@ __cold int pipeline_connect(struct comp_dev *comp, struct comp_buffer *buffer, else comp_info(comp, "connect buffer %d as source", buf_get_id(buffer)); - PPL_LOCK(buffer->core); + PPL_LOCK(comp); comp_list = comp_buffer_list(comp, dir); ret = buffer_attach(buffer, comp_list, dir); @@ -343,7 +346,7 @@ __cold void pipeline_disconnect(struct comp_dev *comp, struct comp_buffer *buffe else comp_dbg(comp, "disconnect buffer %d as source", buf_get_id(buffer)); - PPL_LOCK(buffer->core); + PPL_LOCK(comp); comp_list = comp_buffer_list(comp, dir); buffer_detach(buffer, comp_list, dir); diff --git a/src/audio/pipeline/static_pipeline_loader.c b/src/audio/pipeline/static_pipeline_loader.c index dcd7ae88eb0f..7d1a29afd4e9 100644 --- a/src/audio/pipeline/static_pipeline_loader.c +++ b/src/audio/pipeline/static_pipeline_loader.c @@ -564,6 +564,9 @@ static int sof_static_topology_init_comps(const struct sof_static_topology *topo .id = cdesc->id, .pipeline_id = cdesc->pipeline_id, .core = comp_core, +#if CONFIG_IPC_MAJOR_4 + .ppl_core = comp_core, +#endif .proc_domain = COMP_PROCESSING_DOMAIN_LL, .frame_fmt = cdesc->caps.default_fmt, .type = drv_type, diff --git a/src/idc/idc.c b/src/idc/idc.c index 8fdecb8ba173..5a6745719db2 100644 --- a/src/idc/idc.c +++ b/src/idc/idc.c @@ -84,51 +84,39 @@ static void idc_ipc(void) ipc_cmd(ipc->comp_data); } -static int idc_ipc4_bind(uint32_t comp_id) +static int idc_ipc4_bind(uint32_t comp_id, void *payload) { struct ipc_comp_dev *ipc_dev; - struct idc_payload *payload; - struct bind_info *bu; + struct bind_info *bu = payload; ipc_dev = ipc_get_comp_by_id(ipc_get(), comp_id); if (!ipc_dev) return -ENODEV; - payload = idc_payload_get(*idc_get(), cpu_get_id()); - bu = (struct bind_info *)payload; - return comp_bind(ipc_dev->cd, bu); } -static int idc_ipc4_unbind(uint32_t comp_id) +static int idc_ipc4_unbind(uint32_t comp_id, void *payload) { struct ipc_comp_dev *ipc_dev; - struct idc_payload *payload; - struct bind_info *bu; + struct bind_info *bu = payload; ipc_dev = ipc_get_comp_by_id(ipc_get(), comp_id); if (!ipc_dev) return -ENODEV; - payload = idc_payload_get(*idc_get(), cpu_get_id()); - bu = (struct bind_info *)payload; - return comp_unbind(ipc_dev->cd, bu); } -static int idc_get_attribute(uint32_t comp_id) +static int idc_get_attribute(uint32_t comp_id, void *payload) { struct ipc_comp_dev *ipc_dev; - struct idc_payload *idc_payload; - struct get_attribute_remote_payload *get_attr_payload; + struct get_attribute_remote_payload *get_attr_payload = payload; ipc_dev = ipc_get_comp_by_id(ipc_get(), comp_id); if (!ipc_dev) return -ENODEV; - idc_payload = idc_payload_get(*idc_get(), cpu_get_id()); - get_attr_payload = (struct get_attribute_remote_payload *)idc_payload; - return comp_get_attribute(ipc_dev->cd, get_attr_payload->type, get_attr_payload->value); } @@ -137,14 +125,11 @@ static int idc_get_attribute(uint32_t comp_id) * \param[in] comp_id Component id to have params set. * \return Error code. */ -static int idc_params(uint32_t comp_id) +static int idc_params(uint32_t comp_id, void *payload) { struct ipc *ipc = ipc_get(); struct ipc_comp_dev *ipc_dev; - struct idc *idc = *idc_get(); - struct idc_payload *payload = idc_payload_get(idc, cpu_get_id()); - struct sof_ipc_stream_params *params = - (struct sof_ipc_stream_params *)payload; + struct sof_ipc_stream_params *params = payload; int ret; ipc_dev = ipc_get_comp_by_id(ipc, comp_id); @@ -185,6 +170,12 @@ static int idc_prepare(uint32_t comp_id) /* we're running LL on different core, so allocate our own task */ if (!dev->task && dev->ipc_config.proc_domain == COMP_PROCESSING_DOMAIN_LL) { +#ifdef CONFIG_SOF_USERSPACE_LL + /* user-space LL modules must run on their pipeline core */ + tr_err(&idc_tr, "comp %#x: LL module on a remote core", comp_id); + ret = -EINVAL; + goto out; +#endif /* allocate task for shared component */ dev->task = sof_heap_alloc(dev->drv->user_heap, SOF_MEM_FLAG_USER, sizeof(*dev->task), 0); @@ -218,12 +209,10 @@ static int idc_prepare(uint32_t comp_id) * \param[in] comp_id Component id to be triggered. * \return Error code. */ -static int idc_trigger(uint32_t comp_id) +static int idc_trigger(uint32_t comp_id, void *payload) { struct ipc *ipc = ipc_get(); struct ipc_comp_dev *ipc_dev; - struct idc *idc = *idc_get(); - struct idc_payload *payload = idc_payload_get(idc, cpu_get_id()); uint32_t cmd = *(uint32_t *)payload; ipc_dev = ipc_get_comp_by_id(ipc, comp_id); @@ -356,6 +345,7 @@ static void idc_secondary_core_crashed(const uint32_t header) */ void idc_cmd(struct idc_msg *msg) { + void *payload = idc_payload_get(*idc_get(), cpu_get_id()); uint32_t type = iTS(msg->header); int ret = 0; @@ -377,25 +367,25 @@ void idc_cmd(struct idc_msg *msg) dbg_path_hot_stop_watching(); break; case iTS(IDC_MSG_BIND): - ret = idc_ipc4_bind(msg->extension); + ret = idc_ipc4_bind(msg->extension, payload); break; case iTS(IDC_MSG_UNBIND): - ret = idc_ipc4_unbind(msg->extension); + ret = idc_ipc4_unbind(msg->extension, payload); break; case iTS(IDC_MSG_GET_ATTRIBUTE): - ret = idc_get_attribute(msg->extension); + ret = idc_get_attribute(msg->extension, payload); break; case iTS(IDC_MSG_FREE): ret = idc_comp_free(msg->extension); break; case iTS(IDC_MSG_PARAMS): - ret = idc_params(msg->extension); + ret = idc_params(msg->extension, payload); break; case iTS(IDC_MSG_PREPARE): ret = idc_prepare(msg->extension); break; case iTS(IDC_MSG_TRIGGER): - ret = idc_trigger(msg->extension); + ret = idc_trigger(msg->extension, payload); break; case iTS(IDC_MSG_RESET): ret = idc_reset(msg->extension); @@ -421,6 +411,42 @@ void idc_cmd(struct idc_msg *msg) idc_msg_status_set(ret, cpu_get_id()); } +#ifdef CONFIG_SOF_USERSPACE_LL +/** + * \brief Executes a component IDC message in a user-space thread. + * \param[in] msg IDC message, forwarded by a user thread on another core. + * \param[in,out] payload Message payload. + * \return Error code. + * + * Only the component messages, that can be sent from the user-space IPC + * and LL threads, are supported. + */ +int idc_cmd_user(const struct idc_msg *msg, void *payload) +{ + switch (iTS(msg->header)) { + case iTS(IDC_MSG_BIND): + return idc_ipc4_bind(msg->extension, payload); + case iTS(IDC_MSG_UNBIND): + return idc_ipc4_unbind(msg->extension, payload); + case iTS(IDC_MSG_GET_ATTRIBUTE): + return idc_get_attribute(msg->extension, payload); + case iTS(IDC_MSG_FREE): + return idc_comp_free(msg->extension); + case iTS(IDC_MSG_PARAMS): + return idc_params(msg->extension, payload); + case iTS(IDC_MSG_PREPARE): + return idc_prepare(msg->extension); + case iTS(IDC_MSG_TRIGGER): + return idc_trigger(msg->extension, payload); + case iTS(IDC_MSG_RESET): + return idc_reset(msg->extension); + default: + tr_err(&idc_tr, "unsupported user msg->header = %#x", msg->header); + return -EINVAL; + } +} +#endif /* CONFIG_SOF_USERSPACE_LL */ + /* Runs on each CPU */ __cold int idc_init(void) { diff --git a/src/idc/zephyr_idc.c b/src/idc/zephyr_idc.c index bba2d32a5ffd..576d18f4243f 100644 --- a/src/idc/zephyr_idc.c +++ b/src/idc/zephyr_idc.c @@ -127,7 +127,7 @@ static struct zephyr_idc_msg idc_work[CONFIG_CORE_COUNT * 2]; /* Protect the above array */ static K_MUTEX_DEFINE(idc_mutex); -int idc_send_msg(struct idc_msg *msg, uint32_t mode) +static int idc_send_msg_kernel(struct idc_msg *msg, uint32_t mode) { struct idc *idc = *idc_get(); struct idc_payload *payload = idc_payload_get(idc, msg->core); @@ -200,6 +200,21 @@ int idc_send_msg(struct idc_msg *msg, uint32_t mode) return ret; } +int idc_send_msg(struct idc_msg *msg, uint32_t mode) +{ +#ifdef CONFIG_SOF_USERSPACE_LL + /* + * IDC uses kernel objects and data. User-space threads instead pass + * the message to the user-space IPC thread of the target core. Such + * messages are always processed synchronously. + */ + if (k_is_user_context()) + return ipc_user_xcore_send(msg); +#endif + + return idc_send_msg_kernel(msg, mode); +} + __cold void idc_init_thread(void) { char thread_name[] = "idc_p4wq0"; diff --git a/src/include/sof/audio/component.h b/src/include/sof/audio/component.h index e8e549913906..e879888053b0 100644 --- a/src/include/sof/audio/component.h +++ b/src/include/sof/audio/component.h @@ -571,6 +571,7 @@ struct comp_ipc_config { #if CONFIG_IPC_MAJOR_4 bool ipc_extended_init; /**< True if extended init is included in IPC payload */ uint32_t ipc_config_size; /**< Size of config received by IPC */ + uint32_t ppl_core; /**< Core of the parent pipeline */ #endif }; diff --git a/src/include/sof/audio/component_ext.h b/src/include/sof/audio/component_ext.h index 3a4c3ce8826e..c0496650df2a 100644 --- a/src/include/sof/audio/component_ext.h +++ b/src/include/sof/audio/component_ext.h @@ -18,8 +18,9 @@ #include #include -/* TODO: the cross-core helpers below send IDC messages, which is kernel only */ +/* in user-space LL builds idc_send_msg() forwards to the remote user IPC thread */ #include +#include #include /** \addtogroup component_infra_api @@ -299,9 +300,18 @@ static inline int comp_ipc4_get_attribute_remote(struct comp_dev *dev, uint32_t if (type != COMP_ATTR_BASE_CONFIG) return -EINVAL; +#ifdef CONFIG_SOF_USERSPACE_LL + /* must be writable by the user-space thread on the remote core */ + base_cfg = sof_heap_alloc(zephyr_ll_user_heap(), SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, + sizeof(*base_cfg), 0); + if (!base_cfg) + return -ENOMEM; + memset(base_cfg, 0, sizeof(*base_cfg)); +#else base_cfg = rzalloc(SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, sizeof(*base_cfg)); if (!base_cfg) return -ENOMEM; +#endif payload.type = type; payload.value = base_cfg; @@ -312,7 +322,11 @@ static inline int comp_ipc4_get_attribute_remote(struct comp_dev *dev, uint32_t memcpy_s(value, sizeof(struct ipc4_base_module_cfg), base_cfg, sizeof(struct ipc4_base_module_cfg)); +#ifdef CONFIG_SOF_USERSPACE_LL + sof_heap_free(zephyr_ll_user_heap(), base_cfg); +#else rfree(base_cfg); +#endif return ret; } #endif /* CONFIG_IPC_MAJOR_4 */ @@ -501,6 +515,23 @@ static inline void comp_make_shared(struct comp_dev *dev) dev->is_shared = true; } +/** + * \brief Get the core of the parent pipeline of a component. + * + * Use this instead of dev->pipeline->core on other cores than the pipeline + * core: struct pipeline isn't coherent between cores, struct comp_dev is. + * \param[in] dev Component device. + * \return Pipeline core, or the component core if it has no pipeline. + */ +static inline uint32_t comp_ppl_core(const struct comp_dev *dev) +{ +#if CONFIG_IPC_MAJOR_4 + return dev->ipc_config.ppl_core; +#else + return dev->ipc_config.core; +#endif +} + /** * \brief Get the global component driver list. * \return Pointer to the component driver list. diff --git a/src/include/sof/ipc/common.h b/src/include/sof/ipc/common.h index 696cb553661d..cdf4fd44c764 100644 --- a/src/include/sof/ipc/common.h +++ b/src/include/sof/ipc/common.h @@ -62,6 +62,31 @@ extern struct tr_ctx ipc_tr; #define IPC_DSP_CORE_COUNT 1 #endif +/** @brief Maximum payload size of a cross-core request between user threads */ +#define IPC_USER_XCORE_PAYLOAD_SIZE 128 + +/** + * @brief Cross-core request slot of a user-space IPC thread. + * + * User-space threads can't use IDC. Instead, they pass IDC messages for + * another core to the user-space IPC thread of that core via this slot. + */ +struct ipc_user_xcore { + /** @brief Serializes senders to this core */ + struct k_sem *lock; + /** @brief Signalled by the target core when the request is done */ + struct k_sem *done; + /** @brief IDC message header and extension */ + uint32_t header; + uint32_t extension; + /** @brief Result from the target core */ + int result; + /** @brief Copy of IPC4 bind / unbind message, referenced by the payload */ + uint32_t ipc4_bind_data[2]; + /** @brief Copy of the IDC message payload */ + uint8_t payload[IPC_USER_XCORE_PAYLOAD_SIZE] __aligned(8); +}; + struct ipc_user { struct k_thread *thread[IPC_DSP_CORE_COUNT]; struct k_sem *sem; @@ -96,6 +121,8 @@ struct ipc_user { * Size verified by BUILD_ASSERT in handler-user.c. */ uint8_t init_drv_data[160] __aligned(4); + /** @brief Cross-core request slots, one per target core */ + struct ipc_user_xcore xcore[IPC_DSP_CORE_COUNT]; }; struct ipc { diff --git a/src/include/sof/schedule/ll_schedule_domain.h b/src/include/sof/schedule/ll_schedule_domain.h index c369242640af..3860f5c96bf7 100644 --- a/src/include/sof/schedule/ll_schedule_domain.h +++ b/src/include/sof/schedule/ll_schedule_domain.h @@ -229,6 +229,8 @@ void zephyr_ll_user_resources_init(void); /** * @brief Grant thread access to per-core Low-Latency mutexes. + * + * Kernel context only. * @param[in,out] thread Target thread. * @param[in] core Target core ID. */ diff --git a/src/ipc/ipc-common.c b/src/ipc/ipc-common.c index bbc4a9b8a7d3..c20d3935dfdc 100644 --- a/src/ipc/ipc-common.c +++ b/src/ipc/ipc-common.c @@ -32,6 +32,9 @@ #include #include #include +#if CONFIG_IPC_MAJOR_4 +#include +#endif #include #include #include @@ -423,6 +426,15 @@ void ipc_schedule_process(struct ipc *ipc) #define IPC_USER_EVENT_CMD BIT(0) #define IPC_USER_EVENT_STOP BIT(1) +#define IPC_USER_EVENT_XCORE BIT(2) + +BUILD_ASSERT(IPC_USER_XCORE_PAYLOAD_SIZE >= IDC_MAX_PAYLOAD_SIZE, + "cross-core request payload smaller than IDC payload"); +#if CONFIG_IPC_MAJOR_4 +BUILD_ASSERT(sizeof(((struct ipc_user_xcore *)0)->ipc4_bind_data) == + sizeof(struct ipc4_module_bind_unbind), + "cross-core request bind data size mismatch"); +#endif SOF_DEFINE_REG_UUID(sec_core_init); @@ -438,8 +450,13 @@ static struct ipc_user *ipc_user_kptr; static struct k_sem *ipc_user_ksem; static struct k_event *ipc_user_kevent[CONFIG_CORE_COUNT]; static struct k_thread *ipc_user_kthread[CONFIG_CORE_COUNT]; +static struct k_thread *ipc_user_kaudio_thread[CONFIG_CORE_COUNT]; static bool ipc_user_kinit_needed[CONFIG_CORE_COUNT]; +/* Cross-core request slot semaphores, static so they're never freed */ +static struct k_sem ipc_user_xcore_lock[CONFIG_CORE_COUNT]; +static struct k_sem ipc_user_xcore_done[CONFIG_CORE_COUNT]; + /** * @brief Forward an IPC command to the user-space thread. * @@ -474,8 +491,11 @@ int ipc_user_forward_cmd(uint32_t primary, uint32_t extension, unsigned int core ipc->task_mask |= IPC_TASK_IN_THREAD; k_spin_unlock(&ipc->lock, key); - /* Wake the user thread */ - k_event_set(ipc_user_kevent[core], IPC_USER_EVENT_CMD); + /* + * Wake the user thread. Post, not set: a cross-core request + * (IPC_USER_EVENT_XCORE) may be pending on the same event. + */ + k_event_post(ipc_user_kevent[core], IPC_USER_EVENT_CMD); /* Wait for user thread to complete */ ret = k_sem_take(ipc_user_ksem, K_MSEC(100)); @@ -493,6 +513,67 @@ int ipc_user_forward_cmd(uint32_t primary, uint32_t extension, unsigned int core return !ret ? pdata->result : ret; } +/** + * @brief Pass an IDC message to the user-space IPC thread of another core. + * + * Called from user-space threads (IPC or LL) instead of idc_send_msg(), + * which is kernel-only. Blocks until the target core has processed the + * message and returns its result. + * + * @param msg IDC message + * @return Result from the target core, or negative error code + */ +int ipc_user_xcore_send(const struct idc_msg *msg) +{ + struct ipc_user *ipc_user = ipc_get()->ipc_user_pdata; + struct ipc_user_xcore *xc; + int ret; + + if (msg->core >= CONFIG_CORE_COUNT || msg->size > sizeof(xc->payload) || + !ipc_user->event[msg->core]) + return -EINVAL; + + xc = &ipc_user->xcore[msg->core]; + + k_sem_take(xc->lock, K_FOREVER); + + xc->header = msg->header; + xc->extension = msg->extension; + if (msg->size) { + ret = memcpy_s(xc->payload, sizeof(xc->payload), msg->payload, msg->size); + if (ret < 0) + goto out; + } + +#if CONFIG_IPC_MAJOR_4 + /* + * The bind / unbind message usually is on the sender's stack, which + * the target thread can't access. Pass a copy of it. + */ + if (iTS(msg->header) == iTS(IDC_MSG_BIND) || iTS(msg->header) == iTS(IDC_MSG_UNBIND)) { + struct bind_info *bind_data = (struct bind_info *)xc->payload; + + if (bind_data->ipc4_data) { + ret = memcpy_s(xc->ipc4_bind_data, sizeof(xc->ipc4_bind_data), + bind_data->ipc4_data, sizeof(*bind_data->ipc4_data)); + if (ret < 0) + goto out; + bind_data->ipc4_data = + (const struct ipc4_module_bind_unbind *)xc->ipc4_bind_data; + } + } +#endif + + /* post, not set: don't clear a pending IPC_USER_EVENT_CMD */ + k_event_post(ipc_user->event[msg->core], IPC_USER_EVENT_XCORE); + k_sem_take(xc->done, K_FOREVER); + ret = xc->result; +out: + k_sem_give(xc->lock); + + return ret; +} + /** * @brief Protocol-specific dispatch of a forwarded IPC command. * @@ -524,11 +605,24 @@ static void ipc_user_thread_fn(void *p1, void *p2, void *p3) for (;;) { uint32_t mask = k_event_wait_safe(ipc_user->event[core], - IPC_USER_EVENT_CMD | IPC_USER_EVENT_STOP, + IPC_USER_EVENT_CMD | IPC_USER_EVENT_STOP | + IPC_USER_EVENT_XCORE, false, K_FOREVER); LOG_DBG("IPC user wake, mask %u", mask); + if (mask & IPC_USER_EVENT_XCORE) { + struct ipc_user_xcore *xc = &ipc_user->xcore[core]; + const struct idc_msg msg = { + .header = xc->header, + .extension = xc->extension, + .core = core, + }; + + xc->result = idc_cmd_user(&msg, xc->payload); + k_sem_give(xc->done); + } + if (mask & IPC_USER_EVENT_CMD) { ipc_user->result = ipc_user_thread_dispatch(ipc_user); @@ -605,6 +699,55 @@ __cold static int ipc_user_init_thread(struct ipc_user *ipc_user, unsigned int c return ret; } +/* + * Grant the threads of the newly initialized core and of the other already + * initialized cores access to each other's cross-core request slots and LL + * scheduler locks. Uses only the kernel-only copies of the thread state. + */ +__cold static void ipc_user_xcore_grant(unsigned int core) +{ + struct k_thread *new_threads[] = { + ipc_user_kthread[core], ipc_user_kaudio_thread[core], + }; + unsigned int c, i; + + assert_can_be_cold(); + + for (c = 0; c < CONFIG_CORE_COUNT; c++) { + struct k_thread *threads[] = { + ipc_user_kthread[c], ipc_user_kaudio_thread[c], + }; + + /* + * Grant the slot semaphores of all cores. Note: braces needed, + * k_thread_access_grant() expands to multiple statements. + */ + for (i = 0; i < ARRAY_SIZE(new_threads); i++) { + if (new_threads[i]) { + k_thread_access_grant(new_threads[i], &ipc_user_xcore_lock[c], + &ipc_user_xcore_done[c]); + } + } + + if (!ipc_user_kthread[c]) + continue; + + for (i = 0; i < ARRAY_SIZE(threads); i++) { + if (!threads[i]) + continue; + k_thread_access_grant(threads[i], ipc_user_kevent[core]); + user_ll_grant_access(threads[i], core); + } + + for (i = 0; i < ARRAY_SIZE(new_threads); i++) { + if (!new_threads[i]) + continue; + k_thread_access_grant(new_threads[i], ipc_user_kevent[c]); + user_ll_grant_access(new_threads[i], c); + } + } +} + __cold int ipc_user_init_secondary(unsigned int core) { struct ipc_user *ipc_user = ipc_user_kptr; @@ -640,7 +783,9 @@ __cold int ipc_user_init_secondary(unsigned int core) } k_thread_access_grant(ipc_user_kthread[core], audio_thread); + ipc_user_kaudio_thread[core] = audio_thread; ipc_user->audio_thread[core] = audio_thread; + ipc_user_xcore_grant(core); return 0; } @@ -740,6 +885,15 @@ __cold static void ipc_user_init(void) k_sem_init(ipc_user_ksem, 0, 1); + for (core = 0; core < CONFIG_CORE_COUNT; core++) { + struct ipc_user_xcore *xc = &ipc_user->xcore[core]; + + xc->lock = &ipc_user_xcore_lock[core]; + xc->done = &ipc_user_xcore_done[core]; + k_sem_init(xc->lock, 1, 1); + k_sem_init(xc->done, 0, 1); + } + ret = ipc_user_init_thread(ipc_user, PLATFORM_PRIMARY_CORE_ID); if (ret < 0) { LOG_ERR("user IPC thread initialization failed"); @@ -776,7 +930,9 @@ __cold static void ipc_user_init(void) * k_thread_access_grant(audio_thread, dai_mutex) from user context. */ k_thread_access_grant(ipc_user_kthread[PLATFORM_PRIMARY_CORE_ID], audio_thread); + ipc_user_kaudio_thread[PLATFORM_PRIMARY_CORE_ID] = audio_thread; ipc_user->audio_thread[PLATFORM_PRIMARY_CORE_ID] = audio_thread; + ipc_user_xcore_grant(PLATFORM_PRIMARY_CORE_ID); k_thread_start(ipc_user_kthread[PLATFORM_PRIMARY_CORE_ID]); diff --git a/src/ipc/ipc-helper.c b/src/ipc/ipc-helper.c index 3480f113a4a2..ddba0b918bb9 100644 --- a/src/ipc/ipc-helper.c +++ b/src/ipc/ipc-helper.c @@ -221,6 +221,16 @@ __cold int comp_buffer_connect(struct comp_dev *comp, uint32_t comp_core, comp_make_shared(comp); } + /* + * A component on another core than its pipeline (e.g. a DP module) + * must be shared too, so that its operations, called on the pipeline + * core, are passed to its own core. The buffer check above misses it, + * if the component is on the primary core, which owns all cross-core + * buffers. + */ + if (comp_ppl_core(comp) != comp_core && !comp->is_shared) + comp_make_shared(comp); + return pipeline_connect(comp, buffer, dir); } diff --git a/src/ipc/ipc4/helper.c b/src/ipc/ipc4/helper.c index d5a0e095b3f4..29fd6f3798d9 100644 --- a/src/ipc/ipc4/helper.c +++ b/src/ipc/ipc4/helper.c @@ -132,6 +132,7 @@ __cold static int ipc4_comp_new_config(struct comp_ipc_config *ipc_config, ipc_config->id = comp_id; ipc_config->pipeline_id = module_init->extension.r.ppl_instance_id; ipc_config->core = module_init->extension.r.core_id; + ipc_config->ppl_core = ipc_config->core; ipc_config->ipc_config_size = module_init->extension.r.param_block_size * sizeof(uint32_t); ipc_config->ipc_extended_init = module_init->extension.r.extended_init; if (ipc_config->ipc_config_size > MAILBOX_HOSTBOX_SIZE) { @@ -168,15 +169,22 @@ __cold static int ipc4_comp_new_config(struct comp_ipc_config *ipc_config, /* Without cross-core streams a stream is processed entirely on a * single core, so a module cannot live on another core than its - * pipeline. + * pipeline. With user-space LL, only DP modules can run on another + * core: an LL module would need an own LL task on that core, which + * is set up by the IDC handlers in kernel mode only. */ - if (!IS_ENABLED(CONFIG_CROSS_CORE_STREAM) && ppl->core != ipc_config->core) { + if (ppl->core != ipc_config->core && + (!IS_ENABLED(CONFIG_CROSS_CORE_STREAM) || + (IS_ENABLED(CONFIG_SOF_USERSPACE_LL) && + ipc_config->proc_domain != COMP_PROCESSING_DOMAIN_DP))) { tr_err(&ipc_tr, - "comp 0x%x: core %u != pipeline %u core %u, cross-core disabled", + "comp 0x%x: core %u != pipeline %u core %u, not supported", comp_id, (uint32_t)ipc_config->core, (uint32_t)ipc_config->pipeline_id, (uint32_t)ppl->core); return -EINVAL; } + + ipc_config->ppl_core = ppl->core; } return 0; @@ -589,14 +597,11 @@ __cold static inline int ipc_comp_free_remote(struct comp_dev *dev) return idc_send_msg(&msg, IDC_BLOCKING); } -__cold static int ipc_pipeline_module_free(uint32_t pipeline_id) +__cold static int ipc_pipeline_module_free(uint32_t pipeline_id, uint32_t ppl_core) { struct ipc *ipc = ipc_get(); struct ipc_comp_dev *icd; int ret; -#ifdef CONFIG_SOF_USERSPACE_LL - int ppl_core; -#endif assert_can_be_cold(); @@ -605,8 +610,14 @@ __cold static int ipc_pipeline_module_free(uint32_t pipeline_id) return IPC4_SUCCESS; #ifdef CONFIG_SOF_USERSPACE_LL - ppl_core = icd->core; + /* + * Lock the pipeline core's LL scheduler. Not the core of the first + * module, which can be a remote one: freeing a remote module takes + * the LL lock of its core on that core. + */ user_ll_lock_sched(ppl_core); +#else + ARG_UNUSED(ppl_core); #endif while (icd) { @@ -672,7 +683,7 @@ __cold int ipc_pipeline_free(struct ipc *ipc, uint32_t comp_id) if (!cpu_is_me(ipc_pipe->core)) return ipc4_process_on_core(ipc_pipe->core, false); - ret = ipc_pipeline_module_free(ipc_pipe->pipeline->pipeline_id); + ret = ipc_pipeline_module_free(ipc_pipe->pipeline->pipeline_id, ipc_pipe->core); if (ret != IPC4_SUCCESS) { tr_err(&ipc_tr, "module free () failed"); return ret; @@ -734,19 +745,69 @@ __cold static struct comp_buffer *ipc4_create_buffer(struct comp_dev *src, bool */ #if CONFIG_SOF_USERSPACE_LL -#error "CONFIG_SOF_USERSPACE_LL not compatible with cross-core streams" +/* + * Cores, whose LL threads can access the two modules or the buffer between + * them: the cores of the modules and of their pipelines. The pipeline core + * differs from the module core for DP modules on another core. + */ +__cold static uint32_t ll_block_cores(struct comp_dev *src, struct comp_dev *dst) +{ + assert_can_be_cold(); + + return BIT(src->ipc_config.core) | BIT(dst->ipc_config.core) | + BIT(comp_ppl_core(src)) | BIT(comp_ppl_core(dst)); +} + +/* + * The LL thread holds its scheduler lock for the whole LL cycle, so taking + * the lock both blocks the next cycle and waits for the current one to + * finish. Take the locks in ascending core order to avoid deadlocks. + */ +__cold static void ll_lock_cores(uint32_t cores) +{ + int core; + + assert_can_be_cold(); + + for (core = 0; core < CONFIG_CORE_COUNT; core++) + if (cores & BIT(core)) + user_ll_lock_sched(core); +} + +__cold static void ll_unlock_cores(uint32_t cores) +{ + int core; + + assert_can_be_cold(); + + for (core = CONFIG_CORE_COUNT - 1; core >= 0; core--) + if (cores & BIT(core)) + user_ll_unlock_sched(core); +} + +#define ll_block(src, dst, flags) \ + do { \ + ll_lock_cores(ll_block_cores(src, dst)); \ + ARG_UNUSED(flags); \ + } while (0) + +#define ll_unblock(src, dst, flags) \ + do { \ + ll_unlock_cores(ll_block_cores(src, dst)); \ + ARG_UNUSED(flags); \ + } while (0) #else -#define ll_block(src_core, dst_core, flags) \ +#define ll_block(src, dst, flags) \ do { \ - if (src_core != dst_core) \ + if ((src)->ipc_config.core != (dst)->ipc_config.core) \ domain_block(sof_get()->platform_timer_domain); \ else \ irq_local_disable(flags); \ } while (0) -#define ll_unblock(src_core, dst_core, flags) \ +#define ll_unblock(src, dst, flags) \ do { \ - if (src_core != dst_core) \ + if ((src)->ipc_config.core != (dst)->ipc_config.core) \ domain_unblock(sof_get()->platform_timer_domain); \ else \ irq_local_enable(flags); \ @@ -758,6 +819,10 @@ __cold static struct comp_buffer *ipc4_create_buffer(struct comp_dev *src, bool */ __cold static int ll_wait_finished_on_core(struct comp_dev *dev) { +#if CONFIG_SOF_USERSPACE_LL + /* ll_block() has already waited for the LL cycle to finish */ + return 0; +#else /* To make sure (blocked) LL has finished its current cycle, it is * enough to send any blocking IDC to the core. Since IDC task has lower * priority then LL thread and cannot preempt it, execution of IDC task @@ -781,25 +846,26 @@ __cold static int ll_wait_finished_on_core(struct comp_dev *dev) } return 0; +#endif } #else /* CONFIG_CROSS_CORE_STREAM */ #if CONFIG_SOF_USERSPACE_LL -/* note: cross-core streams are disabled so src_core==dst_core */ -#define ll_block(src_core, dst_core, flags) \ +/* note: cross-core streams are disabled so src and dst are on the same core */ +#define ll_block(src, dst, flags) \ do { \ - user_ll_lock_sched(src_core); \ + user_ll_lock_sched((src)->ipc_config.core); \ ARG_UNUSED(flags); \ } while(0) -#define ll_unblock(src_core, dst_core, flags) \ +#define ll_unblock(src, dst, flags) \ do { \ - user_ll_unlock_sched(src_core); \ + user_ll_unlock_sched((src)->ipc_config.core); \ ARG_UNUSED(flags); \ } while(0) #else -#define ll_block(src_core, dst_core, flags) irq_local_disable(flags) -#define ll_unblock(src_core, dst_core, flags) irq_local_enable(flags) +#define ll_block(src, dst, flags) irq_local_disable(flags) +#define ll_unblock(src, dst, flags) irq_local_enable(flags) #endif #endif /* CONFIG_CROSS_CORE_STREAM */ @@ -999,7 +1065,7 @@ __cold int ipc4_comp_connect(struct ipc *ipc, const struct ipc4_module_bind_unbi * blocked on corresponding core(s) to prevent IPC or IDC task getting preempted which * could result in buffers being only half connected when a pipeline task gets executed. */ - ll_block(source->ipc_config.core, sink->ipc_config.core, flags); + ll_block(source, sink, flags); #if CONFIG_CROSS_CORE_STREAM if (cross_core_bind) { @@ -1053,7 +1119,7 @@ __cold int ipc4_comp_connect(struct ipc *ipc, const struct ipc4_module_bind_unbi source->direction_set = true; } - ll_unblock(source->ipc_config.core, sink->ipc_config.core, flags); + ll_unblock(source, sink, flags); return IPC4_SUCCESS; @@ -1067,7 +1133,7 @@ __cold int ipc4_comp_connect(struct ipc *ipc, const struct ipc4_module_bind_unbi e_sink_connect: pipeline_disconnect(source, buffer, PPL_CONN_DIR_COMP_TO_BUFFER); free: - ll_unblock(source->ipc_config.core, sink->ipc_config.core, flags); + ll_unblock(source, sink, flags); buffer_free(buffer); return IPC4_INVALID_RESOURCE_STATE; } @@ -1138,24 +1204,24 @@ __cold int ipc4_comp_disconnect(struct ipc *ipc, const struct ipc4_module_bind_u * IPC or IDC task getting preempted which could result in buffers being only half connected * when a pipeline task gets executed. */ - ll_block(src->ipc_config.core, sink->ipc_config.core, flags); + ll_block(src, sink, flags); if (cross_core_unbind) { #if CONFIG_CROSS_CORE_STREAM /* Make sure LL has finished on both cores */ if (!cpu_is_me(src->ipc_config.core)) if (ll_wait_finished_on_core(src) < 0) { - ll_unblock(src->ipc_config.core, sink->ipc_config.core, flags); + ll_unblock(src, sink, flags); return IPC4_FAILURE; } if (!cpu_is_me(sink->ipc_config.core)) if (ll_wait_finished_on_core(sink) < 0) { - ll_unblock(src->ipc_config.core, sink->ipc_config.core, flags); + ll_unblock(src, sink, flags); return IPC4_FAILURE; } #else tr_err(&ipc_tr, "Cross-core binding is disabled"); - ll_unblock(src->ipc_config.core, sink->ipc_config.core, flags); + ll_unblock(src, sink, flags); return IPC4_FAILURE; #endif /* CONFIG_CROSS_CORE_STREAM */ } @@ -1172,7 +1238,7 @@ __cold int ipc4_comp_disconnect(struct ipc *ipc, const struct ipc4_module_bind_u unbind_data.source = audio_buffer_get_source(&buffer->audio_buffer); ret1 = comp_unbind(sink, &unbind_data); - ll_unblock(src->ipc_config.core, sink->ipc_config.core, flags); + ll_unblock(src, sink, flags); buffer_free(buffer); diff --git a/src/schedule/zephyr_domain.c b/src/schedule/zephyr_domain.c index 5ceee36fa98a..6c8ed5c66b80 100644 --- a/src/schedule/zephyr_domain.c +++ b/src/schedule/zephyr_domain.c @@ -63,7 +63,7 @@ struct zephyr_domain { #endif struct zephyr_domain_thread domain_thread[CONFIG_CORE_COUNT]; struct ll_schedule_domain *ll_domain; -#if CONFIG_CROSS_CORE_STREAM +#if CONFIG_CROSS_CORE_STREAM && !CONFIG_SOF_USERSPACE_LL atomic_t block; struct k_mutex block_mutex; struct k_condvar block_condvar; @@ -106,7 +106,7 @@ static void zephyr_domain_thread_fn(void *p1, void *p2, void *p3) cycles0 = k_cycle_get_32(); #endif -#if CONFIG_CROSS_CORE_STREAM +#if CONFIG_CROSS_CORE_STREAM && !CONFIG_SOF_USERSPACE_LL /* * If zephyr_domain->block is set -- block LL scheduler from starting its * next cycle. @@ -124,7 +124,7 @@ static void zephyr_domain_thread_fn(void *p1, void *p2, void *p3) &zephyr_domain->block_mutex, K_FOREVER); k_mutex_unlock(&zephyr_domain->block_mutex); } -#endif /* CONFIG_CROSS_CORE_STREAM */ +#endif /* CONFIG_CROSS_CORE_STREAM && !CONFIG_SOF_USERSPACE_LL */ if (dt->handler) dt->handler(dt->arg); @@ -539,7 +539,7 @@ struct k_thread *zephyr_ll_domain_thread(void) #endif /* CONFIG_SOF_USERSPACE_LL */ -#if CONFIG_CROSS_CORE_STREAM +#if CONFIG_CROSS_CORE_STREAM && !CONFIG_SOF_USERSPACE_LL static void zephyr_domain_block(struct ll_schedule_domain *domain) { struct zephyr_domain *zephyr_domain = ll_sch_domain_get_pdata(domain); @@ -562,7 +562,7 @@ static void zephyr_domain_unblock(struct ll_schedule_domain *domain) k_condvar_broadcast(&zephyr_domain->block_condvar); k_mutex_unlock(&zephyr_domain->block_mutex); } -#endif /* CONFIG_CROSS_CORE_STREAM */ +#endif /* CONFIG_CROSS_CORE_STREAM && !CONFIG_SOF_USERSPACE_LL */ APP_TASK_DATA static const struct ll_schedule_domain_ops zephyr_domain_ops = { #ifdef CONFIG_SOF_USERSPACE_LL @@ -574,7 +574,7 @@ APP_TASK_DATA static const struct ll_schedule_domain_ops zephyr_domain_ops = { .domain_register = zephyr_domain_register, .domain_unregister = zephyr_domain_unregister, #endif -#if CONFIG_CROSS_CORE_STREAM +#if CONFIG_CROSS_CORE_STREAM && !CONFIG_SOF_USERSPACE_LL .domain_block = zephyr_domain_block, .domain_unblock = zephyr_domain_unblock, #endif @@ -625,7 +625,7 @@ struct ll_schedule_domain *zephyr_domain_init(int clk) zephyr_domain->timer = &zephyr_domain->timer_obj; #endif -#if CONFIG_CROSS_CORE_STREAM +#if CONFIG_CROSS_CORE_STREAM && !CONFIG_SOF_USERSPACE_LL atomic_set(&zephyr_domain->block, 0); k_mutex_init(&zephyr_domain->block_mutex); k_condvar_init(&zephyr_domain->block_condvar); diff --git a/src/schedule/zephyr_ll.c b/src/schedule/zephyr_ll.c index 3edebbca5b20..4c4fd7059c30 100644 --- a/src/schedule/zephyr_ll.c +++ b/src/schedule/zephyr_ll.c @@ -56,6 +56,13 @@ struct zephyr_ll_pdata { */ static APP_SYSUSER_BSS struct k_mutex *zephyr_ll_locks[CONFIG_CORE_COUNT]; +/* + * Kernel-only copy of zephyr_ll_locks[]. That array is writable by user + * threads, so kernel code granting access to the locks must use this copy, + * otherwise a user thread could make it grant access to any kernel object. + */ +static struct k_mutex *zephyr_ll_klocks[CONFIG_CORE_COUNT]; + /* * Shadow ownership tracking for the per-core LL mutex, used only to * power user_ll_assert_locked(). The kernel k_mutex object is not @@ -703,6 +710,7 @@ static void zephyr_ll_scheduler_free(void *data, uint32_t flags) #if CONFIG_SOF_USERSPACE_LL zephyr_ll_locks[sch->core] = NULL; + zephyr_ll_klocks[sch->core] = NULL; k_object_free(sch->lock); #endif @@ -776,9 +784,9 @@ struct task *zephyr_ll_task_alloc(void) void user_ll_grant_access(struct k_thread *thread, int core) { - assert(core < CONFIG_CORE_COUNT && zephyr_ll_locks[core] != NULL); + assert(core < CONFIG_CORE_COUNT && zephyr_ll_klocks[core] != NULL); - k_thread_access_grant(thread, zephyr_ll_locks[core]); + k_thread_access_grant(thread, zephyr_ll_klocks[core]); } /** @@ -908,6 +916,7 @@ __cold int zephyr_ll_scheduler_init(struct ll_schedule_domain *domain) } assert(core < CONFIG_CORE_COUNT && zephyr_ll_locks[core] == NULL); zephyr_ll_locks[core] = sch->lock; + zephyr_ll_klocks[core] = sch->lock; k_mutex_init(sch->lock); tr_dbg(&ll_tr, "ll-scheduler init done, sch %p sch->lock %p", sch, sch->lock); diff --git a/zephyr/include/rtos/idc.h b/zephyr/include/rtos/idc.h index 2afe69d8e186..f47025ff5bf6 100644 --- a/zephyr/include/rtos/idc.h +++ b/zephyr/include/rtos/idc.h @@ -202,6 +202,11 @@ enum task_state idc_do_cmd(void *data); void idc_cmd(struct idc_msg *msg); +#ifdef CONFIG_SOF_USERSPACE_LL +int idc_cmd_user(const struct idc_msg *msg, void *payload); +int ipc_user_xcore_send(const struct idc_msg *msg); +#endif + int idc_msg_status_get(uint32_t core); int idc_init(void);