llext: place relocatable LLEXT modules in virtual pages

src/include/sof/lib_manager.h

@@ -119,6 +119,7 @@ struct lib_manager_module {
 	bool domain_dp;
 	struct lib_manager_segment_desc segment[LIB_MANAGER_N_SEGMENTS];
 	struct lib_manager_segment_desc writable_merged;
+	struct lib_manager_segment_desc vma; /* Reserved virtual pages for ET_REL modules */
 };
 
 struct lib_manager_mod_ctx {

src/library_manager/llext_manager.c

@@ -22,6 +22,7 @@
 #include <sof/lib/cpu-clk-manager.h>
 #include <sof/lib_manager.h>
 #include <sof/lib/regions_mm.h>
+#include <sof/lib/vpage.h>
 #include <sof/llext_manager.h>
 #include <sof/audio/module_adapter/module/generic.h>
 #include <sof/audio/module_adapter/module/modules.h>
@@ -299,20 +300,29 @@ __cold static int llext_manager_load_module(struct lib_manager_module *mctx)
 	struct llext_loader *ldr = &mctx->ebl->loader;
 	const struct llext *ext = mctx->llext;
 
-	/* find dedicated virtual memory zone */
-	const struct sys_mm_drv_region *virtual_memory_regions = sys_mm_drv_query_memory_regions();
 	const struct sys_mm_drv_region *virtual_region;
 
-	if (!virtual_memory_regions)
-		return -EFAULT;
+	if (mctx->vma.size) {
+		/* relocated to pages, reserved from the virtual page allocator */
+		virtual_region = vpage_get_region();
+		if (!virtual_region)
+			return -EFAULT;
+	} else {
+		/* find dedicated virtual memory zone */
+		const struct sys_mm_drv_region *virtual_memory_regions =
+			sys_mm_drv_query_memory_regions();
 
-	SYS_MM_DRV_MEMORY_REGION_FOREACH(virtual_memory_regions, virtual_region) {
-		if (virtual_region->attr == VIRTUAL_REGION_LLEXT_LIBRARIES_ATTR)
-			break;
-	}
+		if (!virtual_memory_regions)
+			return -EFAULT;
 
-	if (!virtual_region->size)
-		return -EFAULT;
+		SYS_MM_DRV_MEMORY_REGION_FOREACH(virtual_memory_regions, virtual_region) {
+			if (virtual_region->attr == VIRTUAL_REGION_LLEXT_LIBRARIES_ATTR)
+				break;
+		}
+
+		if (!virtual_region->size)
+			return -EFAULT;
+	}
 
 	/* Copy Code */
 	ret = llext_manager_load_data_from_storage(virtual_region, ldr, ext, LLEXT_MEM_TEXT,
@@ -434,6 +444,99 @@ static bool llext_manager_section_detached(const elf_shdr_t *shdr)
 	return shdr->sh_addr < SOF_MODULE_DRAM_LINK_END;
 }
 
+/*
+ * Relocatable (ET_REL) modules are linked at placeholder addresses. Reserve
+ * virtual pages for all sections, that will be copied to SRAM, and move them
+ * there, keeping their relative placement, so that llext_load() relocates the
+ * module to run at those addresses. Detached sections stay in DRAM and keep
+ * their addresses. Shared object (ET_DYN) modules can only run at addresses,
+ * fixed at build time, they stay in the dedicated virtual memory region.
+ */
+__cold static int llext_manager_place(struct lib_manager_module *mctx)
+{
+	static const uint8_t elf_magic[] = {0x7f, 'E', 'L', 'F'};
+	uint8_t *elf = (uint8_t *)mctx->ebl->buf;
+	size_t len = mctx->ebl->len;
+	const elf_ehdr_t *ehdr = (const elf_ehdr_t *)elf;
+	uintptr_t start = UINTPTR_MAX, end = 0, base;
+	size_t align = PAGE_SZ, size;
+	elf_shdr_t *shdr;
+	unsigned int i;
+	void *vma;
+
+	assert_can_be_cold();
+
+	/* llext_load() validates the image, only check what is used here */
+	if (len < sizeof(*ehdr) || memcmp(ehdr->e_ident, elf_magic, sizeof(elf_magic)))
+		return -ENOEXEC;
+
+	/* without a virtual page region keep the addresses from the build */
+	if (ehdr->e_type != ET_REL || !vpage_get_region())
+		return 0;
+
+	if (ehdr->e_shentsize != sizeof(*shdr) || ehdr->e_shoff > len ||
+	    ehdr->e_shnum > (len - ehdr->e_shoff) / sizeof(*shdr) ||
+	    !IS_ALIGNED(elf + ehdr->e_shoff, sizeof(uint32_t))) {
+		tr_err(&lib_manager_tr, "invalid section header table");
+		return -ENOEXEC;
+	}
+
+	shdr = (elf_shdr_t *)(elf + ehdr->e_shoff);
+
+	for (i = 0; i < ehdr->e_shnum; i++) {
+		if (!(shdr[i].sh_flags & SHF_ALLOC) || !shdr[i].sh_size ||
+		    llext_manager_section_detached(shdr + i))
+			continue;
+
+		if (shdr[i].sh_addr + shdr[i].sh_size < shdr[i].sh_addr ||
+		    !is_power_of_two(shdr[i].sh_addralign ? : 1)) {
+			tr_err(&lib_manager_tr, "invalid section %u", i);
+			return -ENOEXEC;
+		}
+
+		start = MIN(start, shdr[i].sh_addr);
+		end = MAX(end, shdr[i].sh_addr + shdr[i].sh_size);
+		align = MAX(align, shdr[i].sh_addralign);
+	}
+
+	if (!end)
+		return 0;
+
+	/*
+	 * Moving by a multiple of the largest section alignment, at least a
+	 * page, preserves the alignment of all sections. If that alignment is
+	 * larger than a page, reserve extra pages to find an aligned start.
+	 */
+	start = ALIGN_DOWN(start, align);
+	size = ALIGN_UP(end - start, PAGE_SZ) + align - PAGE_SZ;
+
+	vma = vpage_reserve(size / PAGE_SZ);
+	if (!vma) {
+		tr_err(&lib_manager_tr, "cannot reserve %#zx bytes of virtual memory", size);
+		return -ENOMEM;
+	}
+
+	base = ALIGN_UP((uintptr_t)vma, align);
+
+	for (i = 0; i < ehdr->e_shnum; i++) {
+		if (!(shdr[i].sh_flags & SHF_ALLOC) || !shdr[i].sh_size ||
+		    llext_manager_section_detached(shdr + i))
+			continue;
+
+		shdr[i].sh_addr += base - start;
+	}
+
+	/* The ELF image is kept in DRAM over DSP power cycles */
+	sys_cache_data_flush_range(shdr, ehdr->e_shnum * sizeof(*shdr));
+
+	mctx->vma.addr = (uintptr_t)vma;
+	mctx->vma.size = size;
+
+	tr_dbg(&lib_manager_tr, "placed %#lx-%#lx at %#lx", start, end, base);
+
+	return 0;
+}
+
 __cold static int llext_manager_link(const char *name,
 				     struct lib_manager_module *mctx, const void **buildinfo,
 				     const struct sof_man_module_manifest **mod_manifest)
@@ -468,8 +571,23 @@ __cold static int llext_manager_link(const char *name,
 			.section_detached = llext_manager_section_detached,
 			.keep_section_info = true,
 		};
+		bool placed = false;
 
+		if (!*llext) {
+			ret = llext_manager_place(mctx);
+			if (ret < 0)
+				return ret;
+
+			placed = mctx->vma.size;
+		}
+
+		/* a positive value is the use count of an already loaded extension */
 		ret = llext_load(ldr, name, llext, &ldr_parm);
+		if (ret < 0 && placed) {
+			vpage_release((void *)mctx->vma.addr);
+			mctx->vma.addr = 0;
+			mctx->vma.size = 0;
+		}
 		if (ret)
 			return ret;
 	}
@@ -583,6 +701,8 @@ __cold static int llext_manager_mod_init(struct lib_manager_mod_ctx *ctx,
 			ctx->mod[n_mod].llext = NULL;
 			ctx->mod[n_mod].ebl = NULL;
 			ctx->mod[n_mod].n_dependent = 0;
+			ctx->mod[n_mod].vma.addr = 0;
+			ctx->mod[n_mod].vma.size = 0;
 			ctx->mod[n_mod].start_idx = i;
 			if (n_mod)
 				ctx->mod[n_mod - 1].n_mod = i - ctx->mod[n_mod - 1].start_idx;

src/library_manager/llext_manager_dram.c

@@ -5,6 +5,7 @@
 #include <rtos/alloc.h>
 
 #include <sof/lib_manager.h>
+#include <sof/lib/vpage.h>
 #include <ipc/topology.h>
 
 #include <zephyr/llext/buf_loader.h>
@@ -232,6 +233,18 @@ int llext_manager_restore_from_dram(void)
 				/* Not instantiated - nothing to restore */
 				continue;
 
+			/*
+			 * The virtual page allocator starts out empty on every
+			 * boot, reserve the pages the module is linked at again
+			 */
+			if (mod[k].vma.size &&
+			    vpage_reserve_at((void *)mod[k].vma.addr,
+					     mod[k].vma.size / CONFIG_MM_DRV_PAGE_SIZE) < 0) {
+				tr_err(&lib_manager_tr, "cannot reclaim %#zx bytes at %#lx",
+				       mod[k].vma.size, mod[k].vma.addr);
+				goto nomem;
+			}
+
 			/* Loaders are supplied by the caller */
 			struct llext_buf_loader *bldr = rmalloc(SOF_MEM_FLAG_KERNEL | SOF_MEM_FLAG_COHERENT,
 								sizeof(*bldr));
@@ -329,8 +342,11 @@ int llext_manager_restore_from_dram(void)
 			continue;
 
 		for (k = 0; k < ctx->n_mod; k++) {
-			if (mod[k].llext)
+			if (mod[k].llext) {
 				llext_unload(&mod[k].llext);
+				if (mod[k].vma.size)
+					vpage_release((void *)mod[k].vma.addr);
+			}
 
 			if (mod[k].ebl)
 				rfree(mod[k].ebl);

zephyr/CMakeLists.txt

@@ -283,8 +283,12 @@ if (CONFIG_SOC_SERIES_INTEL_ADSP_ACE)
 		${SOF_PLATFORM_PATH}/novalake/lib/clk.c
 	)
 
+	# the LLEXT library manager reserves virtual pages for loadable modules
+	if(CONFIG_SOF_VREGIONS OR CONFIG_LIBRARY_MANAGER)
+		zephyr_library_sources(lib/vpage.c)
+	endif()
+
 	zephyr_library_sources_ifdef(CONFIG_SOF_VREGIONS
-		lib/vpage.c
 		lib/vregion.c
 	)
 

zephyr/include/sof/lib/vpage.h

@@ -11,6 +11,7 @@
 #define __SOF_LIB_VPAGE_H__
 
 #include <zephyr/kernel.h>
+#include <zephyr/drivers/mm/system_mm.h>
 #include <stdint.h>
 
 #ifdef __cplusplus
@@ -41,6 +42,51 @@ void *vpage_alloc(unsigned int pages);
  */
 void vpage_free(void *ptr);
 
+/**
+ * @brief Reserve contiguous virtual pages.
+ *
+ * Reserves a specified number of contiguous virtual memory pages without
+ * mapping any physical pages to them. For users that need to map the pages
+ * themselves, e.g. with different permissions for parts of the range.
+ *
+ * @param[in] pages Number of pages (typically 4KB per page) to reserve.
+ * @return Pointer to the reserved virtual memory region, or NULL on failure.
+ */
+void *vpage_reserve(unsigned int pages);
+
+/**
+ * @brief Reserve contiguous virtual pages at a given address.
+ *
+ * Like vpage_reserve() but for a range at a fixed address, e.g. for users that
+ * had the range reserved before the DSP was powered down and are restored at
+ * the same address.
+ *
+ * @param[in] ptr Page aligned address of the first page to reserve.
+ * @param[in] pages Number of pages (typically 4KB per page) to reserve.
+ * @return 0 on success or a negative error code if any of the pages is already
+ *	   in use or outside of the virtual page region.
+ */
+int vpage_reserve_at(void *ptr, unsigned int pages);
+
+/**
+ * @brief Release reserved virtual pages.
+ *
+ * Releases virtual memory pages, reserved with vpage_reserve() or
+ * vpage_reserve_at(). The caller has to unmap any pages it has mapped first.
+ *
+ * @param[in] ptr Pointer to the first reserved page.
+ */
+void vpage_release(void *ptr);
+
+/**
+ * @brief Get the virtual memory region managed by the allocator.
+ *
+ * Users of vpage_reserve() need it to map pages in the reserved range.
+ *
+ * @return The region, or NULL if no virtual page region could be created.
+ */
+const struct sys_mm_drv_region *vpage_get_region(void);
+
 /** @}*/
 
 #ifdef __cplusplus