diff --git a/src/audio/buffers/audio_buffer.c b/src/audio/buffers/audio_buffer.c index 1ecf8472dd65..f8a3087b0e64 100644 --- a/src/audio/buffers/audio_buffer.c +++ b/src/audio/buffers/audio_buffer.c @@ -24,7 +24,10 @@ int audio_buffer_attach_secondary_buffer(struct sof_audio_buffer *buffer, bool at_input, struct sof_audio_buffer *secondary_buffer) { - if (buffer->secondary_buffer_sink || buffer->secondary_buffer_source) + /* check per-side: allow attaching on both sides (needed for DP-to-DP) */ + if (at_input && buffer->secondary_buffer_sink) + return -EINVAL; + if (!at_input && buffer->secondary_buffer_source) return -EINVAL; /* secondary buffer must share audio params with the primary buffer */ @@ -48,6 +51,50 @@ int audio_buffer_sync_secondary_buffer(struct sof_audio_buffer *buffer, size_t l struct sof_source *data_src; struct sof_sink *data_dst; + if (buffer->secondary_buffer_sink && buffer->secondary_buffer_source) { + /* + * DP-to-DP case: both secondary buffers present. + * Data flows: input_ring_buffer -> comp_buffer -> output_ring_buffer + * + * This buffer may be synced by two DP modules during the same LL cycle: + * - The source DP module syncs it via comp_dev_for_each_consumer (output) + * - The sink DP module syncs it via comp_dev_for_each_producer (input) + * + * Both steps run in order (source DP first, then sink DP). Performing + * them both here in a single call ensures atomicity and correct + * rate-limiting. The second call for the same buffer will be a no-op + * since the comp_buffer will be empty. + * + * Step 1: copy from input secondary buffer to primary (comp_buffer). + * No limit on input side - copy all available data. + */ + data_src = audio_buffer_get_source(buffer->secondary_buffer_sink); + data_dst = &buffer->_sink_api; + + size_t data_available = source_get_data_available(data_src); + size_t free_size = sink_get_free_size(data_dst); + size_t to_copy = MIN(data_available, free_size); + + err = source_to_sink_copy(data_src, data_dst, true, to_copy); + if (err) + return err; + + /* + * Step 2: copy from primary (comp_buffer) to output secondary buffer. + * Apply the limit to the output side to control how much data + * is made available to the downstream DP module per LL cycle. + */ + data_src = &buffer->_source_api; + data_dst = audio_buffer_get_sink(buffer->secondary_buffer_source); + + data_available = source_get_data_available(data_src); + free_size = sink_get_free_size(data_dst); + to_copy = MIN(MIN(data_available, free_size), limit); + + err = source_to_sink_copy(data_src, data_dst, true, to_copy); + return err; + } + if (buffer->secondary_buffer_sink) { /* * audio_buffer sink API is shadowed, that means there's a secondary_buffer @@ -203,18 +250,16 @@ uint32_t audio_buffer_sink_get_lft(struct sof_sink *sink) return us_in_buffer; /* - * TODO, Currently there's no DP to DP connection - * >>> the code below is never accessible and won't work because of cache incoherence <<< - * - * to make DP to DP connection possible: + * NOTE: DP-to-DP connections are now supported via dual ring_buffers + * attached as secondary buffers on both sides of a comp_buffer. + * Data cascades: ring_buf_src -> comp_buffer -> ring_buf_sink + * with syncing during each LL cycle. * - * 1) module data must be ALWAYS located in non cached memory alias, allowing - * cross core access to params like period (needed below) and calling - * module_get_deadline for the next module, regardless of cores the modules are - * running on - * 2) comp_buffer must be removed from all pipeline code, replaced with a generic abstract - * class audio_buffer - allowing using comp_buffer and ring_buffer without current - * "hybrid buffer" solution + * Future improvements: + * 1) module data should be in non-cached memory alias for reliable + * cross-core access to params like period and deadlines + * 2) comp_buffer should be replaced with generic audio_buffer + * throughout pipeline code (Pipeline 2.0) */ } diff --git a/src/audio/buffers/ring_buffer.c b/src/audio/buffers/ring_buffer.c index 11ebd736dfe4..646ea1d26fde 100644 --- a/src/audio/buffers/ring_buffer.c +++ b/src/audio/buffers/ring_buffer.c @@ -10,6 +10,7 @@ #include #include #include +#include #include #include @@ -85,7 +86,6 @@ static inline void ring_buffer_writeback_shared(struct ring_buffer *ring_buffer, dcache_writeback_region(ptr, size); } - /** * @brief remove the queue from the list, free memory */ @@ -94,11 +94,20 @@ static void ring_buffer_free(struct sof_audio_buffer *audio_buffer) if (!audio_buffer) return; - struct ring_buffer *ring_buffer = container_of(audio_buffer, - struct ring_buffer, audio_buffer); + struct ring_buffer *ring_buffer = + container_of(audio_buffer, struct ring_buffer, audio_buffer); + struct k_heap *heap = audio_buffer->heap; + + sof_heap_free(heap, (__sparse_force void *)ring_buffer->_data_buffer); + sof_heap_free(heap, ring_buffer); + + /* decrement DP heap client_count, matching the increment in ipc_comp_connect */ + if (heap) { + struct dp_heap_user *dp_user = container_of(heap, struct dp_heap_user, heap); - sof_heap_free(audio_buffer->heap, (__sparse_force void *)ring_buffer->_data_buffer); - sof_heap_free(audio_buffer->heap, ring_buffer); + if (!--dp_user->client_count) + rfree(dp_user); + } } static void ring_buffer_reset(struct sof_audio_buffer *audio_buffer) diff --git a/src/ipc/ipc4/helper.c b/src/ipc/ipc4/helper.c index a06d3dc7bc59..3f0e8021221e 100644 --- a/src/ipc/ipc4/helper.c +++ b/src/ipc/ipc4/helper.c @@ -643,18 +643,14 @@ __cold int ipc_comp_connect(struct ipc *ipc, ipc_pipe_comp_connect *_connect) struct k_heap *dp_heap; #if CONFIG_ZEPHYR_DP_SCHEDULER - if (source->ipc_config.proc_domain == COMP_PROCESSING_DOMAIN_DP && - sink->ipc_config.proc_domain == COMP_PROCESSING_DOMAIN_DP) { - tr_err(&ipc_tr, "DP to DP binding is not supported: can't bind %x to %x", - src_id, sink_id); - return IPC4_INVALID_REQUEST; - } - + bool src_is_dp = source->ipc_config.proc_domain == COMP_PROCESSING_DOMAIN_DP; + bool sink_is_dp = sink->ipc_config.proc_domain == COMP_PROCESSING_DOMAIN_DP; + bool dp_to_dp = src_is_dp && sink_is_dp; struct comp_dev *dp; - if (sink->ipc_config.proc_domain == COMP_PROCESSING_DOMAIN_DP) + if (sink_is_dp) dp = sink; - else if (source->ipc_config.proc_domain == COMP_PROCESSING_DOMAIN_DP) + else if (src_is_dp) dp = source; else dp = NULL; @@ -722,8 +718,8 @@ __cold int ipc_comp_connect(struct ipc *ipc, ipc_pipe_comp_connect *_connect) * * size = 2*max(obs of source module, ibs of destination module) * (obs and ibs is single buffer size) - * in case of DP -> LL - * size = 2*ibs of destination (LL) module. DP queue will handle obs of DP module + * in case of DP -> LL or DP -> DP + * size = 2*ibs of destination module. DP queue will handle obs of DP module */ if (source->ipc_config.proc_domain == COMP_PROCESSING_DOMAIN_LL) buf_size = MAX(ibs, obs) * 2; @@ -761,12 +757,13 @@ __cold int ipc_comp_connect(struct ipc *ipc, ipc_pipe_comp_connect *_connect) #if CONFIG_ZEPHYR_DP_SCHEDULER struct ring_buffer *ring_buffer = NULL; - if (sink->ipc_config.proc_domain == COMP_PROCESSING_DOMAIN_DP || - source->ipc_config.proc_domain == COMP_PROCESSING_DOMAIN_DP) { + if (src_is_dp || sink_is_dp) { struct processing_module *srcmod = comp_mod(source); struct module_data *src_module_data = &srcmod->priv; struct processing_module *dstmod = comp_mod(sink); struct module_data *dst_module_data = &dstmod->priv; + bool is_shared = audio_buffer_is_shared(&buffer->audio_buffer); + uint32_t buf_id = buf_get_id(buffer); /* * Handle cases where the size of the ring buffer depends on the @@ -778,16 +775,57 @@ __cold int ipc_comp_connect(struct ipc *ipc, ipc_pipe_comp_connect *_connect) */ ring_buffer = ring_buffer_create(dp, MAX(ibs, dst_module_data->mpd.in_buff_size), MAX(obs, src_module_data->mpd.out_buff_size), - audio_buffer_is_shared(&buffer->audio_buffer), - buf_get_id(buffer)); + is_shared, buf_id); if (!ring_buffer) { buffer_free(buffer); return IPC4_OUT_OF_MEMORY; } + /* track ring_buffer as a client of the DP heap */ + if (dp_heap) { + struct dp_heap_user *dp_rb_user = + container_of(dp_heap, struct dp_heap_user, heap); + + dp_rb_user->client_count++; + } + /* data destination module needs to use ring_buffer */ audio_buffer_attach_secondary_buffer(&buffer->audio_buffer, dp == source, &ring_buffer->audio_buffer); + + /* + * DP-to-DP binding: both source and sink are DP modules. + * A second ring_buffer is needed on the other side of the comp_buffer + * so each DP module has its own lock-free ring_buffer interface. + * Data flows: src_DP → ring_buf_src → comp_buffer → ring_buf_sink → sink_DP + * The comp_buffer acts as the intermediary synced during LL cycles. + */ + if (dp_to_dp) { + struct ring_buffer *ring_buffer2; + struct k_heap *src_heap = source->mod->priv.resources.heap; + + ring_buffer2 = + ring_buffer_create(source, + MAX(ibs, dst_module_data->mpd.in_buff_size), + MAX(obs, src_module_data->mpd.out_buff_size), + is_shared, buf_id); + if (!ring_buffer2) { + buffer_free(buffer); + return IPC4_OUT_OF_MEMORY; + } + + /* track ring_buffer2 on source's DP heap */ + if (src_heap) { + struct dp_heap_user *src_dp_user = + container_of(src_heap, struct dp_heap_user, heap); + + src_dp_user->client_count++; + } + + /* attach second ring_buffer on the source side */ + audio_buffer_attach_secondary_buffer(&buffer->audio_buffer, dp != source, + &ring_buffer2->audio_buffer); + } } #endif /* CONFIG_ZEPHYR_DP_SCHEDULER */