Max Krasnyansky
b2dd28a3b6
* hex-mm: fix artificial limit in the solver that restricted number of act-prep threads * hex-mm: fix warning * hex-prof: do not apply --top to the timeline report * hmx-mm: add suport for tiled act-processing to better distribute hvx work * hex-l2: add tracing for l2flush events * workqueue: redo the legacy workpool api to match hmx-queue and dma-queue * hmx-mm: fix f32 activation buffer alignmnet for nhvx=5,6,7 * hex-work: minor cleanup for work-queue apis * hex-work: further cleanup of the work-queue api * hex-l2: optimize l2flushes at the opbatch level * hex-work: remove unused mask * hex-work: no need to drop hvx ctx in the work-queue * hex-work: add explicit wakeup/suspend and make threads spin * hex-bufs: mark any non-weight tensor as compute * hex-dma: dma-queue support for alias queues and cached dma * hex-l2: track tensor aliases and delay or skip flushes as much as possible * hex-l2: simplify tensor alias handling * hex-l2: handle overlapping views as a circular list of aliases * hex-tens: add flags helper * hex-l2: add helper for marking tensors clearn/dirty * hex-l2: mark binary and rope outputs as l2-clean and keep the rest as is for now * hex-l2: proper support for handling all tensor overlap scenarios * hex-trace: instrument matmul init code and cleanup trace checks * hex-thread: introduce dedicated main thread with explicit stack and priority * hex-l2: track dirty state as bitmap and introduce threaded flush * hex-trace: remove redundant checks for ctx != null * hex-l2: allocate entire context as one buffer and l2fetch it after big flushes * hex-l2: disable tensor clearing in binary and rope for now seems to cause issues with fusion * hmx-mm: update act proc to use fastdivs and fix DMA overflow * hmx-mm: make MUL_MAT_ID kernels robust to multi-chunk cases (start_row>0) * hex-queue: remove obsolete queue interfaces and flush hmx-queue at the end of the op-batch * hex-queue: dont use early wakeup for small op-batches * hex-tensors: properly cap max_tensors in op-batches and dirty_map * hex-l2: make sure threaded l2flush does proper rounding * hex-l2: factor out htp_tensor_flush for reuse (if needed) * hex-l2: optimize tensor flushes by coalescing flush-all * hex-l2: optimize multi-threaded flush * hex-drv: futureproof version checks * hexagon: fix errors and warnings on windows * hex-main: update main thread to only use dspqueue_read, dspqueue_peek is not available on some platforms * hex-main: add fallback mode for dspqueue with callbacks * hex-main: introduce fallback mode for using dspqueue callbacks for full op processing * hex-main: remove early wakeup, not helping and seems to cause some errors with certain batch sizes * hex-l2: make sure to use invalidate version of flushall * hex-l2: dont try to trace early l2flush at the start of op-batch * hex-main: remove offset_ctx that must be zero anyway * hex-hmx: fix hmx_queue_depth to use idx_write - idx_read * hex-hmx: use atomic_load for idx_read/write * hex-main: add static assert to make sure n_threads are aligned
107 lines
2.8 KiB
C
107 lines
2.8 KiB
C
#include "dma-queue.h"
|
|
|
|
#include <stdbool.h>
|
|
#include <stdlib.h>
|
|
#include <string.h>
|
|
|
|
#pragma clang diagnostic ignored "-Wunused-function"
|
|
|
|
static inline uint32_t pow2_ceil(uint32_t x) {
|
|
if (x <= 1) {
|
|
return 1;
|
|
}
|
|
int p = 2;
|
|
x--;
|
|
while (x >>= 1) {
|
|
p <<= 1;
|
|
}
|
|
return p;
|
|
}
|
|
|
|
static inline uintptr_t align_up(uintptr_t addr, size_t align) {
|
|
return (addr + align - 1) & ~(align - 1);
|
|
}
|
|
|
|
size_t dma_queue_sizeof(size_t capacity) {
|
|
capacity = pow2_ceil(capacity);
|
|
|
|
size_t size_q = sizeof(dma_queue);
|
|
size_t offset_r = align_up(size_q, HEX_L2_LINE_SIZE);
|
|
size_t size_r = sizeof(dma_ring);
|
|
size_t offset_desc = align_up(offset_r + size_r, HEX_L2_LINE_SIZE);
|
|
size_t size_desc = capacity * sizeof(dma_descriptor_2d);
|
|
size_t offset_dptr = align_up(offset_desc + size_desc, HEX_L2_LINE_SIZE);
|
|
size_t size_dptr = capacity * sizeof(dma_ptr);
|
|
|
|
return offset_dptr + size_dptr;
|
|
}
|
|
|
|
size_t dma_queue_alignof(void) {
|
|
return HEX_L2_LINE_SIZE;
|
|
}
|
|
|
|
dma_queue_t dma_queue_init(void * ptr, size_t capacity, uintptr_t vtcm_base, size_t vtcm_size, struct htp_thread_trace * trace) {
|
|
capacity = pow2_ceil(capacity);
|
|
|
|
size_t size_q = sizeof(dma_queue);
|
|
size_t offset_r = align_up(size_q, HEX_L2_LINE_SIZE);
|
|
size_t size_r = sizeof(dma_ring);
|
|
size_t offset_desc = align_up(offset_r + size_r, HEX_L2_LINE_SIZE);
|
|
size_t size_desc = capacity * sizeof(dma_descriptor_2d);
|
|
size_t offset_dptr = align_up(offset_desc + size_desc, HEX_L2_LINE_SIZE);
|
|
size_t size_dptr = capacity * sizeof(dma_ptr);
|
|
|
|
size_t total_size = offset_dptr + size_dptr;
|
|
memset(ptr, 0, total_size);
|
|
|
|
dma_queue * q = (dma_queue *) ptr;
|
|
dma_ring * r = (dma_ring *) ((uintptr_t) ptr + offset_r);
|
|
|
|
q->ring = r;
|
|
q->nocache = 0;
|
|
q->alias = false;
|
|
|
|
r->trace = trace;
|
|
r->vtcm_base = vtcm_base;
|
|
r->vtcm_end = vtcm_base + vtcm_size;
|
|
r->capacity = capacity;
|
|
r->idx_mask = capacity - 1;
|
|
r->push_idx = 0;
|
|
r->pop_idx = 0;
|
|
|
|
r->desc = (dma_descriptor_2d *) ((uintptr_t) ptr + offset_desc);
|
|
r->dptr = (dma_ptr *) ((uintptr_t) ptr + offset_dptr);
|
|
r->tail = &r->desc[capacity - 1];
|
|
|
|
FARF(HIGH, "dma-queue: capacity %u, unified memory size %zu\n", capacity, total_size);
|
|
|
|
return q;
|
|
}
|
|
|
|
void dma_queue_free(dma_queue_t q) {
|
|
(void) q;
|
|
}
|
|
|
|
size_t dma_queue_alias_sizeof(void) {
|
|
return sizeof(dma_queue);
|
|
}
|
|
|
|
dma_queue_t dma_queue_alias_init(void * ptr, dma_queue_t main_q, uint8_t nocache) {
|
|
dma_queue * q = (dma_queue *) ptr;
|
|
memset(q, 0, sizeof(dma_queue));
|
|
|
|
q->ring = main_q->ring;
|
|
q->nocache = nocache;
|
|
q->alias = true;
|
|
|
|
return q;
|
|
}
|
|
|
|
void dma_queue_alias_free(dma_queue_t q) {
|
|
(void) q;
|
|
}
|
|
|
|
void dma_queue_flush(dma_queue_t q) {
|
|
while (dma_queue_pop(q).dst != NULL) ;
|
|
}
|