diff --git a/ring/common.go b/ring/common.go index a6dd86b..01cabe6 100644 --- a/ring/common.go +++ b/ring/common.go @@ -17,6 +17,17 @@ func args(r *Ring, obj []unsafe.Pointer) (*C.struct_rte_ring, return (*C.struct_rte_ring)(r), C.uintptr_t(uintptr(unsafe.Pointer(&obj[0]))), C.uint(len(obj)) } +func argsElem(r *Ring, objtable unsafe.Pointer, esize uintptr, n int) (*C.struct_rte_ring, + C.uint, C.uintptr_t, C.uint) { + return (*C.struct_rte_ring)(r), C.uint(esize), C.uintptr_t(uintptr(unsafe.Pointer(objtable))), C.uint(n) +} + +func argsSliceElem(r *Ring, obj []unsafe.Pointer) (*C.struct_rte_ring, + C.uint, C.uintptr_t, C.uint) { + x, p, n := args(r, obj) + return x, C.uint(ptrSize), p, n +} + func ret(out C.struct_compound_int) (rc, n uint32) { return uint32(out.rc), uint32(out.n) } diff --git a/ring/dequeue.go b/ring/dequeue.go index 3ca89fa..4c5f79b 100644 --- a/ring/dequeue.go +++ b/ring/dequeue.go @@ -9,10 +9,6 @@ package ring #include #include "ring.h" - -struct someptr { - void *p; -}; */ import "C" @@ -20,25 +16,26 @@ import ( "unsafe" ) +const ( + ptrSize = unsafe.Sizeof(unsafe.Pointer(nil)) +) + // Dequeue dequeues single object from Ring. func (r *Ring) Dequeue() (unsafe.Pointer, bool) { - objs := []unsafe.Pointer{nil} - n, _ := r.DequeueBulk(objs) - return objs[0], n != 0 + var p unsafe.Pointer + return p, r.DequeueElem(unsafe.Pointer(&p), ptrSize) } // ScDequeue dequeues single object from Ring. func (r *Ring) ScDequeue() (unsafe.Pointer, bool) { - objs := []unsafe.Pointer{nil} - n, _ := r.ScDequeueBulk(objs) - return objs[0], n != 0 + var p unsafe.Pointer + return p, r.ScDequeueElem(unsafe.Pointer(&p), ptrSize) } // McDequeue dequeues single object from Ring. func (r *Ring) McDequeue() (unsafe.Pointer, bool) { - objs := []unsafe.Pointer{nil} - n, _ := r.McDequeueBulk(objs) - return objs[0], n != 0 + var p unsafe.Pointer + return p, r.McDequeueElem(unsafe.Pointer(&p), ptrSize) } // McDequeueBulk dequeues objects into given slice of pointers. @@ -46,7 +43,7 @@ func (r *Ring) McDequeue() (unsafe.Pointer, bool) { // amount of remaining ring entries in the ring after the enqueue // operation has finished. func (r *Ring) McDequeueBulk(obj []unsafe.Pointer) (n, avail uint32) { - return ret(C.mc_dequeue_bulk(args(r, obj))) + return ret(C.mc_dequeue_bulk_elem(argsSliceElem(r, obj))) } // ScDequeueBulk dequeues objects into given slice of pointers. @@ -54,7 +51,7 @@ func (r *Ring) McDequeueBulk(obj []unsafe.Pointer) (n, avail uint32) { // amount of remaining ring entries in the ring after the enqueue // operation has finished. func (r *Ring) ScDequeueBulk(obj []unsafe.Pointer) (n, avail uint32) { - return ret(C.sc_dequeue_bulk(args(r, obj))) + return ret(C.sc_dequeue_bulk_elem(argsSliceElem(r, obj))) } // DequeueBulk dequeues objects into given slice of pointers. @@ -62,7 +59,7 @@ func (r *Ring) ScDequeueBulk(obj []unsafe.Pointer) (n, avail uint32) { // amount of remaining ring entries in the ring after the enqueue // operation has finished. func (r *Ring) DequeueBulk(obj []unsafe.Pointer) (n, avail uint32) { - return ret(C.dequeue_bulk(args(r, obj))) + return ret(C.dequeue_bulk_elem(argsSliceElem(r, obj))) } // McDequeueBurst dequeues objects into given slice of pointers. @@ -70,7 +67,7 @@ func (r *Ring) DequeueBulk(obj []unsafe.Pointer) (n, avail uint32) { // entries in the ring after the enqueue operation has finished. // after the enqueue operation has finished. func (r *Ring) McDequeueBurst(obj []unsafe.Pointer) (n, avail uint32) { - return ret(C.mc_dequeue_burst(args(r, obj))) + return ret(C.mc_dequeue_burst_elem(argsSliceElem(r, obj))) } // ScDequeueBurst dequeues objects into given slice of pointers. @@ -78,7 +75,7 @@ func (r *Ring) McDequeueBurst(obj []unsafe.Pointer) (n, avail uint32) { // entries in the ring after the enqueue operation has finished. // after the enqueue operation has finished. func (r *Ring) ScDequeueBurst(obj []unsafe.Pointer) (n, avail uint32) { - return ret(C.sc_dequeue_burst(args(r, obj))) + return ret(C.sc_dequeue_burst_elem(argsSliceElem(r, obj))) } // DequeueBurst dequeues objects into given slice of pointers. @@ -86,5 +83,5 @@ func (r *Ring) ScDequeueBurst(obj []unsafe.Pointer) (n, avail uint32) { // entries in the ring after the enqueue operation has finished. // after the enqueue operation has finished. func (r *Ring) DequeueBurst(obj []unsafe.Pointer) (n, avail uint32) { - return ret(C.dequeue_burst(args(r, obj))) + return ret(C.dequeue_burst_elem(argsSliceElem(r, obj))) } diff --git a/ring/dequeue_elem.go b/ring/dequeue_elem.go new file mode 100644 index 0000000..6dc8018 --- /dev/null +++ b/ring/dequeue_elem.go @@ -0,0 +1,80 @@ +package ring + +/* +#include + +#include +#include +#include +#include + +#include "ring.h" +*/ +import "C" + +import ( + "unsafe" +) + +// DequeueElem dequeues an object from given Ring. +func (r *Ring) DequeueElem(obj unsafe.Pointer, esize uintptr) bool { + n, _ := r.DequeueBulkElem(obj, esize, 1) + return n != 0 +} + +// ScDequeueElem dequeues an object from given Ring. +func (r *Ring) ScDequeueElem(obj unsafe.Pointer, esize uintptr) bool { + n, _ := r.ScDequeueBulkElem(obj, esize, 1) + return n != 0 +} + +// McDequeueElem dequeues an object from given Ring. +func (r *Ring) McDequeueElem(obj unsafe.Pointer, esize uintptr) bool { + n, _ := r.McDequeueBulkElem(obj, esize, 1) + return n != 0 +} + +// McDequeueBulkElem dequeues given objects to slice from Ring. +// Returns number of dequeued objects (either 0 or len(obj)) and +// amount of space in the ring after the dequeue operation has +// finished. +func (r *Ring) McDequeueBulkElem(objtable unsafe.Pointer, esize uintptr, n int) (processed, free uint32) { + return ret(C.mc_dequeue_bulk_elem(argsElem(r, objtable, esize, n))) +} + +// ScDequeueBulkElem dequeues given objects to slice from Ring. +// Returns number of dequeued objects (either 0 or len(obj)) and +// amount of space in the ring after the dequeue operation has +// finished. +func (r *Ring) ScDequeueBulkElem(objtable unsafe.Pointer, esize uintptr, n int) (processed, free uint32) { + return ret(C.sc_dequeue_bulk_elem(argsElem(r, objtable, esize, n))) +} + +// DequeueBulkElem dequeues given objects to slice from Ring. +// Returns number of dequeued objects (either 0 or len(obj)) and +// amount of space in the ring after the dequeue operation has +// finished. +func (r *Ring) DequeueBulkElem(objtable unsafe.Pointer, esize uintptr, n int) (processed, free uint32) { + return ret(C.dequeue_bulk_elem(argsElem(r, objtable, esize, n))) +} + +// McDequeueBurstElem dequeues given objects to slice from Ring. +// Returns number of dequeued objects and amount of space in the ring +// after the dequeue operation has finished. +func (r *Ring) McDequeueBurstElem(objtable unsafe.Pointer, esize uintptr, n int) (processed, free uint32) { + return ret(C.mc_dequeue_burst_elem(argsElem(r, objtable, esize, n))) +} + +// ScDequeueBurstElem dequeues given objects to slice from Ring. +// Returns number of dequeued objects and amount of space in the ring +// after the dequeue operation has finished. +func (r *Ring) ScDequeueBurstElem(objtable unsafe.Pointer, esize uintptr, n int) (processed, free uint32) { + return ret(C.sc_dequeue_burst_elem(argsElem(r, objtable, esize, n))) +} + +// DequeueBurstElem dequeues given objects to slice from Ring. Returns +// number of dequeued objects and amount of space in the ring after +// the dequeue operation has finished. +func (r *Ring) DequeueBurstElem(objtable unsafe.Pointer, esize uintptr, n int) (processed, free uint32) { + return ret(C.dequeue_burst_elem(argsElem(r, objtable, esize, n))) +} diff --git a/ring/enqueue.go b/ring/enqueue.go index b6fbafa..29c5cc4 100644 --- a/ring/enqueue.go +++ b/ring/enqueue.go @@ -9,10 +9,6 @@ package ring #include #include "ring.h" - -struct someptr { - void *p; -}; */ import "C" @@ -43,7 +39,7 @@ func (r *Ring) MpEnqueue(obj unsafe.Pointer) bool { // amount of space in the ring after the enqueue operation has // finished. func (r *Ring) MpEnqueueBulk(obj []unsafe.Pointer) (n, free uint32) { - return ret(C.mp_enqueue_bulk(args(r, obj))) + return ret(C.mp_enqueue_bulk_elem(argsSliceElem(r, obj))) } // SpEnqueueBulk enqueues given objects from slice into Ring. @@ -51,7 +47,7 @@ func (r *Ring) MpEnqueueBulk(obj []unsafe.Pointer) (n, free uint32) { // amount of space in the ring after the enqueue operation has // finished. func (r *Ring) SpEnqueueBulk(obj []unsafe.Pointer) (n, free uint32) { - return ret(C.sp_enqueue_bulk(args(r, obj))) + return ret(C.sp_enqueue_bulk_elem(argsSliceElem(r, obj))) } // EnqueueBulk enqueues given objects from slice into Ring. @@ -59,26 +55,26 @@ func (r *Ring) SpEnqueueBulk(obj []unsafe.Pointer) (n, free uint32) { // amount of space in the ring after the enqueue operation has // finished. func (r *Ring) EnqueueBulk(obj []unsafe.Pointer) (n, free uint32) { - return ret(C.enqueue_bulk(args(r, obj))) + return ret(C.enqueue_bulk_elem(argsSliceElem(r, obj))) } // MpEnqueueBurst enqueues given objects from slice into Ring. // Returns number of enqueued objects and amount of space in the ring // after the enqueue operation has finished. func (r *Ring) MpEnqueueBurst(obj []unsafe.Pointer) (n, free uint32) { - return ret(C.mp_enqueue_burst(args(r, obj))) + return ret(C.mp_enqueue_burst_elem(argsSliceElem(r, obj))) } // SpEnqueueBurst enqueues given objects from slice into Ring. // Returns number of enqueued objects and amount of space in the ring // after the enqueue operation has finished. func (r *Ring) SpEnqueueBurst(obj []unsafe.Pointer) (n, free uint32) { - return ret(C.sp_enqueue_burst(args(r, obj))) + return ret(C.sp_enqueue_burst_elem(argsSliceElem(r, obj))) } // EnqueueBurst enqueues given objects from slice into Ring. Returns // number of enqueued objects and amount of space in the ring after // the enqueue operation has finished. func (r *Ring) EnqueueBurst(obj []unsafe.Pointer) (n, free uint32) { - return ret(C.enqueue_burst(args(r, obj))) + return ret(C.enqueue_burst_elem(argsSliceElem(r, obj))) } diff --git a/ring/enqueue_elem.go b/ring/enqueue_elem.go new file mode 100644 index 0000000..9bcdef7 --- /dev/null +++ b/ring/enqueue_elem.go @@ -0,0 +1,80 @@ +package ring + +/* +#include + +#include +#include +#include +#include + +#include "ring.h" +*/ +import "C" + +import ( + "unsafe" +) + +// EnqueueElem enqueues an object into given Ring. +func (r *Ring) EnqueueElem(obj unsafe.Pointer, esize uintptr) bool { + n, _ := r.EnqueueBulkElem(obj, esize, 1) + return n != 0 +} + +// SpEnqueueElem enqueues an object into given Ring. +func (r *Ring) SpEnqueueElem(obj unsafe.Pointer, esize uintptr) bool { + n, _ := r.SpEnqueueBulkElem(obj, esize, 1) + return n != 0 +} + +// MpEnqueueElem enqueues an object into given Ring. +func (r *Ring) MpEnqueueElem(obj unsafe.Pointer, esize uintptr) bool { + n, _ := r.MpEnqueueBulkElem(obj, esize, 1) + return n != 0 +} + +// MpEnqueueBulkElem enqueues given objects from slice into Ring. +// Returns number of enqueued objects (either 0 or len(obj)) and +// amount of space in the ring after the enqueue operation has +// finished. +func (r *Ring) MpEnqueueBulkElem(objtable unsafe.Pointer, esize uintptr, n int) (processed, free uint32) { + return ret(C.mp_enqueue_bulk_elem(argsElem(r, objtable, esize, n))) +} + +// SpEnqueueBulkElem enqueues given objects from slice into Ring. +// Returns number of enqueued objects (either 0 or len(obj)) and +// amount of space in the ring after the enqueue operation has +// finished. +func (r *Ring) SpEnqueueBulkElem(objtable unsafe.Pointer, esize uintptr, n int) (processed, free uint32) { + return ret(C.sp_enqueue_bulk_elem(argsElem(r, objtable, esize, n))) +} + +// EnqueueBulkElem enqueues given objects from slice into Ring. +// Returns number of enqueued objects (either 0 or len(obj)) and +// amount of space in the ring after the enqueue operation has +// finished. +func (r *Ring) EnqueueBulkElem(objtable unsafe.Pointer, esize uintptr, n int) (processed, free uint32) { + return ret(C.enqueue_bulk_elem(argsElem(r, objtable, esize, n))) +} + +// MpEnqueueBurstElem enqueues given objects from slice into Ring. +// Returns number of enqueued objects and amount of space in the ring +// after the enqueue operation has finished. +func (r *Ring) MpEnqueueBurstElem(objtable unsafe.Pointer, esize uintptr, n int) (processed, free uint32) { + return ret(C.mp_enqueue_burst_elem(argsElem(r, objtable, esize, n))) +} + +// SpEnqueueBurstElem enqueues given objects from slice into Ring. +// Returns number of enqueued objects and amount of space in the ring +// after the enqueue operation has finished. +func (r *Ring) SpEnqueueBurstElem(objtable unsafe.Pointer, esize uintptr, n int) (processed, free uint32) { + return ret(C.sp_enqueue_burst_elem(argsElem(r, objtable, esize, n))) +} + +// EnqueueBurstElem enqueues given objects from slice into Ring. Returns +// number of enqueued objects and amount of space in the ring after +// the enqueue operation has finished. +func (r *Ring) EnqueueBurstElem(objtable unsafe.Pointer, esize uintptr, n int) (processed, free uint32) { + return ret(C.enqueue_burst_elem(argsElem(r, objtable, esize, n))) +} diff --git a/ring/native.go b/ring/native.go new file mode 100644 index 0000000..7c2dffc --- /dev/null +++ b/ring/native.go @@ -0,0 +1,150 @@ +package ring + +/* +#include +#include +#include +#include + +enum { + OFF_RING_CONS = offsetof(struct rte_ring, cons), + OFF_RING_PROD = offsetof(struct rte_ring, prod), +}; +*/ +import "C" +import ( + "sync/atomic" + "unsafe" +) + +type ringBehavior uint32 + +const ( + queueFixed ringBehavior = C.RTE_RING_QUEUE_FIXED + queueVariable ringBehavior = C.RTE_RING_QUEUE_VARIABLE +) + +type syncType uint32 + +const ( + syncMT syncType = C.RTE_RING_SYNC_MT + syncST syncType = C.RTE_RING_SYNC_ST + syncRTS syncType = C.RTE_RING_SYNC_MT_RTS + syncHTS syncType = C.RTE_RING_SYNC_MT_HTS +) + +type ringHeadTail C.struct_rte_ring_headtail +type htsHeadtail C.struct_rte_ring_hts_headtail +type rtsHeadtail C.struct_rte_ring_rts_headtail + +type headTail struct { + head uint32 + tail uint32 +} + +func (r *Ring) cons() unsafe.Pointer { + p := unsafe.Pointer(r) + return unsafe.Add(p, C.OFF_RING_CONS) +} + +func (r *Ring) prod() unsafe.Pointer { + p := unsafe.Pointer(r) + return unsafe.Add(p, C.OFF_RING_PROD) +} + +func (r *Ring) ringHeadTailProd() *ringHeadTail { + return (*ringHeadTail)(r.prod()) +} + +func (r *Ring) ringHeadTailCons() *ringHeadTail { + return (*ringHeadTail)(r.cons()) +} + +func (r *Ring) htsHeadtailProd() *htsHeadtail { + return (*htsHeadtail)(r.prod()) +} + +func (r *Ring) htsHeadtailCons() *htsHeadtail { + return (*htsHeadtail)(r.cons()) +} + +func (r *Ring) rtsHeadtailProd() *rtsHeadtail { + return (*rtsHeadtail)(r.prod()) +} + +func (r *Ring) rtsHeadtailCons() *rtsHeadtail { + return (*rtsHeadtail)(r.cons()) +} + +func (r *Ring) moveConsHead(isSc bool, n uint32, behavior ringBehavior, oldHead, newHead, entries *uint32) uint32 { + cons := (*headTail)(r.cons()) + prod := (*headTail)(r.prod()) + max := n + success := false + + for { + n = max + + // XXX: there is a memory barrier in this place to protect + // reordering of load/load. In Go we place atomic load for + // uint32 instead. + *oldHead = atomic.LoadUint32(&cons.head) + + *entries = prod.tail - *oldHead + + if n > *entries { + if behavior == queueFixed { + n = 0 + } else { + n = *entries + } + } + + if n == 0 { + return 0 + } + + *newHead = *oldHead + n + + if isSc { + cons.head = *newHead + success = true + } else { + success = atomic.CompareAndSwapUint32(&cons.head, *oldHead, *newHead) + } + + if success { + break + } + } + + return n +} + +func (r *Ring) dequeueElems(oldHead uint32, objTable unsafe.Pointer, eSize, n uint32) { + panic("TODO") +} + +func (r *Ring) updateTail(ht *headTail, oldHead, nextHead uint32, isSc, enqueue bool) { + panic("TODO") +} + +func (r *Ring) doDequeueElem(objTable unsafe.Pointer, eSize, n uint32, behavior ringBehavior, st syncType, available *uint32) uint32 { + var consHead, consNext, entries uint32 + cons := (*headTail)(r.cons()) + isSc := st != syncMT + + n = r.moveConsHead(isSc, n, behavior, &consHead, &consNext, &entries) + if n == 0 { + goto end + } + + r.dequeueElems(consHead, objTable, eSize, n) + r.updateTail(cons, consHead, consNext, isSc, false) + +end: + if available != nil { + *available = entries - n + } + return n +} diff --git a/ring/ring.h b/ring/ring.h index e56ad20..f0cfa6f 100644 --- a/ring/ring.h +++ b/ring/ring.h @@ -8,30 +8,35 @@ struct compound_int { typedef void * ptr_t; -#define GO_RING_FUNC(func) \ -static struct compound_int func(struct rte_ring *r, \ +#define GO_RING_FUNC_ELEM(func) \ +static struct compound_int func ## _elem( \ + struct rte_ring *r, unsigned int esize, \ uintptr_t objs, unsigned int n) { \ struct compound_int out; \ - void **obj_table = (typeof(obj_table))objs; \ - out.rc = rte_ring_ ## func(r, obj_table, n, &out.n); \ + void *obj_table = (typeof(obj_table))objs; \ + out.rc = rte_ring_ ## func ## _elem(r, obj_table, esize, \ + n, &out.n); \ return out; \ } +#define GO_RING_OP(func) \ + GO_RING_FUNC_ELEM(func) + // wrap dequeue API -GO_RING_FUNC(mc_dequeue_burst) -GO_RING_FUNC(mc_dequeue_bulk) -GO_RING_FUNC(sc_dequeue_burst) -GO_RING_FUNC(sc_dequeue_bulk) -GO_RING_FUNC(dequeue_burst) -GO_RING_FUNC(dequeue_bulk) +GO_RING_OP(mc_dequeue_burst) +GO_RING_OP(mc_dequeue_bulk) +GO_RING_OP(sc_dequeue_burst) +GO_RING_OP(sc_dequeue_bulk) +GO_RING_OP(dequeue_burst) +GO_RING_OP(dequeue_bulk) // wrap enqueue API -GO_RING_FUNC(mp_enqueue_burst) -GO_RING_FUNC(mp_enqueue_bulk) -GO_RING_FUNC(sp_enqueue_burst) -GO_RING_FUNC(sp_enqueue_bulk) -GO_RING_FUNC(enqueue_burst) -GO_RING_FUNC(enqueue_bulk) +GO_RING_OP(mp_enqueue_burst) +GO_RING_OP(mp_enqueue_bulk) +GO_RING_OP(sp_enqueue_burst) +GO_RING_OP(sp_enqueue_bulk) +GO_RING_OP(enqueue_burst) +GO_RING_OP(enqueue_bulk) #endif /* _RING_H_ */