From c437f07ee482bb6086ba8372e671bb4b624f89f9 Mon Sep 17 00:00:00 2001 From: Yuval Degani Date: Mon, 27 Feb 2017 14:27:22 -0800 Subject: [PATCH 01/15] Support for registering virtual addresses that are not wrapped with a bytebuffer --- src/main/java/com/ibm/disni/rdma/verbs/IbvPd.java | 4 ++++ .../java/com/ibm/disni/rdma/verbs/RdmaVerbs.java | 2 ++ .../com/ibm/disni/rdma/verbs/impl/NatRegMrCall.java | 13 +++++++++++++ .../com/ibm/disni/rdma/verbs/impl/RdmaVerbsNat.java | 9 +++++++++ 4 files changed, 28 insertions(+) diff --git a/src/main/java/com/ibm/disni/rdma/verbs/IbvPd.java b/src/main/java/com/ibm/disni/rdma/verbs/IbvPd.java index a4aff5f2..5e34a5c9 100644 --- a/src/main/java/com/ibm/disni/rdma/verbs/IbvPd.java +++ b/src/main/java/com/ibm/disni/rdma/verbs/IbvPd.java @@ -72,6 +72,10 @@ public SVCRegMr regMr(ByteBuffer buffer, int access) throws IOException { return verbs.regMr(this, buffer, access); } + public SVCRegMr regMr(long address, int length, int access) throws IOException { + return verbs.regMr(this, address, length, access); + } + public int deallocPd() throws Exception { return verbs.deallocPd(this); } diff --git a/src/main/java/com/ibm/disni/rdma/verbs/RdmaVerbs.java b/src/main/java/com/ibm/disni/rdma/verbs/RdmaVerbs.java index 29b91698..b924eb21 100644 --- a/src/main/java/com/ibm/disni/rdma/verbs/RdmaVerbs.java +++ b/src/main/java/com/ibm/disni/rdma/verbs/RdmaVerbs.java @@ -113,6 +113,8 @@ public abstract IbvCQ createCQ(IbvContext context, * @throws Exception on failure. */ public abstract SVCRegMr regMr(IbvPd pd, ByteBuffer buffer, int access) throws IOException; + + public abstract SVCRegMr regMr(IbvPd pd, long address, int length, int access) throws IOException; /** * Deregister memory. diff --git a/src/main/java/com/ibm/disni/rdma/verbs/impl/NatRegMrCall.java b/src/main/java/com/ibm/disni/rdma/verbs/impl/NatRegMrCall.java index 298ee1c0..a298ba93 100644 --- a/src/main/java/com/ibm/disni/rdma/verbs/impl/NatRegMrCall.java +++ b/src/main/java/com/ibm/disni/rdma/verbs/impl/NatRegMrCall.java @@ -64,6 +64,19 @@ public void set(IbvPd pd, ByteBuffer buffer, int access) { this.valid = true; } + public void set(IbvPd pd, long address, int length, int access) { + this.pd = (NatIbvPd) pd; + this.bufferCapacity = length; + this.userAddress = address; + + if (cmd != null){ + cmd.free(); + cmd = null; + } + this.cmd = memAlloc.allocate(3*4, MemoryAllocation.MemType.DIRECT, this.getClass().getCanonicalName()); + this.valid = true; + } + public SVCRegMr execute() throws IOException { cmd.getBuffer().clear(); long objId = nativeDispatcher._regMr(pd.getObjId(), userAddress, bufferCapacity, cmd.address(), cmd.address() + 4, cmd.address() + 8); diff --git a/src/main/java/com/ibm/disni/rdma/verbs/impl/RdmaVerbsNat.java b/src/main/java/com/ibm/disni/rdma/verbs/impl/RdmaVerbsNat.java index 87e43a98..7f966966 100644 --- a/src/main/java/com/ibm/disni/rdma/verbs/impl/RdmaVerbsNat.java +++ b/src/main/java/com/ibm/disni/rdma/verbs/impl/RdmaVerbsNat.java @@ -136,6 +136,15 @@ public SVCRegMr regMr(IbvPd pd, ByteBuffer buffer, int access) throws IOExceptio return regMrCall; } + public SVCRegMr regMr(IbvPd pd, long address, int length, int access) throws IOException { + NatRegMrCall regMrCall = regList.poll(); + if (regMrCall == null){ + regMrCall = new NatRegMrCall(this, nativeDispatcher, memAlloc); + } + regMrCall.set(pd, address, length, access); + return regMrCall; + + } public SVCDeregMr deregMr(IbvMr mr) throws IOException { NatDeregMrCall deregMrCall = deregList.poll(); From 0e21ba7f9a5ee847cdc2cb32c953e53d13260b08 Mon Sep 17 00:00:00 2001 From: Yuval Degani Date: Mon, 13 Mar 2017 11:45:44 -0700 Subject: [PATCH 02/15] Support for specifying access permission on RegMR --- .../com_ibm_disni_rdma_verbs_impl_NativeDispatcher.cpp | 3 +-- .../verbs/com_ibm_disni_rdma_verbs_impl_NativeDispatcher.h | 4 ++-- src/main/java/com/ibm/disni/rdma/verbs/IbvMr.java | 6 ++++-- src/main/java/com/ibm/disni/rdma/verbs/impl/NatIbvMr.java | 4 ++-- .../java/com/ibm/disni/rdma/verbs/impl/NatRegMrCall.java | 6 ++++-- .../com/ibm/disni/rdma/verbs/impl/NativeDispatcher.java | 2 +- 6 files changed, 14 insertions(+), 11 deletions(-) diff --git a/libdisni/src/verbs/com_ibm_disni_rdma_verbs_impl_NativeDispatcher.cpp b/libdisni/src/verbs/com_ibm_disni_rdma_verbs_impl_NativeDispatcher.cpp index b6acac52..70fb5557 100644 --- a/libdisni/src/verbs/com_ibm_disni_rdma_verbs_impl_NativeDispatcher.cpp +++ b/libdisni/src/verbs/com_ibm_disni_rdma_verbs_impl_NativeDispatcher.cpp @@ -825,9 +825,8 @@ JNIEXPORT jint JNICALL Java_com_ibm_disni_rdma_verbs_impl_NativeDispatcher__1mod * Signature: (IIJIJJJ)V */ JNIEXPORT jlong JNICALL Java_com_ibm_disni_rdma_verbs_impl_NativeDispatcher__1regMr - (JNIEnv *env, jobject obj, jlong pd, jlong address, jint len, jlong lkey, jlong rkey, jlong handle){ + (JNIEnv *env, jobject obj, jlong pd, jlong address, jint len, jint access, jlong lkey, jlong rkey, jlong handle){ struct ibv_pd *protection = NULL; - int access = IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_WRITE | IBV_ACCESS_REMOTE_READ; void *addr = (void *) address; //jint ret = -1; unsigned long long obj_id = -1; diff --git a/libdisni/src/verbs/com_ibm_disni_rdma_verbs_impl_NativeDispatcher.h b/libdisni/src/verbs/com_ibm_disni_rdma_verbs_impl_NativeDispatcher.h index f0ed659e..9546d72a 100644 --- a/libdisni/src/verbs/com_ibm_disni_rdma_verbs_impl_NativeDispatcher.h +++ b/libdisni/src/verbs/com_ibm_disni_rdma_verbs_impl_NativeDispatcher.h @@ -186,10 +186,10 @@ JNIEXPORT jint JNICALL Java_com_ibm_disni_rdma_verbs_impl_NativeDispatcher__1mod /* * Class: com_ibm_disni_verbs_impl_NativeDispatcher * Method: _regMr - * Signature: (JJIJJJ)J + * Signature: (JJIIJJJ)J */ JNIEXPORT jlong JNICALL Java_com_ibm_disni_rdma_verbs_impl_NativeDispatcher__1regMr - (JNIEnv *, jobject, jlong, jlong, jint, jlong, jlong, jlong); + (JNIEnv *, jobject, jlong, jlong, jint, jint, jlong, jlong, jlong); /* * Class: com_ibm_disni_verbs_impl_NativeDispatcher diff --git a/src/main/java/com/ibm/disni/rdma/verbs/IbvMr.java b/src/main/java/com/ibm/disni/rdma/verbs/IbvMr.java index ac065597..131ffd53 100644 --- a/src/main/java/com/ibm/disni/rdma/verbs/IbvMr.java +++ b/src/main/java/com/ibm/disni/rdma/verbs/IbvMr.java @@ -51,6 +51,7 @@ public class IbvMr { protected IbvContext context; protected long addr; protected int length; + protected int access; protected int lkey; protected int rkey; protected int handle; @@ -59,11 +60,12 @@ public class IbvMr { // this.context = null; // } - public IbvMr(IbvContext context, long addr, int length, int lkey, int rkey, int handle) throws IOException { + public IbvMr(IbvContext context, long addr, int length, int access, int lkey, int rkey, int handle) throws IOException { this.verbs = RdmaVerbs.open(); this.context = context; this.addr = addr; this.length = length; + this.access = access; this.lkey = lkey; this.rkey = rkey; this.handle = handle; @@ -153,7 +155,7 @@ public int getHandle() { } public String toString() { - return "handle=" + handle + ", addr= " + addr + ", length=" + length + ", lkey=" + lkey + ", rkey=" + rkey; + return "handle=" + handle + ", addr= " + addr + ", length=" + length + ", access= " + access + ", lkey=" + lkey + ", rkey=" + rkey; } /** diff --git a/src/main/java/com/ibm/disni/rdma/verbs/impl/NatIbvMr.java b/src/main/java/com/ibm/disni/rdma/verbs/impl/NatIbvMr.java index 4f687d87..6b2f2d95 100644 --- a/src/main/java/com/ibm/disni/rdma/verbs/impl/NatIbvMr.java +++ b/src/main/java/com/ibm/disni/rdma/verbs/impl/NatIbvMr.java @@ -29,8 +29,8 @@ public class NatIbvMr extends IbvMr implements NatObject { private long objId; - public NatIbvMr(long objId, IbvContext context, long addr, int length, int lkey, int rkey, int handle) throws IOException{ - super(context, addr, length, lkey, rkey, handle); + public NatIbvMr(long objId, IbvContext context, long addr, int length, int access, int lkey, int rkey, int handle) throws IOException{ + super(context, addr, length, access, lkey, rkey, handle); this.objId = objId; } diff --git a/src/main/java/com/ibm/disni/rdma/verbs/impl/NatRegMrCall.java b/src/main/java/com/ibm/disni/rdma/verbs/impl/NatRegMrCall.java index a298ba93..ade4de69 100644 --- a/src/main/java/com/ibm/disni/rdma/verbs/impl/NatRegMrCall.java +++ b/src/main/java/com/ibm/disni/rdma/verbs/impl/NatRegMrCall.java @@ -36,6 +36,7 @@ public class NatRegMrCall extends SVCRegMr { private MemoryAllocation memAlloc; private NatIbvPd pd; + private int access; private int bufferCapacity; private MemBuf cmd; private long userAddress; @@ -53,6 +54,7 @@ public NatRegMrCall(RdmaVerbsNat verbs, NativeDispatcher nativeDispatcher, Memor public void set(IbvPd pd, ByteBuffer buffer, int access) { this.pd = (NatIbvPd) pd; + this.access = access; this.bufferCapacity = buffer.capacity(); this.userAddress = ((sun.nio.ch.DirectBuffer) buffer).address(); @@ -79,14 +81,14 @@ public void set(IbvPd pd, long address, int length, int access) { public SVCRegMr execute() throws IOException { cmd.getBuffer().clear(); - long objId = nativeDispatcher._regMr(pd.getObjId(), userAddress, bufferCapacity, cmd.address(), cmd.address() + 4, cmd.address() + 8); + long objId = nativeDispatcher._regMr(pd.getObjId(), userAddress, bufferCapacity, access, cmd.address(), cmd.address() + 4, cmd.address() + 8); if (objId <= 0){ throw new IOException("Memory registration failed with " + objId); } else { int lkey = cmd.getBuffer().getInt(); int rkey = cmd.getBuffer().getInt(); int handle = cmd.getBuffer().getInt(); - this.mr = new NatIbvMr(objId, null, userAddress, bufferCapacity, lkey, rkey, handle); + this.mr = new NatIbvMr(objId, null, userAddress, bufferCapacity, access, lkey, rkey, handle); } return this; diff --git a/src/main/java/com/ibm/disni/rdma/verbs/impl/NativeDispatcher.java b/src/main/java/com/ibm/disni/rdma/verbs/impl/NativeDispatcher.java index a413847e..3f24634d 100644 --- a/src/main/java/com/ibm/disni/rdma/verbs/impl/NativeDispatcher.java +++ b/src/main/java/com/ibm/disni/rdma/verbs/impl/NativeDispatcher.java @@ -112,7 +112,7 @@ public class NativeDispatcher { public native long _createCompChannel(long context); public native long _createCQ(long context, long compChannel, int ncqe, int comp_vector); public native int _modifyQP(long qp, long attr); - public native long _regMr(long pd, long addr, int len, long lkey, long rkey, long handle); + public native long _regMr(long pd, long addr, int len, int access, long lkey, long rkey, long handle); public native int _deregMr(long handle); public native int _postSend(long qp, long wrList); public native int _postRecv(long qp, long wrList); From 576eff2308c91062d1e01d3375f879a9c8dd7b52 Mon Sep 17 00:00:00 2001 From: Yuval Degani Date: Wed, 15 Mar 2017 11:15:16 -0700 Subject: [PATCH 03/15] Missing parameter assignment in NatRegMrCall.set() + refactor set() implementation --- .../ibm/disni/rdma/verbs/impl/NatRegMrCall.java | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/src/main/java/com/ibm/disni/rdma/verbs/impl/NatRegMrCall.java b/src/main/java/com/ibm/disni/rdma/verbs/impl/NatRegMrCall.java index ade4de69..4718f68f 100644 --- a/src/main/java/com/ibm/disni/rdma/verbs/impl/NatRegMrCall.java +++ b/src/main/java/com/ibm/disni/rdma/verbs/impl/NatRegMrCall.java @@ -53,23 +53,14 @@ public NatRegMrCall(RdmaVerbsNat verbs, NativeDispatcher nativeDispatcher, Memor } public void set(IbvPd pd, ByteBuffer buffer, int access) { - this.pd = (NatIbvPd) pd; - this.access = access; - this.bufferCapacity = buffer.capacity(); - this.userAddress = ((sun.nio.ch.DirectBuffer) buffer).address(); - - if (cmd != null){ - cmd.free(); - cmd = null; - } - this.cmd = memAlloc.allocate(3*4, MemoryAllocation.MemType.DIRECT, this.getClass().getCanonicalName()); - this.valid = true; + set(pd, ((sun.nio.ch.DirectBuffer) buffer).address(), buffer.capacity(), access); } public void set(IbvPd pd, long address, int length, int access) { this.pd = (NatIbvPd) pd; - this.bufferCapacity = length; this.userAddress = address; + this.bufferCapacity = length; + this.access = access; if (cmd != null){ cmd.free(); From d5d37c72b0640106129e89a4fdc31da7dce02226 Mon Sep 17 00:00:00 2001 From: Yuval Degani Date: Wed, 3 May 2017 16:01:20 -0700 Subject: [PATCH 04/15] Allow a postRecv to provide an empty sgl with num_sge set to --- .../ibm/disni/rdma/verbs/impl/NatPostRecvCall.java | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/main/java/com/ibm/disni/rdma/verbs/impl/NatPostRecvCall.java b/src/main/java/com/ibm/disni/rdma/verbs/impl/NatPostRecvCall.java index be5006ac..1dcca805 100644 --- a/src/main/java/com/ibm/disni/rdma/verbs/impl/NatPostRecvCall.java +++ b/src/main/java/com/ibm/disni/rdma/verbs/impl/NatPostRecvCall.java @@ -65,16 +65,18 @@ public void set(IbvQP qp, List wrList) { long wrOffset = NatIbvRecvWR.CSIZE; for (IbvRecvWR recvWR : wrList){ NatIbvRecvWR natRecvWR = new NatIbvRecvWR(recvWR); - natRecvWR.setPtr_sge_list(sgeOffset); natRecvWR.setNext(wrOffset); - wrNatList.add(natRecvWR); - sgeNatList.addAll(recvWR.getSg_list()); - + size += NatIbvRecvWR.CSIZE; - size += recvWR.getSg_list().size()*NatIbvSge.CSIZE; wrOffset += NatIbvRecvWR.CSIZE; - sgeOffset += recvWR.getSg_list().size()*NatIbvSge.CSIZE; + + if (recvWR.getNum_sge() > 0) { + natRecvWR.setPtr_sge_list(sgeOffset); + sgeNatList.addAll(recvWR.getSg_list()); + size += recvWR.getSg_list().size()*NatIbvSge.CSIZE; + sgeOffset += recvWR.getSg_list().size()*NatIbvSge.CSIZE; + } } if (cmd != null){ cmd.free(); From a05786fee929fac9e3a2dff6134491c04568c78a Mon Sep 17 00:00:00 2001 From: Patrick Stuedi Date: Wed, 24 May 2017 16:54:44 +0200 Subject: [PATCH 05/15] Version update 1.1 due to memory registration updates --- .../verbs/com_ibm_disni_rdma_verbs_impl_NativeDispatcher.cpp | 2 +- pom.xml | 2 +- .../java/com/ibm/disni/rdma/verbs/impl/NativeDispatcher.java | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/libdisni/src/verbs/com_ibm_disni_rdma_verbs_impl_NativeDispatcher.cpp b/libdisni/src/verbs/com_ibm_disni_rdma_verbs_impl_NativeDispatcher.cpp index 70fb5557..798ed0e8 100644 --- a/libdisni/src/verbs/com_ibm_disni_rdma_verbs_impl_NativeDispatcher.cpp +++ b/libdisni/src/verbs/com_ibm_disni_rdma_verbs_impl_NativeDispatcher.cpp @@ -45,7 +45,7 @@ using namespace std; //#define MAX_WR 200; #define MAX_SGE 4; //#define N_CQE 200 -#define JVERBS_JNI_VERSION 30; +#define JVERBS_JNI_VERSION 31; //global resource id counter static unsigned long long counter = 0; diff --git a/pom.xml b/pom.xml index fab3ebe3..a47aeb39 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ com.ibm.disni disni jar - 1.0 + 1.1 disni http://maven.apache.org diff --git a/src/main/java/com/ibm/disni/rdma/verbs/impl/NativeDispatcher.java b/src/main/java/com/ibm/disni/rdma/verbs/impl/NativeDispatcher.java index 3f24634d..138eaf00 100644 --- a/src/main/java/com/ibm/disni/rdma/verbs/impl/NativeDispatcher.java +++ b/src/main/java/com/ibm/disni/rdma/verbs/impl/NativeDispatcher.java @@ -30,7 +30,7 @@ public class NativeDispatcher { private static final Logger logger = DiSNILogger.getLogger(); - private static int JVERBS_VERSION = 30; + private static int JVERBS_VERSION = 31; static { System.loadLibrary("disni"); From ed5c31146d33dd94666c59efd448ba13d7bcdc65 Mon Sep 17 00:00:00 2001 From: PepperJo Date: Fri, 16 Jun 2017 14:40:09 +0200 Subject: [PATCH 06/15] add metadata to pom file to adhere to Maven Central requirements --- pom.xml | 95 +++++++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 93 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index a47aeb39..d4470717 100644 --- a/pom.xml +++ b/pom.xml @@ -1,12 +1,50 @@ 4.0.0 + com.ibm.disni disni jar 1.1 disni - http://maven.apache.org + DiSNI (Direct Storage and Networking Interface) is a Java library for direct storage and networking access from userpace. + http://github.com/zrlio/disni + + + + The Apache License, Version 2.0 + http://www.apache.org/licenses/LICENSE-2.0.txt + + + + + + Patrick Stuedi + stu@zurich.ibm.com + + + Jonas Pfefferle + jpf@zurich.ibm.com + + + Animesh Trivedi + atr@zurich.ibm.com + + + + + scm:git:git://github.com/zrlio/disni.git + scm:git:ssh://github.com:zrlio/disni.git + http://github.com/zrlio/disni/tree/master + + + + + ossrh + https://oss.sonatype.org/content/repositories/snapshots + + + junit @@ -14,7 +52,7 @@ 3.8.1 test - + org.slf4j slf4j-log4j12 1.7.12 @@ -62,6 +100,59 @@ + + org.sonatype.plugins + nexus-staging-maven-plugin + 1.6.7 + true + + ossrh + https://oss.sonatype.org/ + false + + + + + org.apache.maven.plugins + maven-gpg-plugin + 1.5 + + + sign-artifacts + verify + + sign + + + + + + + org.apache.maven.plugins + maven-source-plugin + 2.2.1 + + + attach-sources + + jar-no-fork + + + + + + org.apache.maven.plugins + maven-javadoc-plugin + 2.9.1 + + + attach-javadocs + + jar + + + + From 8c0b87d45cef23c88e2d295026ec2399cea34c50 Mon Sep 17 00:00:00 2001 From: PepperJo Date: Fri, 16 Jun 2017 14:40:56 +0200 Subject: [PATCH 07/15] add missing authors --- AUTHORS | 2 ++ 1 file changed, 2 insertions(+) diff --git a/AUTHORS b/AUTHORS index d86fd74e..529db7fd 100644 --- a/AUTHORS +++ b/AUTHORS @@ -1 +1,3 @@ Patrick Stuedi +Jonas Pfefferle +Animesh Trivedi From 288e59bf41cf01fa970f0357adf785f283ff3638 Mon Sep 17 00:00:00 2001 From: Yuval Degani Date: Mon, 19 Jun 2017 10:39:40 -0700 Subject: [PATCH 08/15] Fixed propagation of return values for some verb calls --- ...disni_rdma_verbs_impl_NativeDispatcher.cpp | 22 +++++-------------- .../com/ibm/disni/rdma/verbs/RdmaCmId.java | 16 +++++++------- .../disni/rdma/verbs/RdmaEventChannel.java | 4 ++-- 3 files changed, 16 insertions(+), 26 deletions(-) diff --git a/libdisni/src/verbs/com_ibm_disni_rdma_verbs_impl_NativeDispatcher.cpp b/libdisni/src/verbs/com_ibm_disni_rdma_verbs_impl_NativeDispatcher.cpp index 70fb5557..29bfed72 100644 --- a/libdisni/src/verbs/com_ibm_disni_rdma_verbs_impl_NativeDispatcher.cpp +++ b/libdisni/src/verbs/com_ibm_disni_rdma_verbs_impl_NativeDispatcher.cpp @@ -487,14 +487,6 @@ JNIEXPORT jint JNICALL Java_com_ibm_disni_rdma_verbs_impl_NativeDispatcher__1dis if (cm_listen_id != NULL){ ret = rdma_disconnect(cm_listen_id); - if (ret == 0){ - pthread_rwlock_wrlock(&mut_cm_id); - map_cm_id.erase(id); - pthread_rwlock_unlock(&mut_cm_id); - log("j2c::disconnect: ret %i\n", ret); - } else { - log("j2c::disconnect: rdma_disconnect failed\n"); - } } else { log("j2c:disconnect: cm_listen_id null\n"); } @@ -541,7 +533,6 @@ JNIEXPORT jint JNICALL Java_com_ibm_disni_rdma_verbs_impl_NativeDispatcher__1des pthread_rwlock_wrlock(&mut_cm_id); map_cm_id.erase(id); pthread_rwlock_unlock(&mut_cm_id); - ret = 0; } return ret; @@ -561,11 +552,13 @@ JNIEXPORT jint JNICALL Java_com_ibm_disni_rdma_verbs_impl_NativeDispatcher__1des cm_listen_id = map_cm_id[id]; pthread_rwlock_unlock(&mut_cm_id); - if (cm_listen_id != NULL){ + if (cm_listen_id != NULL && cm_listen_id->qp != NULL){ + jlong obj_id = createObjectId(cm_listen_id->qp); + pthread_rwlock_wrlock(&mut_qp); + map_qp.erase(obj_id); + pthread_rwlock_unlock(&mut_qp); + rdma_destroy_qp(cm_listen_id); - pthread_rwlock_wrlock(&mut_cm_id); - map_cm_id.erase(id); - pthread_rwlock_unlock(&mut_cm_id); ret = 0; } @@ -1078,7 +1071,6 @@ JNIEXPORT jint JNICALL Java_com_ibm_disni_rdma_verbs_impl_NativeDispatcher__1des pthread_rwlock_wrlock(&mut_comp_channel); map_comp_channel.erase(channel); pthread_rwlock_unlock(&mut_comp_channel); - ret = 0; } return ret; @@ -1102,7 +1094,6 @@ JNIEXPORT jint JNICALL Java_com_ibm_disni_rdma_verbs_impl_NativeDispatcher__1dea pthread_rwlock_wrlock(&mut_pd); map_pd.erase(pd); pthread_rwlock_unlock(&mut_pd); - ret = 0; } return ret; @@ -1126,7 +1117,6 @@ JNIEXPORT jint JNICALL Java_com_ibm_disni_rdma_verbs_impl_NativeDispatcher__1des pthread_rwlock_wrlock(&mut_cq); map_cq.erase(cq); pthread_rwlock_unlock(&mut_cq); - ret = 0; } return ret; diff --git a/src/main/java/com/ibm/disni/rdma/verbs/RdmaCmId.java b/src/main/java/com/ibm/disni/rdma/verbs/RdmaCmId.java index 71b03713..a40ffebf 100644 --- a/src/main/java/com/ibm/disni/rdma/verbs/RdmaCmId.java +++ b/src/main/java/com/ibm/disni/rdma/verbs/RdmaCmId.java @@ -145,16 +145,16 @@ public int accept(RdmaConnParam connParam) throws IOException{ return cm.accept(this, connParam); } - public void disconnect() throws IOException{ - cm.disconnect(this); + public int disconnect() throws IOException{ + return cm.disconnect(this); } - public void destroyId() throws IOException { - cm.destroyCmId(this); + public int destroyId() throws IOException { + return cm.destroyCmId(this); } - public void destroyQP() throws IOException { - cm.destroyQP(this); + public int destroyQP() throws IOException { + return cm.destroyQP(this); } public SocketAddress getSource() throws IOException { @@ -165,7 +165,7 @@ public SocketAddress getDestination() throws IOException { return cm.getDstAddr(this); } - public void destroyEp() throws IOException { - cm.destroyEp(this); + public int destroyEp() throws IOException { + return cm.destroyEp(this); } } diff --git a/src/main/java/com/ibm/disni/rdma/verbs/RdmaEventChannel.java b/src/main/java/com/ibm/disni/rdma/verbs/RdmaEventChannel.java index 82c1610a..7e92dfad 100644 --- a/src/main/java/com/ibm/disni/rdma/verbs/RdmaEventChannel.java +++ b/src/main/java/com/ibm/disni/rdma/verbs/RdmaEventChannel.java @@ -65,7 +65,7 @@ public RdmaCmEvent getCmEvent(int timeout) throws IOException{ return cm.getCmEvent(this, timeout); } - public void destroyEventChannel() throws IOException{ - cm.destroyEventChannel(this); + public int destroyEventChannel() throws IOException{ + return cm.destroyEventChannel(this); } } From d32adaf3c59f35d57d0e4a346c588eb08836da71 Mon Sep 17 00:00:00 2001 From: Patrick Stuedi Date: Wed, 28 Jun 2017 14:29:20 +0200 Subject: [PATCH 09/15] Update README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 87ff79b4..7f3c12b2 100644 --- a/README.md +++ b/README.md @@ -39,12 +39,12 @@ Common steps: ## Programming with DiSNI -To use DiSNI in your maven application use the following snippet in your pom.xml file (you need to build DiSNI before to update your local maven repo): +DiSNI is part of maven central, therefore the simplest way to use DiSNI in your maven application is to add the following snippet to your application pom.xml file. com.ibm.disni disni - 1.0 + 1.1 The RDMA and NVMf APIs in DiSNI are both following the Group/Endpoint model which is based on three key data types (interfaces): From f5d8f3e2ef4058fc10623b61a994def95404826c Mon Sep 17 00:00:00 2001 From: Yuval Degani Date: Wed, 28 Jun 2017 17:36:20 -0700 Subject: [PATCH 10/15] Added enumeration for IBV_WC opcodes --- .../java/com/ibm/disni/rdma/verbs/IbvWC.java | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/main/java/com/ibm/disni/rdma/verbs/IbvWC.java b/src/main/java/com/ibm/disni/rdma/verbs/IbvWC.java index c2944da1..479075bf 100644 --- a/src/main/java/com/ibm/disni/rdma/verbs/IbvWC.java +++ b/src/main/java/com/ibm/disni/rdma/verbs/IbvWC.java @@ -48,6 +48,22 @@ public enum IbvWcStatus { IBV_WC_SUCCESS, IBV_WC_LOC_LEN_ERR, IBV_WC_LOC_QP_OP_ERR, IBV_WC_LOC_EEC_OP_ERR, IBV_WC_LOC_PROT_ERR, IBV_WC_WR_FLUSH_ERR, IBV_WC_MW_BIND_ERR, IBV_WC_BAD_RESP_ERR, IBV_WC_LOC_ACCESS_ERR, IBV_WC_REM_INV_REQ_ERR, IBV_WC_REM_ACCESS_ERR, IBV_WC_REM_OP_ERR, IBV_WC_RETRY_EXC_ERR, IBV_WC_RNR_RETRY_EXC_ERR, IBV_WC_LOC_RDD_VIOL_ERR, IBV_WC_REM_INV_RD_REQ_ERR, IBV_WC_REM_ABORT_ERR, IBV_WC_INV_EECN_ERR, IBV_WC_INV_EEC_STATE_ERR, IBV_WC_FATAL_ERR, IBV_WC_RESP_TIMEOUT_ERR, IBV_WC_GENERAL_ERR }; + public enum IbvWcOpcode { + IBV_WC_SEND(0), + IBV_WC_RDMA_WRITE(1), + IBV_WC_RDMA_READ(2), + IBV_WC_COMP_SWAP(3), + IBV_WC_FETCH_ADD(4), + IBV_WC_BIND_MW(5), + IBV_WC_RECV(128), + IBV_WC_RECV_RDMA_WITH_IMM(129); + + private int opcode; + IbvWcOpcode(int opcode) { this.opcode = opcode; } + + public int getOpcode() { return opcode; } + } + public static int CQ_OK = 0; public static int CQ_EMPTY = -1; public static int CQ_POLL_ERR = -2; From 81efd971f0faba8cb9a80ee53dc0c52db4e4c384 Mon Sep 17 00:00:00 2001 From: Patrick Stuedi Date: Fri, 30 Jun 2017 10:59:06 +0200 Subject: [PATCH 11/15] Adding Yuval Degani to AUTHORS & pom --- AUTHORS | 1 + pom.xml | 7 +++++++ 2 files changed, 8 insertions(+) diff --git a/AUTHORS b/AUTHORS index 529db7fd..96921ddd 100644 --- a/AUTHORS +++ b/AUTHORS @@ -1,3 +1,4 @@ Patrick Stuedi Jonas Pfefferle Animesh Trivedi +Yuval Degani diff --git a/pom.xml b/pom.xml index d4470717..3f5b6965 100644 --- a/pom.xml +++ b/pom.xml @@ -30,6 +30,10 @@ Animesh Trivedi atr@zurich.ibm.com + + Yuval Degani + yuvaldeg@mellanox.com + @@ -143,6 +147,9 @@ org.apache.maven.plugins maven-javadoc-plugin + + -Xdoclint:none + 2.9.1 From fadd67e5487a7d65f16fb213b35e1e83483c585b Mon Sep 17 00:00:00 2001 From: Yuval Degani Date: Mon, 31 Jul 2017 10:58:01 -0700 Subject: [PATCH 12/15] Propagate CPU comp_vector correctly in createCQ --- .../verbs/com_ibm_disni_rdma_verbs_impl_NativeDispatcher.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/libdisni/src/verbs/com_ibm_disni_rdma_verbs_impl_NativeDispatcher.cpp b/libdisni/src/verbs/com_ibm_disni_rdma_verbs_impl_NativeDispatcher.cpp index eeeec9c7..de890e5c 100644 --- a/libdisni/src/verbs/com_ibm_disni_rdma_verbs_impl_NativeDispatcher.cpp +++ b/libdisni/src/verbs/com_ibm_disni_rdma_verbs_impl_NativeDispatcher.cpp @@ -756,6 +756,7 @@ JNIEXPORT jlong JNICALL Java_com_ibm_disni_rdma_verbs_impl_NativeDispatcher__1cr struct ibv_comp_channel *comp_channel = NULL; unsigned long long obj_id = -1; int _ncqe = ncqe; + int _comp_vector = comp_vector; pthread_rwlock_rdlock(&mut_context); context = map_context[ctx]; @@ -765,7 +766,7 @@ JNIEXPORT jlong JNICALL Java_com_ibm_disni_rdma_verbs_impl_NativeDispatcher__1cr pthread_rwlock_unlock(&mut_comp_channel); if (context != NULL && comp_channel != NULL){ - struct ibv_cq *cq = ibv_create_cq(context, _ncqe, NULL, comp_channel, 0); + struct ibv_cq *cq = ibv_create_cq(context, _ncqe, NULL, comp_channel, _comp_vector); if (cq != NULL){ obj_id = createObjectId(cq); pthread_rwlock_wrlock(&mut_cq); From 56469f7c9b96d9b83973d41f493c6345f1c05c98 Mon Sep 17 00:00:00 2001 From: Jonas Pfefferle Date: Wed, 2 Aug 2017 22:15:17 +0200 Subject: [PATCH 13/15] suppress javadoc errors (for now) --- pom.xml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pom.xml b/pom.xml index d4470717..5ab129d4 100644 --- a/pom.xml +++ b/pom.xml @@ -144,6 +144,9 @@ org.apache.maven.plugins maven-javadoc-plugin 2.9.1 + + -Xdoclint:none + attach-javadocs From f097f02b8081122d2095ba9ff74dd8f2c5e3abd6 Mon Sep 17 00:00:00 2001 From: Jonas Pfefferle Date: Wed, 2 Aug 2017 22:20:51 +0200 Subject: [PATCH 14/15] fix versioning --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 5ab129d4..e1d43ef2 100644 --- a/pom.xml +++ b/pom.xml @@ -5,7 +5,7 @@ com.ibm.disni disni jar - 1.1 + 1.2 disni DiSNI (Direct Storage and Networking Interface) is a Java library for direct storage and networking access from userpace. http://github.com/zrlio/disni From 247fe8abe54c90b450d2a4b0679e59cfa83205f6 Mon Sep 17 00:00:00 2001 From: Jonas Pfefferle Date: Wed, 2 Aug 2017 22:31:15 +0200 Subject: [PATCH 15/15] fix pom file multiple configuration for javadoc --- pom.xml | 3 --- 1 file changed, 3 deletions(-) diff --git a/pom.xml b/pom.xml index 1039a69a..85de245f 100644 --- a/pom.xml +++ b/pom.xml @@ -151,9 +151,6 @@ -Xdoclint:none 2.9.1 - - -Xdoclint:none - attach-javadocs