Add 'zero-copy' in libiscsi for reads.

It is not real zero-copy since the data is still copied in the kernel,
but it avoids copying the data inside libiscsi as well as in the callback.

For SCSI tasks that will return data from the target, the application can now
specify application buffers for libiscsi to read the data directly into.
This is done by calling scsi_task_add_data_in_buffer(task, ...

These buffers need not be linear, you can specify different areas to read into
by calling this function several times.

See examples/iscsiclient.c for an example.

examples/iscsiclient.c

@@ -13,7 +13,7 @@
  */
 
 /* This is the host/port we connect to.*/
-#define TARGET "127.0.0.1:3260"
+#define TARGET "10.1.1.27:3260"
 
 #include <stdio.h>
 #include <stdlib.h>
@@ -33,6 +33,8 @@ struct client_state {
        int block_size;
 };
 
+unsigned char small_buffer[512];
+
 void tm_at_cb(struct iscsi_context *iscsi _U_, int status _U_, void *command_data _U_, void *private_data)
 {
 	struct client_state *clnt = (struct client_state *)private_data;
@@ -105,8 +107,8 @@ void read10_cb(struct iscsi_context *iscsi, int status, void *command_data, void
 	}
 
 	printf("READ10 successful. Block content:\n");
-	for (i=0;i<task->datain.size;i++) {
-		printf("%02x ", task->datain.data[i]);
+	for (i=0;i<512;i++) {
+		printf("%02x ", small_buffer[i]);
 		if (i%16==15)
 			printf("\n");
 		if (i==69)
@@ -152,12 +154,19 @@ void read6_cb(struct iscsi_context *iscsi, int status, void *command_data, void
 	}
 	printf("...\n");
 
-	if (iscsi_read10_task(iscsi, clnt->lun, 0, clnt->block_size, clnt->block_size, read10_cb, private_data) == NULL) {
+	scsi_free_scsi_task(task);
+
+	if ((task = iscsi_read10_task(iscsi, clnt->lun, 0, clnt->block_size, clnt->block_size, read10_cb, private_data)) == NULL) {
 		printf("failed to send read10 command\n");
-		scsi_free_scsi_task(task);
 		exit(10);
 	}
-	scsi_free_scsi_task(task);
+	/* provide a buffer from the application to read into instead
+	 * of copying and linearizing the data. This saves two copies
+	 * of the data. One in libiscsi and one in the application
+	 * callback.
+	 */
+	scsi_task_add_data_in_buffer(task, 128, &small_buffer[0]);
+	scsi_task_add_data_in_buffer(task, 512-128, &small_buffer[128]);
 }
 
 void readcapacity10_cb(struct iscsi_context *iscsi, int status, void *command_data, void *private_data)

include/iscsi-private.h

@@ -248,6 +248,9 @@ int iscsi_send_target_nop_out(struct iscsi_context *iscsi, uint32_t ttt);
 void iscsi_set_error(struct iscsi_context *iscsi, const char *error_string,
 		     ...);
 
+unsigned char *iscsi_get_user_in_buffer(struct iscsi_context *iscsi, struct iscsi_in_pdu *in, uint32_t pos, ssize_t *count);
+unsigned char *scsi_task_get_data_in_buffer(struct scsi_task *task, uint32_t pos, ssize_t *count);
+
 unsigned long crc32c(char *buf, int len);
 
 #endif /* __iscsi_private_h__ */

include/iscsi.h

@@ -626,4 +626,29 @@ struct scsi_task *
 iscsi_synchronizecache10_sync(struct iscsi_context *iscsi, int lun, int lba,
 			      int num_blocks, int syncnv, int immed);
 
+
+/*
+ * This function is used when  the application wants to specify its own buffers to read the data
+ * from the DATA-IN PDUs into.
+ * The main use is for SCSI read operations to have them write directly into the application buffers to
+ * avoid the two copies that would occur otherwise.
+ * First copy from the individual DATA-IN blobs to linearize the buffer and the second in the callback
+ * to copy the data from the linearized buffer into the application buffer.
+ *
+ * This also supports reading into a vector of buffers by calling this function multiple times.
+ * The individual buffers will be filled in the same order as they were created.
+ *
+ * Example:
+ *     task = iscsi_read10_task(    ( 2 512byte blocks into two buffers)
+ *     scsi_task_add_data_buffer(task, first_buffer, 512
+ *     scsi_task_add_data_buffer(task, second_buffer, 512
+ *
+ *
+ * If you use this function you can not use task->datain in the callback.
+ * task->datain.size will be 0 and
+ * task->datain.data will be NULL
+ */
+int scsi_task_add_data_in_buffer(struct scsi_task *task, int len, unsigned char *buf);
+
+
 #endif /* __iscsi_h__ */

include/scsi-lowlevel.h

@@ -148,6 +148,8 @@ struct scsi_task {
 	uint32_t itt;
 	uint32_t cmdsn;
 	uint32_t lun;
+
+	struct scsi_data_buffer *in_buffers;
 };
 
 void scsi_free_scsi_task(struct scsi_task *task);

lib/crc32c.c

@@ -14,6 +14,7 @@
    You should have received a copy of the GNU Lesser General Public License
    along with this program; if not, see <http://www.gnu.org/licenses/>.
 */
+#include <unistd.h>
 #include "iscsi.h"
 #include "iscsi-private.h"
 

lib/nop.c

@@ -16,6 +16,7 @@
 */
 
 #include <stdio.h>
+#include <unistd.h>
 #include "iscsi.h"
 #include "iscsi-private.h"
 

lib/scsi-command.c

@@ -435,12 +435,15 @@ iscsi_process_scsi_data_in(struct iscsi_context *iscsi, struct iscsi_pdu *pdu,
 	}
 	dsl = ntohl(*(uint32_t *)&in->hdr[4])&0x00ffffff;
 
-	if (iscsi_add_data(iscsi, &pdu->indata,
-			   in->data, dsl, 0)
-	    != 0) {
-		iscsi_set_error(iscsi, "Out-of-memory: failed to add data "
+	/* Dont add to reassembly buffer if we already have a user buffer */
+	if (scsi_task_get_data_in_buffer(task, 0, NULL) == NULL) {
+		if (iscsi_add_data(iscsi, &pdu->indata,
+				   in->data, dsl, 0)
+		    != 0) {
+		    	iscsi_set_error(iscsi, "Out-of-memory: failed to add data "
 				"to pdu in buffer.");
-		return -1;
+			return -1;
+		}
 	}
 
 
@@ -733,3 +736,29 @@ iscsi_synchronizecache10_task(struct iscsi_context *iscsi, int lun, int lba,
 	return task;
 }
 
+unsigned char *
+iscsi_get_user_in_buffer(struct iscsi_context *iscsi, struct iscsi_in_pdu *in, uint32_t pos, ssize_t *count)
+{
+	struct iscsi_pdu *pdu;
+	uint32_t len, offset;
+	uint32_t itt;
+
+	if ((in->hdr[0] & 0x3f) != ISCSI_PDU_DATA_IN) {
+		return NULL;
+	}
+
+	len    = ntohl(*(uint32_t *)&in->hdr[4])&0x00ffffff;
+	offset = ntohl(*(uint32_t *)&in->hdr[40]);
+
+	itt = ntohl(*(uint32_t *)&in->hdr[16]);
+	for (pdu = iscsi->waitpdu; pdu; pdu = pdu->next) {
+		if (pdu->itt == itt) {
+			break;
+		}
+	}
+	if (pdu == NULL) {
+		return NULL;
+	}
+
+	return scsi_task_get_data_in_buffer(pdu->scsi_cbdata->task, offset + pos, count);
+}

lib/scsi-lowlevel.c

@@ -936,3 +936,50 @@ scsi_get_task_private_ptr(struct scsi_task *task)
 {
 	return task->ptr;
 }
+
+
+
+struct scsi_data_buffer {
+       struct scsi_data_buffer *next;
+       int len;
+       unsigned char *data;
+};
+
+int
+scsi_task_add_data_in_buffer(struct scsi_task *task, int len, unsigned char *buf)
+{
+	struct scsi_data_buffer *data_buf;
+
+	data_buf = scsi_malloc(task, sizeof(struct scsi_data_buffer));
+	if (data_buf == NULL) {
+		return -1;
+	}
+
+	data_buf->len  = len;
+	data_buf->data = buf;
+
+	SLIST_ADD_END(&task->in_buffers, data_buf);
+	return 0;
+}
+
+unsigned char *
+scsi_task_get_data_in_buffer(struct scsi_task *task, uint32_t pos, ssize_t *count)
+{
+	struct scsi_data_buffer *sdb;
+
+	sdb = task->in_buffers;
+	if (sdb == NULL) {
+		return NULL;
+	}
+
+	while (pos >= sdb->len) {
+	      pos -= sdb->len;
+	      sdb  = sdb->next;
+	}
+
+	if (count && *count > sdb->len - pos) {
+		*count = sdb->len - pos;
+	}
+
+	return &sdb->data[pos];
+}

lib/socket.c

@@ -267,13 +267,7 @@ iscsi_read_from_socket(struct iscsi_context *iscsi)
 
 	data_size = iscsi_get_pdu_data_size(&in->hdr[0]);
 	if (data_size != 0) {
-		if (in->data == NULL) {
-			in->data = malloc(data_size);
-			if (in->data == NULL) {
-				iscsi_set_error(iscsi, "Out-of-memory: failed to malloc iscsi_in_pdu->data(%d)", (int)data_size);
-				return -1;
-			}
-		}
+		unsigned char *buf = NULL;
 
 		/* No more data right now */
 		if (socket_count == 0) {
@@ -283,7 +277,22 @@ iscsi_read_from_socket(struct iscsi_context *iscsi)
 		if (count > socket_count) {
 			count = socket_count;
 		}
-		count = read(iscsi->fd, &in->data[in->data_pos], count);
+
+		/* first try to see if we already have a user buffer */
+		buf = iscsi_get_user_in_buffer(iscsi, in, in->data_pos, &count);
+		/* if not, allocate one */
+		if (buf == NULL) {
+			if (in->data == NULL) {
+				in->data = malloc(data_size);
+				if (in->data == NULL) {
+					iscsi_set_error(iscsi, "Out-of-memory: failed to malloc iscsi_in_pdu->data(%d)", (int)data_size);
+					return -1;
+				}
+			}
+			buf = &in->data[in->data_pos];
+		}
+
+		count = read(iscsi->fd, buf, count);
 		if (count < 0) {
 			if (errno == EINTR) {
 				return 0;

lib/task_mgmt.c

@@ -16,6 +16,7 @@
 */
 
 #include <stdio.h>
+#include <unistd.h>
 #include "iscsi.h"
 #include "iscsi-private.h"
 #include "scsi-lowlevel.h"