From 389707f28098e32f3f772de725e3e9e2cd0ba12c Mon Sep 17 00:00:00 2001 From: David Howells Date: Wed, 22 Jan 2014 00:55:24 +0000 Subject: [PATCH] Move to mostly asynchronous model and emit server side pieces Move to a mostly asynchronous model for sending and receiving RPC calls and also emit server side pieces. Signed-off-by: David Howells --- .gitignore | 6 + Makefile | 8 +- af_rxrpc.c | 1049 ++++++++++++++++++++++++++--------- af_rxrpc.h | 12 +- compile_pykafs.py | 5 + py_rxconn.c | 152 +++++ py_rxgen.c | 718 ++++++++++++++++++++---- py_rxgen.h | 28 +- rpc-api/bos.xg | 12 +- rpc-api/cb.xg | 18 +- rpc-api/fs.xg | 2 +- rpc-api/ka.xg | 73 ++- rpc-api/pts.xg | 9 +- rpc-api/vldb.xg | 1 - rpc-api/volumeserver.xg | 24 +- rxgen.h | 159 +++++- rxgen/emit_c_struct.pm | 36 +- rxgen/emit_c_sync_funcs.pm | 912 ++++++++++++++++-------------- rxgen/emit_py_module.pm | 50 +- rxgen/emit_py_sync_funcs.pm | 682 ++++++++++++++++------- rxgen/emit_py_types.pm | 62 ++- rxgen/rxgen.pl | 304 +++++----- vl-test.py | 65 +-- 23 files changed, 3084 insertions(+), 1303 deletions(-) create mode 100644 py_rxconn.c diff --git a/.gitignore b/.gitignore index b25c15b..64e3998 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,7 @@ *~ +.rxgen.check +afs_py.c +afs_py.h +afs_xg.c +afs_xg.h +*.so diff --git a/Makefile b/Makefile index 33be856..dc71d6b 100644 --- a/Makefile +++ b/Makefile @@ -7,12 +7,14 @@ GENERATED := afs_xg.c afs_xg.h afs_py.c afs_py.h pykafs.so: $(GENERATED) python3 compile_pykafs.py build -AFS_API := rpc-api/afsuuid.h rpc-api/vldb.xg +#AFS_API := rpc-api/afsuuid.h rpc-api/vldb.xg +AFS_API := $(sort $(wildcard rpc-api/*.h)) $(sort $(wildcard rpc-api/*.xg)) -$(GENERATED): $(AFS_API) $(RXGEN) +.rxgen.check $(GENERATED): $(AFS_API) $(RXGEN) ./rxgen/rxgen.pl $(AFS_API) + touch .rxgen.check clean: find \( -name "*~" -o -name "*.o" -o -name "*.so" \) -delete rm -rf build/ - rm -f $(GENERATED) + rm -f $(GENERATED) .rxgen.check diff --git a/af_rxrpc.c b/af_rxrpc.c index a8280c3..09f58d1 100644 --- a/af_rxrpc.c +++ b/af_rxrpc.c @@ -14,11 +14,16 @@ #include #include #include +#include #include #include #include "af_rxrpc.h" #include "rxgen.h" +#define RXGEN_CALL_MAGIC 0x52584745U +#define RXGEN_BUF_MAGIC (0x52420000U | __LINE__) +#define RXGEN_BUF_DEAD (0x6b6bU | __LINE__) + /* * dump the control messages */ @@ -108,178 +113,6 @@ void dump_cmsg(struct msghdr *msg) } } -/* - * Open a transport endpoint. - */ -static int open_af_rxrpc_endpoint(int local_port, unsigned short service, int exclusive) -{ - struct sockaddr_rxrpc srx; - int fd, ret; - - fd = socket(AF_RXRPC, SOCK_DGRAM, PF_INET); - if (fd < 0) - return -1; - - if (exclusive) { - ret = setsockopt(fd, SOL_RXRPC, RXRPC_EXCLUSIVE_CONNECTION, - NULL, 0); - if (ret == -1) - return -1; - } - - /* Bind an address to the local endpoint */ - srx.srx_family = AF_RXRPC; - srx.srx_service = service; - srx.transport_type = SOCK_DGRAM; - srx.transport_len = sizeof(srx.transport.sin); - srx.transport.sin.sin_family = AF_INET; - srx.transport.sin.sin_port = htons(local_port); - memset(&srx.transport.sin.sin_addr, 0, 4); - - ret = bind(fd, (struct sockaddr *)&srx, sizeof(srx)); - if (ret < 0) - return -1; - - return fd; -} - -/* - * Make a simple synchronous call. - */ -int rxrpc_simple_sync_call(struct sockaddr_rxrpc *srx, - int local_port, int exclusive, - const void *request, size_t reqlen, - void *reply, size_t replen, - uint32_t *_abort_code) -{ - struct cmsghdr *cmsg; - struct msghdr msg; - struct iovec iov[2]; - size_t ctrllen, received; - unsigned char control[128], overrun[sizeof(long)]; - void *preply; - int endpointfd; - int ret, got_eor; - - endpointfd = open_af_rxrpc_endpoint(local_port, 0, exclusive); - if (endpointfd < 0) - return -1; - - srx->srx_family = AF_RXRPC; - srx->transport_type = SOCK_DGRAM; - - /* request an operation */ - ctrllen = 0; - RXRPC_ADD_CALLID(control, ctrllen, 0x12345); - - iov[0].iov_base = (void *)request; - iov[0].iov_len = reqlen; - - msg.msg_name = srx; - msg.msg_namelen = sizeof(*srx); - msg.msg_iov = iov; - msg.msg_iovlen = 1; - msg.msg_control = control; - msg.msg_controllen = ctrllen; - msg.msg_flags = 0; - - //dump_cmsg(&msg); - - ret = sendmsg(endpointfd, &msg, 0); - if (ret == -1) { - close(endpointfd); - return -1; - } - - /* wait for a reply */ - preply = reply; - received = 0; - got_eor = 0; - while (!got_eor) { - iov[0].iov_base = preply; - iov[0].iov_len = replen - received; - iov[1].iov_base = &overrun; - iov[1].iov_len = sizeof(overrun); - - if (replen - received > 0) { - msg.msg_iov = iov; - msg.msg_iovlen = 2; - } else { - msg.msg_iov = iov + 1; - msg.msg_iovlen = 1; - } - - msg.msg_name = srx; - msg.msg_namelen = sizeof(*srx); - msg.msg_control = control; - msg.msg_controllen = sizeof(control); - msg.msg_flags = 0; - - ret = recvmsg(endpointfd, &msg, 0); - if (ret == -1) - return -1; - - //printf("RECV: %d [fl:%d]\n", ret, msg.msg_flags); - //printf("CMSG: %zu\n", msg.msg_controllen); - //printf("IOV: %zu [0]=%zu\n", msg.msg_iovlen, iov[0].iov_len); - - if (msg.msg_flags & MSG_EOR) - got_eor = 1; - - for (cmsg = CMSG_FIRSTHDR(&msg); cmsg; cmsg = CMSG_NXTHDR(&msg, cmsg)) { - unsigned char *p; - int n; - - if (cmsg->cmsg_level != SOL_RXRPC) - continue; - - n = cmsg->cmsg_len - CMSG_ALIGN(sizeof(*cmsg)); - p = CMSG_DATA(cmsg); - - switch (cmsg->cmsg_type) { - case RXRPC_ABORT: - if (n != sizeof(*_abort_code)) - goto internal_error; - memcpy(_abort_code, p, sizeof(*_abort_code)); - ret = ECONNABORTED; - goto error; - - case RXRPC_NET_ERROR: - case RXRPC_LOCAL_ERROR: - if (n != sizeof(ret)) - goto internal_error; - memcpy(&ret, p, sizeof(ret)); - goto error; - - case RXRPC_BUSY: - ret = ECONNREFUSED; - goto error; - - default: - break; - } - } - - if (ret > replen - received) { - ret = EMSGSIZE; - goto error; - } - - preply += ret; - received += ret; - } - - close(endpointfd); - return received; - -internal_error: - ret = EBADMSG; -error: - errno = ret; - close(endpointfd); - return -1; -} - /* * Set up a new connection */ @@ -392,30 +225,156 @@ void rx_close_connection(struct rx_connection *z_conn) } /* - * Send a request off to the service. + * Allocate a call structure. It is given one buffer ready to go. */ -int rxrpc_send_request(struct rx_connection *z_conn, - struct rx_call *call, - struct iovec *request, - int request_ioc) +struct rx_call *rxrpc_alloc_call(struct rx_connection *z_conn, + int incoming_call) +{ + struct rx_call *call; + struct rx_buf *buf; + uint8_t *data; + + call = calloc(1, sizeof(struct rx_call)); + if (!call) + return NULL; + + buf = calloc(1, sizeof(struct rx_buf)); + if (!buf) { + free(call); + return NULL; + } + + data = malloc(RXGEN_BUFFER_SIZE); + if (!data) { + free(buf); + free(call); + return NULL; + } + + //printf("Alloc: buf=%p data=%p\n", buf, data); + + buf->magic = RXGEN_BUF_MAGIC; + buf->buf = data; + + if (incoming_call) + call->state = rx_call_sv_not_started; + else + call->state = rx_call_cl_not_started; + + call->magic = RXGEN_CALL_MAGIC; + call->conn = z_conn; + call->more = 1; + call->buffer_head = buf; + call->buffer_tail = buf; + call->data_start = data; + call->data_cursor = data; + call->data_stop = incoming_call ? data : data + RXGEN_BUFFER_SIZE; + call->buffer_space = RXGEN_BUFFER_SIZE; + return call; +} + +static void rxrpc_check_buf(const struct rx_buf *buf) +{ + if (0) { + if ((buf->magic ^ RXGEN_BUF_MAGIC) & 0xffff0000U) + abort(); + if (buf->io_cursor > RXGEN_BUFFER_SIZE) + abort(); + } +} + +static void rxrpc_check_call(const struct rx_call *call) { + if (0) { + const struct rx_buf *cursor; + + if (call->magic != RXGEN_CALL_MAGIC) + abort(); + + for (cursor = call->buffer_head; cursor != call->buffer_tail; cursor = cursor->next) + rxrpc_check_buf(cursor); + rxrpc_check_buf(cursor); + if (cursor->next) + abort(); + } +} + +/* + * Send buffered data. + */ +int rxrpc_send_data(struct rx_call *call) +{ + struct rx_buf *cursor; struct msghdr msg; size_t ctrllen; unsigned char control[128]; - int ret; + struct iovec iov[16]; + int ioc, ret; + + //printf("-->rxrpc_send_data(%u,%u)\n", call->state, call->data_count); - /* request an operation */ + rxrpc_check_call(call); + + /* Switch into encode state */ + switch (call->state) { + case rx_call_cl_not_started: + case rx_call_sv_processing: + call->state++; + case rx_call_cl_encoding_params: + case rx_call_sv_encoding_response: + break; + default: + fprintf(stderr, "RxRPC: Send in bad call state (%d)\n", call->state); + abort(); + } + + /* Request an operation */ ctrllen = 0; RXRPC_ADD_CALLID(control, ctrllen, (unsigned long)call); - msg.msg_name = &z_conn->peer; - msg.msg_namelen = sizeof(z_conn->peer); - msg.msg_iov = request; - msg.msg_iovlen = request_ioc; + msg.msg_name = &call->conn->peer; + msg.msg_namelen = sizeof(call->conn->peer); + msg.msg_iov = iov; msg.msg_control = control; msg.msg_controllen = ctrllen; msg.msg_flags = 0; + /* We may have a completely sent buffer on the front of the queue if + * this was the last buffer on the last send. The buffer queue isn't + * allowed to be empty at any point. + */ + cursor = call->buffer_head; + if (cursor->io_cursor == RXGEN_BUFFER_SIZE) { + struct rx_buf *sent = cursor; + if (sent == call->buffer_tail) + abort(); + call->buffer_head = cursor = sent->next; + sent->magic = RXGEN_BUF_DEAD; + free(sent->buf); + free(sent); + } + + for (ioc = 0; ioc < 16; ioc++) { + rxrpc_check_buf(cursor); + + unsigned io_cursor = cursor->io_cursor; + unsigned end = RXGEN_BUFFER_SIZE; + + if (io_cursor == RXGEN_BUFFER_SIZE) + abort(); + if (cursor == call->buffer_tail) + end = call->data_cursor - cursor->buf; + + iov[ioc].iov_base = cursor->buf + io_cursor; + iov[ioc].iov_len = end - io_cursor; + if (cursor == call->buffer_tail) { + ioc++; + break; + } + } + msg.msg_iovlen = ioc; + + /* Send the data */ //dump_cmsg(&msg); if (0) { @@ -445,133 +404,673 @@ int rxrpc_send_request(struct rx_connection *z_conn, printf("FLAGS %x\n", msg.msg_flags); } - ret = sendmsg(z_conn->fd, &msg, 0) == -1 ? -1 : 0; + ret = sendmsg(call->conn->fd, &msg, call->more ? MSG_MORE : 0) == -1 ? -1 : 0; if (0) { printf("SENDMSG: %d\n", ret); } - return ret; + if (ret == -1) + return -1; + + call->data_count -= ret; + call->known_to_kernel = 1; + + /* Free up any completely sent buffers, without completely emptying the + * queue. + */ + cursor = call->buffer_head; + do { + struct rx_buf *sent = cursor; + unsigned count = RXGEN_BUFFER_SIZE - cursor->io_cursor; + + if (count > ret) + count = ret; + cursor->io_cursor += count; + ret -= count; + if (cursor == call->buffer_tail) + break; + sent = cursor; + cursor = cursor->next; + call->buffer_head = cursor; + rxrpc_check_buf(sent); + sent->magic = RXGEN_BUF_DEAD; + free(sent->buf); + free(sent); + } while (ret > 0); + + rxrpc_check_call(call); + + if (call->data_count == 0 && !call->more) + call->state++; + + if (!call->more) { + if (call->state == rx_call_cl_encoding_params || + call->state == rx_call_sv_encoding_response) + call->state++; + } + + if (call->state == rx_call_cl_waiting_for_response) { + /* Prepare to decode the response */ + //printf("Prep to decode\n"); + call->data_stop = call->data_cursor = call->data_start = call->buffer_head->buf; + call->buffer_head->io_cursor = 0; + call->data_count = 0; + } + + return 0; } /* - * Wait for a reply to arrive from a single synchronous RPC request. + * Receive data from a socket. */ -int rxrpc_wait_for_sync_reply(struct rx_connection *z_conn, - struct rx_call *call) +int rxrpc_recv_data(struct rx_connection *z_conn) { + struct rx_call *call; + struct rx_buf *bufs[4] = { NULL }, *cursor; struct sockaddr_rxrpc srx; struct cmsghdr *cmsg; struct msghdr msg; - struct iovec iov[2]; + struct iovec iov[4]; unsigned char control[128]; - int ret; - + uint32_t tmpbuf[1]; + int ioc, ret; + + /* Peek at the next message */ + iov[0].iov_base = &tmpbuf; + iov[0].iov_len = sizeof(tmpbuf); + + memset(&msg, 0, sizeof(msg)); + msg.msg_iov = iov; + msg.msg_iovlen = 1; + msg.msg_name = &srx; + msg.msg_namelen = sizeof(srx); + msg.msg_control = control; + msg.msg_controllen = sizeof(control); + msg.msg_flags = 0; + + ret = recvmsg(z_conn->fd, &msg, MSG_PEEK); if (0) { - printf("%s(,{%u,%s,%u/%u})\n", - __func__, call->phase, call->got_eor ? "EOR" : "-", - CIRC_CNT(call->head, call->tail, call->size), call->size); + printf("RECVMSG: %d\n", ret); } + if (ret == -1) + return -1; - /* wait for a reply */ - while (!call->got_eor && - CIRC_CNT(call->head, call->tail, call->size) < call->need_size - ) { - struct iovec *piov = iov; - unsigned size = call->size; - unsigned mask = size - 1; - unsigned head = call->head & mask; - unsigned tail = call->tail & mask; - unsigned space = CIRC_SPACE(head, tail, size); - - if (head >= tail) { - piov->iov_base = (void *)call->reply + head; - piov->iov_len = size - head; - if (piov->iov_len > space) - piov->iov_len = space; - space -= piov->iov_len; - piov++; - if (space > 0) { - piov->iov_base = (void *)call->reply; - piov->iov_len = space; - piov++; + /* Find the call ID. */ + call = NULL; + for (cmsg = CMSG_FIRSTHDR(&msg); cmsg; cmsg = CMSG_NXTHDR(&msg, cmsg)) { + unsigned char *p; + + if (cmsg->cmsg_level != SOL_RXRPC || + cmsg->cmsg_type != RXRPC_USER_CALL_ID) + continue; + + p = CMSG_DATA(cmsg); + call = *(struct rx_call **)p; + break; + } + if (!call) + abort(); + + /* Now actually retrieve that message */ + memset(&msg, 0, sizeof(msg)); + msg.msg_iov = iov; + msg.msg_iovlen = 0; + msg.msg_name = &srx; + msg.msg_namelen = sizeof(srx); + msg.msg_control = control; + msg.msg_controllen = sizeof(control); + msg.msg_flags = 0; + + //printf("Recv: buf[0]=%p data[0]=%p (io=%u)\n", + // call->buffer_tail, call->buffer_tail->buf, call->buffer_tail->io_cursor); + + rxrpc_check_call(call); + + /* Set up some buffers */ + if (call->buffer_tail->io_cursor < RXGEN_BUFFER_SIZE) { + bufs[0] = call->buffer_tail; + iov[0].iov_base = bufs[0]->buf + bufs[0]->io_cursor; + iov[0].iov_len = RXGEN_BUFFER_SIZE - bufs[0]->io_cursor; + msg.msg_iovlen = 1; + } else { + msg.msg_iovlen = 0; + } + + for (ioc = msg.msg_iovlen; ioc < 4; ioc++) { + bufs[ioc] = calloc(1, sizeof(struct rx_buf)); + bufs[ioc]->magic = RXGEN_BUF_MAGIC; + if (!bufs[ioc]) { + while (--ioc >= msg.msg_iovlen) { + bufs[ioc]->magic = RXGEN_BUF_DEAD; + free(bufs[ioc]->buf); + free(bufs[ioc]); } - } else { - piov->iov_base = (void *)call->reply + head; - piov->iov_len = space; - piov++; + return -1; } + bufs[ioc]->buf = malloc(RXGEN_BUFFER_SIZE); + if (!bufs[ioc]->buf) { + bufs[ioc]->magic = RXGEN_BUF_DEAD; + free(bufs[ioc]); + while (--ioc >= msg.msg_iovlen) { + bufs[ioc]->magic = RXGEN_BUF_DEAD; + free(bufs[ioc]->buf); + free(bufs[ioc]); + } + return -1; + } + iov[ioc].iov_base = bufs[ioc]->buf; + iov[ioc].iov_len = RXGEN_BUFFER_SIZE; + } + msg.msg_iovlen = 4; - msg.msg_iov = iov; - msg.msg_iovlen = piov - iov; - - memcpy(&srx, &z_conn->peer, sizeof(struct sockaddr_rxrpc)); - msg.msg_name = &srx; - msg.msg_namelen = sizeof(srx); - msg.msg_control = control; - msg.msg_controllen = sizeof(control); - msg.msg_flags = 0; + ret = recvmsg(z_conn->fd, &msg, 0); + if (0) { + printf("RECVMSG: %d\n", ret); + } + if (ret == -1) + return -1; - if (0) { - unsigned i; - for (i = 0; i < msg.msg_iovlen; i++) - printf("RECV IOV[%02u] %04zu\n", i, msg.msg_iov[i].iov_len); + //printf("RECV: %d [fl:%d]\n", ret, msg.msg_flags); + //printf("CMSG: %zu\n", msg.msg_controllen); + //printf("IOV: %zu [0]=%zu\n", msg.msg_iovlen, iov[0].iov_len); + + /* Attach any used buffers to the call and discard the rest */ + if (ret > 0) { + for (ioc = 0; ioc < 4 && ret > 0; ioc++) { + unsigned added = RXGEN_BUFFER_SIZE - bufs[ioc]->io_cursor; + //printf("xfer[%d] space=%u rem=%d\n", ioc, added, ret); + if (added > ret) + added = ret; + bufs[ioc]->io_cursor += added; + call->data_count += added; + ret -= added; + if (call->buffer_tail == bufs[ioc]) + continue; + call->buffer_tail->next = bufs[ioc]; + call->buffer_tail = bufs[ioc]; + if (ret <= 0) { + ioc++; + break; + } } - ret = recvmsg(z_conn->fd, &msg, 0); - if (0) { - printf("RECVMSG: %d\n", ret); + for (; ioc < 4; ioc++) { + bufs[ioc]->magic = RXGEN_BUF_DEAD; + free(bufs[ioc]->buf); + free(bufs[ioc]); } - if (ret == -1) - return -1; - if (ret > 0) - call->head += ret; + } - //printf("RECV: %d [fl:%d]\n", ret, msg.msg_flags); - //printf("CMSG: %zu\n", msg.msg_controllen); - //printf("IOV: %zu [0]=%zu\n", msg.msg_iovlen, iov[0].iov_len); + rxrpc_check_call(call); - if (msg.msg_flags & MSG_EOR) - call->got_eor = 1; + if (0) { + for (cursor = call->buffer_head; cursor; cursor = cursor->next) + printf("Recv buf=%p data=%p ioc=%u\n", + cursor, cursor->buf, cursor->io_cursor); + } - for (cmsg = CMSG_FIRSTHDR(&msg); cmsg; cmsg = CMSG_NXTHDR(&msg, cmsg)) { - unsigned char *p; - int n; + /* Process the metadata */ + if (msg.msg_flags & MSG_EOR) + call->known_to_kernel = 0; + if (!(msg.msg_flags & MSG_MORE)) + call->more = 0; - if (cmsg->cmsg_level != SOL_RXRPC) - continue; + for (cmsg = CMSG_FIRSTHDR(&msg); cmsg; cmsg = CMSG_NXTHDR(&msg, cmsg)) { + unsigned char *p; + int n; - n = cmsg->cmsg_len - CMSG_ALIGN(sizeof(*cmsg)); - p = CMSG_DATA(cmsg); + if (cmsg->cmsg_level != SOL_RXRPC) + continue; - switch (cmsg->cmsg_type) { - case RXRPC_ABORT: - if (n != sizeof(call->abort_code)) - call->abort_code = 0; - else - memcpy(&call->abort_code, p, - sizeof(call->abort_code)); - z_conn->last_abort_code = call->abort_code; - errno = ECONNABORTED; - return call->abort_code; + n = cmsg->cmsg_len - CMSG_ALIGN(sizeof(*cmsg)); + p = CMSG_DATA(cmsg); - case RXRPC_NET_ERROR: - case RXRPC_LOCAL_ERROR: - if (n != sizeof(ret)) { - errno = EBADMSG; - return -1; - } - memcpy(&ret, p, sizeof(ret)); - errno = ret; + switch (cmsg->cmsg_type) { + case RXRPC_ABORT: + if (n != sizeof(call->abort_code)) + call->abort_code = 0; + else + memcpy(&call->abort_code, p, sizeof(call->abort_code)); + z_conn->last_abort_code = call->abort_code; + call->error_code = ECONNABORTED; + call->state = rx_call_remotely_aborted; + break; + + case RXRPC_NET_ERROR: + if (n != sizeof(ret)) { + errno = EBADMSG; return -1; - - case RXRPC_BUSY: - errno = ECONNREFUSED; + } + memcpy(&ret, p, sizeof(ret)); + call->error_code = ret; + call->state = rx_call_net_error; + break; + + case RXRPC_LOCAL_ERROR: + if (n != sizeof(ret)) { + errno = EBADMSG; return -1; + } + memcpy(&ret, p, sizeof(ret)); + call->error_code = ret; + call->state = rx_call_local_error; + break; + + case RXRPC_BUSY: + call->error_code = ECONNREFUSED; + call->state = rx_call_rejected_busy; + break; + + case RXRPC_ACK: + if (call->state == rx_call_sv_waiting_for_final_ack) { + call->state = rx_call_sv_complete; + } else { + fprintf(stderr, "RxRPC: Recv-Ack in bad call state (%d)\n", + call->state); + abort(); + } + break; - default: - break; + case RXRPC_RESPONSE: + call->secured = 1; + break; + + default: + break; + } + } + + /* Switch into appropriate decode state */ +loop: + switch (call->state) { + case rx_call_cl_waiting_for_response: + case rx_call_sv_waiting_for_opcode: + call->state++; + case rx_call_cl_decoding_response: + case rx_call_sv_decoding_opcode: + case rx_call_sv_decoding_params: + if (call->data_count < call->need_size) { + if (!(msg.msg_flags & MSG_MORE)) { + /* Short data */ + call->error_code = EMSGSIZE; + rxrpc_abort_call(call, 1); } + return 0; + } + + /* Handle reception of the opcode in a server-side call. Note + * that a call might have no parameters, in which case we + * should've got !MSG_MORE and incremented the state again. + */ + if (call->state == rx_call_sv_decoding_opcode) { + fprintf(stderr, "Need to determine service based on opcode\n"); + abort(); + call->state++; + goto loop; + } + + ret = call->decoder(call); + if (ret < 0) { + rxrpc_abort_call(call, 1); + break; + } + + if (ret == 1) + goto loop; /* The decoder wants more data */ + + call->state++; + case rx_call_cl_wait_for_no_MSG_MORE: + case rx_call_sv_wait_for_no_MSG_MORE: + rxgen_dec_discard_excess(call); + if (!(msg.msg_flags & MSG_MORE)) { + call->state++; + if (call->state == rx_call_sv_processing) { + /* Prepare to encode the response */ + call->data_cursor = call->data_start = call->buffer_head->buf; + call->data_stop = call->buffer_head->buf + RXGEN_BUFFER_SIZE; + call->buffer_head->io_cursor = 0; + call->data_count = 0; + call->buffer_space = 0; + + /* Invoke the processor */ + call->processor(call); + return 0; + } + } + break; + + case rx_call_cl_complete: + case rx_call_sv_complete ... rx_call_rejected_busy: + break; + + case rx_call_sv_processing: + call->failed(call); + return 0; + + default: + fprintf(stderr, "RxRPC: Recv in bad call state (%d)\n", call->state); + abort(); + } + + return 0; +} + +/* + * Abort a call. + */ +void rxrpc_abort_call(struct rx_call *call, uint32_t abort_code) +{ + struct msghdr msg; + size_t ctrllen; + unsigned char control[128]; + + if (call->known_to_kernel) { + ctrllen = 0; + RXRPC_ADD_CALLID(control, ctrllen, (unsigned long)call); + RXRPC_ADD_ABORT(control, ctrllen, abort_code); + + msg.msg_name = &call->conn->peer; + msg.msg_namelen = sizeof(call->conn->peer); + msg.msg_iov = NULL; + msg.msg_iovlen = 0; + msg.msg_control = control; + msg.msg_controllen = ctrllen; + msg.msg_flags = 0; + + sendmsg(call->conn->fd, &msg, 0); + call->known_to_kernel = 0; + } + call->state = rx_call_locally_aborted; +} + +/* + * Terminate a call, aborting it if necessary. + */ +void rxrpc_terminate_call(struct rx_call *call, uint32_t abort_code) +{ + struct rx_buf *cursor, *next; + + rxrpc_check_call(call); + if (call->known_to_kernel) + rxrpc_abort_call(call, abort_code); + call->magic = 0x7a7b7c7d; + for (cursor = call->buffer_head; cursor; cursor = next) { + rxrpc_check_buf(cursor); + next = cursor->next; + cursor->magic = RXGEN_BUF_DEAD; + free(cursor->buf); + free(cursor); + } + free(call); +} + +/* + * Run a single call synchronously. + */ +int rxrpc_run_sync_call(struct rx_call *call) +{ + struct pollfd fds[1]; + + while (call->known_to_kernel) { + fds[0].fd = call->conn->fd; + fds[0].events = POLLIN; + fds[0].revents = 0; + + if (poll(fds, 1, -1) == -1) { + fprintf(stderr, "Poll failed: %m\n"); + return -1; + } + if (rxrpc_recv_data(call->conn)) { + fprintf(stderr, "rxrpc_recv_data failed: %m\n"); + return -1; } } + switch (call->state) { + case rx_call_cl_complete: + //printf("Call complete\n"); + return 0; + case rx_call_remotely_aborted ... rx_call_rejected_busy: + //printf("Call failed\n"); + errno = call->error_code; + return -1; + default: + fprintf(stderr, "RxRPC: Ended in bad call state (%d)\n", call->state); + abort(); + } +} + +/* + * Slow path for decoding from data read from the buffer list. + */ +void rxrpc_enc_slow(struct rx_call *call, net_xdr_t data) +{ + struct rx_buf *cursor, *new; + uint8_t *buf; + + rxrpc_check_call(call); + + if (call->error_code) + return; + if (call->data_cursor != call->data_stop) + abort(); + rxrpc_post_enc(call); + + new = calloc(1, sizeof(struct rx_buf)); + if (!new) + goto handle_oom; + new->buf = buf = malloc(RXGEN_BUFFER_SIZE); + if (!buf) + goto handle_oom_buf; + new->magic = RXGEN_BUF_MAGIC; + + *(net_xdr_t *)buf = data; + + cursor = call->buffer_tail; + cursor->next = new; + call->data_cursor = call->data_start = buf + sizeof(net_xdr_t); + call->data_stop = buf + RXGEN_BUFFER_SIZE; + call->buffer_space += RXGEN_BUFFER_SIZE - sizeof(net_xdr_t); + call->buffer_tail = new; + return; + +handle_oom_buf: + new->magic = RXGEN_BUF_DEAD; + free(new); +handle_oom: + call->error_code = ENOMEM; +} + +/* + * Slow path for decoding from data read from the buffer list. + */ +uint32_t rxrpc_dec_slow(struct rx_call *call) +{ + struct rx_buf *cursor, *spent; + net_xdr_t x; + unsigned count; + uint8_t *new_stop; + + rxrpc_check_call(call); + rxrpc_post_dec(call); + + /* More data may have come into a partially filled buffer from which we + * previously read. + */ + cursor = call->buffer_head; + count = cursor->io_cursor & ~3; + new_stop = cursor->buf + count; + if (call->data_stop >= new_stop) { + /* This buffer must then be completely used as we're required to check + * amount received before reading it. + */ + if (new_stop != cursor->buf + RXGEN_BUFFER_SIZE) + abort(); /* Didn't completely consume a buffer */ + if (call->buffer_tail == cursor) + abort(); /* Unexpectedly out of data */ + + /* Move to the next buffer */ + spent = cursor; + cursor = cursor->next; + call->buffer_head = cursor; + + call->data_cursor = call->data_start = cursor->buf; + count = cursor->io_cursor & ~3; + if (count == 0) + abort(); + new_stop = cursor->buf + count; + spent->magic = RXGEN_BUF_DEAD; + free(spent->buf); + free(spent); + } + + call->data_stop = new_stop; + x = *(net_xdr_t *)call->data_cursor; + call->data_cursor += sizeof(x); + return ntohl(x); +} + +/* + * Discard excess received data + */ +int rxgen_dec_discard_excess(struct rx_call *call) +{ + struct rx_buf *spent, *cursor; + unsigned io_cursor; + + rxrpc_check_call(call); + + while (cursor = call->buffer_head, + cursor != call->buffer_tail + ) { + spent = cursor; + call->buffer_head = cursor->next; + spent->magic = RXGEN_BUF_DEAD; + free(spent->buf); + free(spent); + } + + rxrpc_check_call(call); + + io_cursor = cursor->io_cursor; + call->data_start = call->data_cursor = call->data_stop = cursor->buf + io_cursor; + call->data_count = 0; + + if (__builtin_expect(call->error_code, 0)) + return -1; return 0; } + +/* + * Encode a string of bytes + */ +int rxrpc_enc_bytes(struct rx_call *call, const void *data, size_t size) +{ + struct rx_buf *cursor, *new; + uint8_t *buf; + size_t seg; + + rxrpc_check_call(call); + + if (size == 0) + return 0; + if (call->error_code) + return -1; + + for (;;) { + if (call->data_cursor < call->data_stop) { + seg = call->data_stop - call->data_cursor; + if (seg > size) + seg = size; + memcpy(call->data_cursor, data, seg); + data += seg; + size -= seg; + call->data_cursor += seg; + if (size <= 0) + return 0; + } + + rxrpc_check_call(call); + + if (rxrpc_post_enc(call) < 0) + return -1; + + new = calloc(1, sizeof(struct rx_buf)); + if (!new) + goto handle_oom; + new->buf = buf = malloc(RXGEN_BUFFER_SIZE); + if (!buf) + goto handle_oom_buf; + + new->magic = RXGEN_BUF_MAGIC; + cursor = call->buffer_tail; + cursor->next = new; + call->data_cursor = call->data_start = buf; + call->data_stop = buf + RXGEN_BUFFER_SIZE; + call->buffer_space += RXGEN_BUFFER_SIZE; + call->buffer_tail = new; + } + +handle_oom_buf: + free(new); +handle_oom: + call->error_code = ENOMEM; + return -1; +} + +/* + * Decode a string of bytes + */ +void rxrpc_dec_bytes(struct rx_call *call) +{ + struct rx_buf *cursor, *spent; + unsigned needed, count; + uint8_t *new_stop; + + rxrpc_check_call(call); + rxrpc_post_dec(call); + + needed = call->bulk_count - call->bulk_index; + count = call->data_stop - call->data_cursor; + if (count > 0) { + if (count > needed) + count = needed; + memcpy(call->bulk_item + call->bulk_index, call->data_cursor, count); + call->bulk_index += count; + if (call->bulk_index >= call->bulk_count) + return; + } + + /* More data may have come into a partially filled buffer from which we + * previously read. + */ + cursor = call->buffer_head; + count = cursor->io_cursor & ~3; + new_stop = cursor->buf + count; + if (call->data_stop >= new_stop) { + /* This buffer must then be completely used as we're required to check + * amount received before reading it. + */ + if (new_stop != cursor->buf + RXGEN_BUFFER_SIZE) + abort(); /* Didn't completely consume a buffer */ + if (call->buffer_tail == cursor) + abort(); /* Unexpectedly out of data */ + + /* Move to the next buffer */ + spent = cursor; + cursor = cursor->next; + call->buffer_head = cursor; + + call->data_cursor = call->data_start = cursor->buf; + count = cursor->io_cursor & ~3; + if (count == 0) + abort(); + new_stop = cursor->buf + count; + + spent->magic = RXGEN_BUF_DEAD; + free(spent->buf); + free(spent); + } + + call->data_stop = new_stop; + rxrpc_check_call(call); +} diff --git a/af_rxrpc.h b/af_rxrpc.h index 80041fb..ee01f92 100644 --- a/af_rxrpc.h +++ b/af_rxrpc.h @@ -100,8 +100,8 @@ do { \ void *__buffer = (control); \ unsigned int *__data; \ struct cmsghdr *__cmsg; \ - __cmsg = (void *)(control) + (ctrllen); \ - __cmsg->cmsg_len = CMSG_LEN(sizeof((__data)); \ + __cmsg = __buffer + (ctrllen); \ + __cmsg->cmsg_len = CMSG_LEN(sizeof(__data)); \ __cmsg->cmsg_level = SOL_RXRPC; \ __cmsg->cmsg_type = RXRPC_ABORT; \ __data = (void *)CMSG_DATA(__cmsg); \ @@ -110,12 +110,4 @@ do { \ \ } while (0) -/* - * af_rxrpc.c - */ -extern int rxrpc_simple_sync_call(struct sockaddr_rxrpc *srx, - int local_port, int exclusive, - const void *request, size_t reqlen, - void *reply, size_t replen, - uint32_t *_abort_code); #endif /* AF_RXRPC_H */ diff --git a/compile_pykafs.py b/compile_pykafs.py index 0b48c7f..2277e14 100644 --- a/compile_pykafs.py +++ b/compile_pykafs.py @@ -5,6 +5,11 @@ setup(name="kafs", version="0.1", "kafs.c", "afs_py.c", "py_rxgen.c", + "py_rxconn.c", "af_rxrpc.c" ], + extra_compile_args = [ + "-O0", + "-Wp,-U_FORTIFY_SOURCE", + ], )]) diff --git a/py_rxconn.c b/py_rxconn.c new file mode 100644 index 0000000..0203ddd --- /dev/null +++ b/py_rxconn.c @@ -0,0 +1,152 @@ +/* Python RxRPC connection container object + * + * Copyright (C) 2014 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public Licence + * as published by the Free Software Foundation; either version + * 2 of the Licence, or (at your option) any later version. + */ + +#include +#include "structmember.h" +#include +#include "py_rxgen.h" +#include "rxgen.h" + +#if 0 +/* + * Send an RPC call. + */ +static PyObject *py_rx_conn_send(PyObject *_self, PyObject *args) +{ + struct py_rx_connection *self = (struct py_rx_connection *)_self; + self->x = NULL; + return 0; +} + +/* + * Methods applicable to RxRPC connections + */ +static PyMethodDef py_rx_connection_methods[] = { + {"send", (PyCFunction)py_rx_conn_send, METH_VARARGS, + "" }, + {} +}; +#endif + +/* + * RxRPC connection container. + */ +static int +py_rx_connection_init(PyObject *_self, PyObject *args, PyObject *kwds) +{ + struct py_rx_connection *self = (struct py_rx_connection *)_self; + self->x = NULL; + return 0; +} + +static void +py_rx_connection_dealloc(struct py_rx_connection *self) +{ + if (self->x) { + rx_close_connection(self->x); + self->x = NULL; + } + Py_TYPE(self)->tp_free((PyObject *)self); +} + +PyTypeObject py_rx_connectionType = { + PyVarObject_HEAD_INIT(NULL, 0) + "kafs.rx_connection", /*tp_name*/ + sizeof(struct py_rx_connection), /*tp_basicsize*/ + 0, /*tp_itemsize*/ + (destructor)py_rx_connection_dealloc, /*tp_dealloc*/ + 0, /*tp_print*/ + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ + 0, /*tp_compare*/ + 0, /*tp_repr*/ + 0, /*tp_as_number*/ + 0, /*tp_as_sequence*/ + 0, /*tp_as_mapping*/ + 0, /*tp_hash */ + 0, /*tp_call*/ + 0, /*tp_str*/ + 0, /*tp_getattro*/ + 0, /*tp_setattro*/ + 0, /*tp_as_buffer*/ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /*tp_flags*/ + "RxRPC connection container", /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + 0, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + py_rx_connection_init, /* tp_init */ + 0, /* tp_alloc */ + 0, /* tp_new */ +}; + +/* + * Set up an RxRPC connection. + */ +PyObject * +kafs_py_rx_new_connection(PyObject *_self, PyObject *args) +{ + struct py_rx_connection *obj; + struct rx_connection *z_conn; + union { + struct sockaddr sa; + struct sockaddr_in sin; + struct sockaddr_in6 sin6; + } sa; + const char *address = NULL; + socklen_t salen; + uint16_t port, service, local_port = 0, local_service = 0; + int exclusive = 0; + + if (!PyArg_ParseTuple(args, "sHH|HHp", + &address, &port, &service, + &local_port, &local_service, &exclusive)) + return NULL; + + memset(&sa, 0, sizeof(sa)); + if (inet_pton(AF_INET, address, &sa.sin.sin_addr)) { + sa.sin.sin_family = AF_INET; + sa.sin.sin_port = htons(port); + salen = sizeof(sa.sin); + } else if (inet_pton(AF_INET6, address, &sa.sin.sin_addr)) { + sa.sin6.sin6_family = AF_INET6; + sa.sin6.sin6_port = htons(port); + salen = sizeof(sa.sin6); + } else { + return PyExc_TypeError; + } + + obj = (struct py_rx_connection *)_PyObject_New(&py_rx_connectionType); + if (!obj) + return PyExc_MemoryError; + py_rx_connection_init((PyObject *)obj, NULL, NULL); + assert(obj->x == NULL); + + z_conn = rx_new_connection(&sa.sa, salen, service, + local_port, local_service, exclusive); + if (!z_conn) { + Py_DECREF(obj); + return errno == ENOMEM ? PyExc_MemoryError : + PyErr_SetFromErrno(PyExc_IOError); + } + obj->x = z_conn; + return (PyObject *)obj; +} diff --git a/py_rxgen.c b/py_rxgen.c index b05ca71..b32a0c7 100644 --- a/py_rxgen.c +++ b/py_rxgen.c @@ -97,6 +97,41 @@ PyObject *py_rxgen_get_uint8(const void *_array, size_t n, PyObject **cache) return *cache; } + list = PyList_New(n); + if (!list) + return NULL; + + for (i = 0; i < n; i++) { + PyObject *num = PyLong_FromUnsignedLong(array[i]); + if (!num) + goto error; + + if (PyList_SetItem(list, i, num) != 0) { + Py_DECREF(num); + goto error; + } + } + + Py_INCREF(list); + *cache = list; + return list; + +error: + Py_DECREF(list); + return NULL; +} + +PyObject *py_rxgen_get_int8(const void *_array, size_t n, PyObject **cache) +{ + PyObject *list; + const int8_t *array = _array; + int i; + + if (*cache) { + Py_INCREF(*cache); + return *cache; + } + list = PyList_New(n); if (!list) return NULL; @@ -132,6 +167,41 @@ PyObject *py_rxgen_get_uint16(const void *_array, size_t n, PyObject **cache) return *cache; } + list = PyList_New(n); + if (!list) + return NULL; + + for (i = 0; i < n; i++) { + PyObject *num = PyLong_FromUnsignedLong(array[i]); + if (!num) + goto error; + + if (PyList_SetItem(list, i, num) != 0) { + Py_DECREF(num); + goto error; + } + } + + Py_INCREF(list); + *cache = list; + return list; + +error: + Py_DECREF(list); + return NULL; +} + +PyObject *py_rxgen_get_int16(const void *_array, size_t n, PyObject **cache) +{ + PyObject *list; + const int16_t *array = _array; + int i; + + if (*cache) { + Py_INCREF(*cache); + return *cache; + } + list = PyList_New(n); if (!list) return NULL; @@ -167,6 +237,41 @@ PyObject *py_rxgen_get_uint32(const void *_array, size_t n, PyObject **cache) return *cache; } + list = PyList_New(n); + if (!list) + return NULL; + + for (i = 0; i < n; i++) { + PyObject *num = PyLong_FromUnsignedLong(array[i]); + if (!num) + goto error; + + if (PyList_SetItem(list, i, num) != 0) { + Py_DECREF(num); + goto error; + } + } + + Py_INCREF(list); + *cache = list; + return list; + +error: + Py_DECREF(list); + return NULL; +} + +PyObject *py_rxgen_get_int32(const void *_array, size_t n, PyObject **cache) +{ + PyObject *list; + const int32_t *array = _array; + int i; + + if (*cache) { + Py_INCREF(*cache); + return *cache; + } + list = PyList_New(n); if (!list) return NULL; @@ -322,6 +427,123 @@ error: return -1; } +int py_rxgen_premarshal_int8(void *_array, size_t n, PyObject *cache) +{ + PyObject *list; + int8_t *array = _array; + Py_ssize_t i, c; + + if (!cache) + return 0; + + list = PySequence_Fast(cache, "Expecting list or tuple of integers"); + if (!list) + return -1; + c = PySequence_Fast_GET_SIZE(list); + if (c > n) { + PyErr_Format(PyExc_ValueError, + "Expected a sequence of up to %zu size", n); + goto error; + } + + PyErr_Clear(); + for (i = 0; i < c; i++) { + PyObject *p = PySequence_Fast_GET_ITEM(list, i); + unsigned long val = PyLong_AsLong(p); + + if (PyErr_Occurred()) + goto error; + array[i] = val; + } + for (; i < n; i++) + array[i] = 0; + + Py_DECREF(list); + return 0; + +error: + Py_DECREF(list); + return -1; +} + +int py_rxgen_premarshal_int16(void *_array, size_t n, PyObject *cache) +{ + PyObject *list; + int16_t *array = _array; + Py_ssize_t i, c; + + if (!cache) + return 0; + + list = PySequence_Fast(cache, "Expecting list or tuple of integers"); + if (!list) + return -1; + c = PySequence_Fast_GET_SIZE(list); + if (c > n) { + PyErr_Format(PyExc_ValueError, + "Expected a sequence of up to %zu size", n); + goto error; + } + + PyErr_Clear(); + for (i = 0; i < c; i++) { + PyObject *p = PySequence_Fast_GET_ITEM(list, i); + unsigned long val = PyLong_AsLong(p); + + if (PyErr_Occurred()) + goto error; + array[i] = val; + } + for (; i < n; i++) + array[i] = 0; + + Py_DECREF(list); + return 0; + +error: + Py_DECREF(list); + return -1; +} + +int py_rxgen_premarshal_int32(void *_array, size_t n, PyObject *cache) +{ + PyObject *list; + int32_t *array = _array; + Py_ssize_t i, c; + + if (!cache) + return 0; + + list = PySequence_Fast(cache, "Expecting list or tuple of integers"); + if (!list) + return -1; + c = PySequence_Fast_GET_SIZE(list); + if (c > n) { + PyErr_Format(PyExc_ValueError, + "Expected a sequence of up to %zu size", n); + goto error; + } + + PyErr_Clear(); + for (i = 0; i < c; i++) { + PyObject *p = PySequence_Fast_GET_ITEM(list, i); + unsigned long val = PyLong_AsLong(p); + + if (PyErr_Occurred()) + goto error; + array[i] = val; + } + for (; i < n; i++) + array[i] = 0; + + Py_DECREF(list); + return 0; + +error: + Py_DECREF(list); + return -1; +} + PyObject *py_rxgen_get_structs(const void *data, size_t num, size_t size, PyObject **cache, PyObject *(*data_to_type)(const void *elem)) @@ -389,116 +611,422 @@ error: } /* - * RxRPC connection container. + * Assign to a python object's members from its new op's keyword list. */ -static int -py_rx_connection_init(PyObject *_self, PyObject *args, PyObject *kwds) +int py_rxgen_initialise_members(PyObject *obj, PyObject *kw) { - struct py_rx_connection *self = (struct py_rx_connection *)_self; - self->x = NULL; + const PyMemberDef *member; + PyTypeObject *type = (PyTypeObject *)PyObject_Type(obj); + PyObject *key, *value; + Py_ssize_t pos = 0; + + if (!kw || PyDict_Size(kw) <= 0) + return 0; + if (!PyArg_ValidateKeywordArguments(kw)) + return -1; + if (!type->tp_members || !type->tp_members[0].name) { + PyErr_Format(PyExc_AttributeError, + "Calls of %s take no parameters", type->tp_name); + return -1; + } + + while (PyDict_Next(kw, &pos, &key, &value)) { + for (member = type->tp_members; member->name; member++) { + void *p = (void *)obj + member->offset; + if (PyUnicode_CompareWithASCIIString(key, member->name) != 0) + continue; + + switch (member->type) { + case T_CHAR: + case T_BYTE: + *(uint8_t *)p = PyLong_AsLong(value); + goto found; + case T_SHORT: + *(uint16_t *)p = PyLong_AsLong(value); + goto found; + case T_INT: + *(uint32_t *)p = PyLong_AsLong(value); + goto found; + case T_LONGLONG: + *(uint64_t *)p = PyLong_AsLongLong(value); + goto found; + case T_UBYTE: + *(uint8_t *)p = PyLong_AsUnsignedLong(value); + goto found; + case T_USHORT: + *(uint16_t *)p = PyLong_AsUnsignedLong(value); + goto found; + case T_UINT: + *(uint32_t *)p = PyLong_AsUnsignedLong(value); + goto found; + case T_ULONGLONG: + *(uint64_t *)p = PyLong_AsUnsignedLongLong(value); + goto found; + case T_OBJECT_EX: + Py_INCREF(value); + *(PyObject **)p = value; + goto found; + default: + abort(); + } + } + + PyErr_Format(PyExc_AttributeError, + "Calls of %s don't have parameter %S", type->tp_name, key); + return -1; + found: + ; + } + return 0; } -static void -py_rx_connection_dealloc(struct py_rx_connection *self) +/* + * Decode a string of bytes into a preallocated unicode string python object. + */ +void py_dec_string(struct rx_call *call) { - if (self->x) { - rx_close_connection(self->x); - self->x = NULL; + PyObject *str = call->bulk_item; + struct rx_buf *cursor, *spent; + unsigned needed, count; + uint8_t *new_stop; + + if (PyUnicode_KIND(str) != PyUnicode_1BYTE_KIND) + abort(); + + rxrpc_post_dec(call); + + needed = call->bulk_count - call->bulk_index; + count = call->data_stop - call->data_cursor; + if (count > 0) { + if (count > needed) + count = needed; + memcpy(PyUnicode_DATA(str) + call->bulk_index, call->data_cursor, count); + call->bulk_index += count; + rxrpc_dec_align(call); + if (call->bulk_count <= call->bulk_index) + return; + + } + + /* More data may have come into a partially filled buffer from which we + * previously read. + */ + cursor = call->buffer_head; + count = cursor->io_cursor & ~3; + new_stop = cursor->buf + count; + if (call->data_stop >= new_stop) { + /* This buffer must then be completely used as we're required to check + * amount received before reading it. + */ + if (new_stop != cursor->buf + RXGEN_BUFFER_SIZE) + abort(); /* Didn't completely consume a buffer */ + if (call->buffer_tail == cursor) + abort(); /* Unexpectedly out of data */ + + /* Move to the next buffer */ + spent = cursor; + cursor = cursor->next; + call->buffer_head = cursor; + + call->data_cursor = call->data_start = cursor->buf; + count = cursor->io_cursor & ~3; + if (count == 0) + abort(); + new_stop = cursor->buf + count; + + free(spent->buf); + free(spent); } - Py_TYPE(self)->tp_free((PyObject *)self); + + call->data_stop = new_stop; } -PyTypeObject py_rx_connectionType = { - PyVarObject_HEAD_INIT(NULL, 0) - "kafs.rx_connection", /*tp_name*/ - sizeof(struct py_rx_connection), /*tp_basicsize*/ - 0, /*tp_itemsize*/ - (destructor)py_rx_connection_dealloc, /*tp_dealloc*/ - 0, /*tp_print*/ - 0, /*tp_getattr*/ - 0, /*tp_setattr*/ - 0, /*tp_compare*/ - 0, /*tp_repr*/ - 0, /*tp_as_number*/ - 0, /*tp_as_sequence*/ - 0, /*tp_as_mapping*/ - 0, /*tp_hash */ - 0, /*tp_call*/ - 0, /*tp_str*/ - 0, /*tp_getattro*/ - 0, /*tp_setattro*/ - 0, /*tp_as_buffer*/ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /*tp_flags*/ - "RxRPC connection container", /* tp_doc */ - 0, /* tp_traverse */ - 0, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - 0, /* tp_methods */ - 0, /* tp_members */ - 0, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - py_rx_connection_init, /* tp_init */ - 0, /* tp_alloc */ - 0, /* tp_new */ -}; +/* + * Recursively encode a standard C array. + */ +static int py_enc_c_array(struct rx_call *call, + const void *data, int dim, + const Py_ssize_t *dim_size, + Py_ssize_t itemsize) +{ + if (dim == 1) { + /* Data subarray */ + rxrpc_enc_bytes(call, data, *dim_size * itemsize); + rxrpc_enc_align(call); + return rxrpc_post_enc(call); + } else { + /* Pointer subarray */ + const void *const *ptrs = data; + Py_ssize_t i; + for (i = 0; i < *dim_size; i++) + if (py_enc_c_array(call, ptrs[i], dim - 1, dim_size + 1, itemsize) < 0) + return -1; + return 0; + } +} /* - * Set up an RxRPC connection. + * Recursively encode a NumPy-style array. */ -PyObject * -kafs_py_rx_new_connection(PyObject *_self, PyObject *args) +static int py_enc_numpy_array(struct rx_call *call, + const void *data, int dim, + const Py_ssize_t *dim_size, + const Py_ssize_t *dim_stride, + Py_ssize_t itemsize) { - struct py_rx_connection *obj; - struct rx_connection *z_conn; - union { - struct sockaddr sa; - struct sockaddr_in sin; - struct sockaddr_in6 sin6; - } sa; - const char *address = NULL; - socklen_t salen; - uint16_t port, service, local_port = 0, local_service = 0; - int exclusive = 0; - - if (!PyArg_ParseTuple(args, "sHH|HHp", - &address, &port, &service, - &local_port, &local_service, &exclusive)) - return NULL; + int i; + + if (dim == 0) + /* Single data item */ + return rxrpc_enc_bytes(call, data, itemsize); + + for (i = 0; i < *dim_size; i++) { + if (py_enc_numpy_array(call, data + i * *dim_stride, + dim - 1, dim_size + 1, dim_stride + 1, itemsize) < 0) + return -1; + if (dim == 1 && rxrpc_post_enc(call) < 0) + return -1; + } + return 0; +} + +/* + * Recursively encode a Python Imaging Library (PIL)-style array. + */ +static int py_enc_pil_array(struct rx_call *call, + const void *data, int dim, + const Py_ssize_t *dim_size, + const Py_ssize_t *dim_stride, + const Py_ssize_t *dim_suboffset, + Py_ssize_t itemsize) +{ + int i; + + if (dim == 0) + /* Single data item */ + return rxrpc_enc_bytes(call, data, itemsize); + + for (i = 0; i < *dim_size; i++) { + const void *ptr = data + i * *dim_stride; + if (*dim_suboffset >= 0) { + ptr = *((const void *const *)ptr); + ptr += *dim_suboffset; + } + if (py_enc_pil_array(call, ptr, + dim - 1, dim_size + 1, dim_stride + 1, dim_suboffset + 1, + itemsize) < 0) + return -1; + if (dim == 1 && rxrpc_post_enc(call) < 0) + return -1; + } + return 0; +} - memset(&sa, 0, sizeof(sa)); - if (inet_pton(AF_INET, address, &sa.sin.sin_addr)) { - sa.sin.sin_family = AF_INET; - sa.sin.sin_port = htons(port); - salen = sizeof(sa.sin); - } else if (inet_pton(AF_INET6, address, &sa.sin.sin_addr)) { - sa.sin6.sin6_family = AF_INET6; - sa.sin6.sin6_port = htons(port); - salen = sizeof(sa.sin6); +/* + * Encode the contents of a python buffer view. + */ +int py_enc_buffer(struct rx_call *call, Py_buffer *view) +{ + int i; + + if (call->error_code) + return -1; + + if (0) { + printf("PEBUF: l=%zu isz=%zd {", view->len, view->itemsize); + if (view->shape) + for (i = 0; i < view->ndim; i++) + printf(" %zd", view->shape[i]); + printf(" }\n"); + } + + rxrpc_enc(call, view->len); + + if (view->ndim == 0 || (view->ndim == 1 && !view->shape)) { + /* Simple scalar array */ + if (rxrpc_enc_bytes(call, view->buf, view->len) < 0 || + rxrpc_post_enc(call) < 0) + goto error; + goto done; + } + + if (!view->strides) { + /* Standard C array */ + if (py_enc_c_array(call, view->buf, view->ndim, view->shape, view->itemsize) < 0) + goto error; + goto done; + } + + if (!view->suboffsets) { + if (py_enc_numpy_array(call, view->buf, view->ndim, view->shape, view->strides, + view->itemsize) < 0) + goto error; + goto done; + } + + /* PIL-style Python array */ + if (py_enc_pil_array(call, view->buf, view->ndim, view->shape, view->strides, + view->suboffsets, view->itemsize) < 0) + goto error; + +done: + return rxrpc_post_enc(call); + +error: + return -1; +} + +/* + * Recursively decode a standard C array. + */ +static int py_dec_c_array(struct rx_call *call, + void *data, int dim, + const Py_ssize_t *dim_size, + Py_ssize_t itemsize) +{ + if (dim == 1) { + /* Data subarray */ + call->bulk_item = data; + call->bulk_count = *dim_size * itemsize; + call->bulk_index = 0; + rxrpc_dec_bytes(call); + return rxrpc_post_dec(call); } else { - return PyExc_TypeError; + /* Pointer subarray */ + void **ptrs = data; + Py_ssize_t i; + for (i = 0; i < *dim_size; i++) + if (py_dec_c_array(call, ptrs[i], dim - 1, dim_size + 1, itemsize) < 0) + return -1; + return 0; } +} - obj = (struct py_rx_connection *)_PyObject_New(&py_rx_connectionType); - if (!obj) - return PyExc_MemoryError; - py_rx_connection_init((PyObject *)obj, NULL, NULL); - assert(obj->x == NULL); - - z_conn = rx_new_connection(&sa.sa, salen, service, - local_port, local_service, exclusive); - if (!z_conn) { - Py_DECREF(obj); - return errno == ENOMEM ? PyExc_MemoryError : - PyErr_SetFromErrno(PyExc_IOError); - } - obj->x = z_conn; - return (PyObject *)obj; +/* + * Recursively decode a NumPy-style array. + */ +static int py_dec_numpy_array(struct rx_call *call, + void *data, int dim, + const Py_ssize_t *dim_size, + const Py_ssize_t *dim_stride, + Py_ssize_t itemsize) +{ + int i; + + if (dim == 0) { + /* Single data item */ + call->bulk_item = data; + call->bulk_count = itemsize; + call->bulk_index = 0; + rxrpc_dec_bytes(call); + return 0; + } + + for (i = 0; i < *dim_size; i++) { + if (py_dec_numpy_array(call, data + i * *dim_stride, + dim - 1, dim_size + 1, dim_stride + 1, itemsize) < 0) + return -1; + if (dim == 1 && rxrpc_post_dec(call) < 0) + return -1; + } + return 0; +} + +/* + * Recursively decode a Python Imaging Library (PIL)-style array. + */ +static int py_dec_pil_array(struct rx_call *call, + void *data, int dim, + const Py_ssize_t *dim_size, + const Py_ssize_t *dim_stride, + const Py_ssize_t *dim_suboffset, + Py_ssize_t itemsize) +{ + int i; + + if (dim == 0) { + /* Single data item */ + call->bulk_item = data; + call->bulk_count = itemsize; + call->bulk_index = 0; + rxrpc_dec_bytes(call); + return 0; + } + + for (i = 0; i < *dim_size; i++) { + void *ptr = data + i * *dim_stride; + if (*dim_suboffset >= 0) { + ptr = *((void *const *)ptr); + ptr += *dim_suboffset; + } + if (py_dec_pil_array(call, ptr, + dim - 1, dim_size + 1, dim_stride + 1, dim_suboffset + 1, + itemsize) < 0) + return -1; + if (dim == 1 && rxrpc_post_dec(call) < 0) + return -1; + } + return 0; +} + +/* + * Decode the contents of an opaque type + */ +int py_dec_opaque(struct rx_call *call, PyObject *obj) +{ + Py_buffer view; + int i; + + if (call->error_code) + return -1; + + if (PyObject_GetBuffer(obj, &view, PyBUF_FULL) < 0) + return -1; + + if (0) { + printf("PEBUF: l=%zu isz=%zd {", view.len, view.itemsize); + if (view.shape) + for (i = 0; i < view.ndim; i++) + printf(" %zd", view.shape[i]); + printf(" }\n"); + } + + if (view.ndim == 0 || (view.ndim == 1 && !view.shape)) { + /* Simple scalar array */ + call->bulk_item = view.buf; + call->bulk_count = view.len; + call->bulk_index = 0; + rxrpc_dec_bytes(call); + if (rxrpc_post_dec(call) < 0) + goto error; + goto done; + } + + if (!view.strides) { + /* Standard C array */ + if (py_dec_c_array(call, view.buf, view.ndim, view.shape, view.itemsize) < 0) + goto error; + goto done; + } + + if (!view.suboffsets) { + if (py_dec_numpy_array(call, view.buf, view.ndim, view.shape, view.strides, + view.itemsize) < 0) + goto error; + goto done; + } + + /* PIL-style Python array */ + if (py_dec_pil_array(call, view.buf, view.ndim, view.shape, view.strides, + view.suboffsets, view.itemsize) < 0) + goto error; + +done: + PyBuffer_Release(&view); + return call->error_code ? -1 : 0; + +error: + PyBuffer_Release(&view); + return -1; } diff --git a/py_rxgen.h b/py_rxgen.h index 6391379..f566b4b 100644 --- a/py_rxgen.h +++ b/py_rxgen.h @@ -17,10 +17,27 @@ struct py_rx_connection { struct rx_connection *x; }; +struct py_rx_call { + PyObject_HEAD + struct rx_call *x; + struct py_rx_request *req; + struct py_rx_response *resp; +}; + +struct py_rx_request { + PyObject_HEAD +}; + +struct py_rx_response { + PyObject_HEAD +}; + extern PyTypeObject py_rx_connectionType; extern PyObject *kafs_py_rx_new_connection(PyObject *, PyObject *); +extern int py_rxgen_initialise_members(PyObject *obj, PyObject *kw); + /* * Single embedded struct handling */ @@ -32,10 +49,13 @@ extern int py_rxgen_premarshal_struct(void *p, size_t size, size_t offs, int (*premarshal)(PyObject *object)); /* - * Embedded string (char array) handling + * String and opaque type handling */ extern PyObject *py_rxgen_get_string(const void *_p, size_t n); extern int py_rxgen_set_string(void *_p, size_t n, PyObject *val); +extern int py_enc_buffer(struct rx_call *call, Py_buffer *view); +extern void py_dec_string(struct rx_call *call); +extern int py_dec_opaque(struct rx_call *call, PyObject *obj); /* * Embedded general array handling @@ -45,6 +65,9 @@ extern int py_rxgen_set_array(size_t n, PyObject **cache, PyObject *val); /* * Embedded integer array handling */ +extern PyObject *py_rxgen_get_int8(const void *_array , size_t n, PyObject **cache); +extern PyObject *py_rxgen_get_int16(const void *_array, size_t n, PyObject **cache); +extern PyObject *py_rxgen_get_int32(const void *_array, size_t n, PyObject **cache); extern PyObject *py_rxgen_get_uint8(const void *_array , size_t n, PyObject **cache); extern PyObject *py_rxgen_get_uint16(const void *_array, size_t n, PyObject **cache); extern PyObject *py_rxgen_get_uint32(const void *_array, size_t n, PyObject **cache); @@ -52,6 +75,9 @@ extern PyObject *py_rxgen_get_uint32(const void *_array, size_t n, PyObject **ca extern int py_rxgen_premarshal_uint8(void *_array, size_t n, PyObject *cache); extern int py_rxgen_premarshal_uint16(void *_array, size_t n, PyObject *cache); extern int py_rxgen_premarshal_uint32(void *_array, size_t n, PyObject *cache); +extern int py_rxgen_premarshal_int8(void *_array, size_t n, PyObject *cache); +extern int py_rxgen_premarshal_int16(void *_array, size_t n, PyObject *cache); +extern int py_rxgen_premarshal_int32(void *_array, size_t n, PyObject *cache); /* * Embedded struct array handling diff --git a/rpc-api/bos.xg b/rpc-api/bos.xg index 2fb88c4..e7273b3 100644 --- a/rpc-api/bos.xg +++ b/rpc-api/bos.xg @@ -145,7 +145,7 @@ EnumerateInstance (IN uint32_t instance, GetInstanceInfo (IN string instance, OUT string type, - OUT struct bozo_status *status) = 85; + OUT bozo_status *status) = 85; GetInstanceParm (IN string instance, IN uint32_t num, @@ -160,11 +160,11 @@ ListSUsers (IN uint32_t an, ListKeys (IN uint32_t an, OUT uint32_t *kvno, - OUT struct bozo_key *key, - OUT struct bozo_keyInfo *keinfo) = 90; + OUT bozo_key *key, + OUT bozo_keyInfo *keinfo) = 90; AddKey (IN uint32_t an, - IN struct bozo_key *key) = 91; + IN bozo_key *key) = 91; DeleteKey (IN uint32_t an) = 92; @@ -212,12 +212,12 @@ Exec (IN string cmd) = 108; Prune (IN uint32_t flags) = 109; SetRestartTime (IN uint32_t type, - IN struct bozo_netKTime *restartTime) = 110; + IN bozo_netKTime *restartTime) = 110; GetRestartTime (IN uint32_t type, - OUT struct bozo_netKTime *restartTime) = 111; + OUT bozo_netKTime *restartTime) = 111; GetLog(IN string name) split = 112; diff --git a/rpc-api/cb.xg b/rpc-api/cb.xg index 8402462..90882a0 100644 --- a/rpc-api/cb.xg +++ b/rpc-api/cb.xg @@ -84,16 +84,18 @@ const AFS_MAX_INTERFACE_ADDR = 32; struct interfaceAddr { /* for multihomed clients */ int32_t numberOfInterfaces; afsUUID uuid; - ASIS int32_t addr_in[AFS_MAX_INTERFACE_ADDR]; /* interface addresses */ - ASIS int32_t subnetmask[AFS_MAX_INTERFACE_ADDR]; /* subnet masks in net ord */ + uint32_t addr_in[AFS_MAX_INTERFACE_ADDR]; /* interface addresses */ + uint32_t subnetmask[AFS_MAX_INTERFACE_ADDR]; /* subnet masks in net ord */ int32_t mtu[AFS_MAX_INTERFACE_ADDR]; /* MTU */ }; const AFSMAXCELLHOSTS = 8; /*Max VLDB servers per cell*/ -typedef int32_t serverList[AFSMAXCELLHOSTS]; +struct serverList { + int32_t server[AFSMAXCELLHOSTS]; +}; -typedef afs_uint32 cacheConfig<>; +typedef uint32_t cacheConfig<>; CallBack (IN AFSCBFids *a_fidArrayP, @@ -141,14 +143,14 @@ GetCellServDB(IN uint32_t cellIndex, GetLocalCell(OUT string cellName) = 217; -GetCacheConfig(IN afs_uint32 callerVersion, - OUT afs_uint32 *serverVersion, - OUT afs_uint32 *configCount, +GetCacheConfig(IN uint32_t callerVersion, + OUT uint32_t *serverVersion, + OUT uint32_t *configCount, OUT cacheConfig *config) = 218; GetCellByNum(IN int32_t cellNumber, OUT string cellName, OUT serverList *cellHosts) = 65537; -TellMeAboutYourself(OUT struct interfaceAddr *addr, +TellMeAboutYourself(OUT interfaceAddr *addr, OUT Capabilities *capabilities) = 65538; diff --git a/rpc-api/fs.xg b/rpc-api/fs.xg index c297163..ea7dfd2 100644 --- a/rpc-api/fs.xg +++ b/rpc-api/fs.xg @@ -37,7 +37,7 @@ package RXAFS_ -#define AFSUUID_GENERATE +//#define AFSUUID_GENERATE #include "common.h" diff --git a/rpc-api/ka.xg b/rpc-api/ka.xg index bcf982c..b7c31d4 100644 --- a/rpc-api/ka.xg +++ b/rpc-api/ka.xg @@ -123,14 +123,7 @@ const KALOCKED = 180522; -struct ka_CBS { - opaque Seq<>; -}; - -struct ka_BBS { - int32_t MaxSeqLen; - opaque Seq<>; -}; +typedef opaque ka_CBS_Seq<>; struct EncryptionKey { uint8_t key[8]; @@ -171,27 +164,29 @@ struct kaentryinfo { package KAA_ -Authenticate (IN kaname name, - IN kaname instance, +Authenticate (IN kaname *name, + IN kaname *instance, IN Date start_time, IN Date end_time, - IN ka_CBS *request, - INOUT ka_BBS *answer) = 21; + IN ka_CBS_Seq *request, + INOUT int32_t *answer_BBS_MaxSeqLen, + INOUT opaque *answer_BBS_Seq<>) = 21; -ChangePassword (IN kaname name, - IN kaname instance, - IN ka_CBS arequest, - INOUT ka_BBS *oanswer) = 2; +ChangePassword (IN kaname *name, + IN kaname *instance, + IN ka_CBS_Seq *arequest, + INOUT int32_t *oanswer_BBS_MaxSeqLen, + INOUT opaque *oanswer_BBS_Seq<>) = 2; package KAM_ -SetPassword (IN kaname name, - IN kaname instance, +SetPassword (IN kaname *name, + IN kaname *instance, IN int32_t kvno, - IN EncryptionKey password) = 4; + IN EncryptionKey *password) = 4; -SetFields (IN kaname name, - IN kaname instance, +SetFields (IN kaname *name, + IN kaname *instance, IN int32_t flags, IN Date user_expiration, IN int32_t max_ticket_lifetime, @@ -199,12 +194,12 @@ SetFields (IN kaname name, IN int32_t spare1, IN int32_t spare2) = 5; -CreateUser (IN kaname name, - IN kaname instance, - IN EncryptionKey password) = 6; +CreateUser (IN kaname *name, + IN kaname *instance, + IN EncryptionKey *password) = 6; -GetEntry (IN kaname name, - IN kaname instance, +GetEntry (IN kaname *name, + IN kaname *instance, IN uint32_t major_version, OUT kaentryinfo *entry) = 8; @@ -212,17 +207,19 @@ GetEntry (IN kaname name, package KAT_ GetTicket_old (IN int32_t kvno, - IN kaname auth_domain, - IN struct ka_CBS *aticket, - IN kaname name, - IN kaname instance, - IN struct ka_CBS *atimes, - INOUT struct ka_BBS *oanswer) = 3; + IN kaname *auth_domain, + IN ka_CBS_Seq *aticket, + IN kaname *name, + IN kaname *instance, + IN ka_CBS_Seq *atimes, + INOUT int32_t *oanswer_BBS_MaxSeqLen, + INOUT opaque *oanswer_BBS_Seq<>) = 3; GetTicket (IN int32_t kvno, - IN kaname auth_domain, - IN struct ka_CBS *aticket, - IN kaname name, - IN kaname instance, - IN struct ka_CBS *atimes, - INOUT struct ka_BBS *oanswer) = 23; + IN kaname *auth_domain, + IN ka_CBS_Seq *aticket, + IN kaname *name, + IN kaname *instance, + IN ka_CBS_Seq *atimes, + INOUT int32_t *oanswer_BBS_MaxSeqLen, + INOUT opaque *oanswer_BBS_Seq<>) = 23; diff --git a/rpc-api/pts.xg b/rpc-api/pts.xg index cd6c5d2..84059e5 100644 --- a/rpc-api/pts.xg +++ b/rpc-api/pts.xg @@ -68,7 +68,6 @@ const PRSIZE = 10; const COSIZE = 39; const PRSRV = 73; const ENTRYSIZE = 192; -const HASHSIZE = 8191; const PRDBVERSION = 0; @@ -171,7 +170,9 @@ struct prcheckentry { }; -typedef char prname[PR_MAXNAMELEN]; +struct prname { + char prname[PR_MAXNAMELEN]; +}; typedef prname namelist; @@ -215,10 +216,10 @@ INewEntry(IN string name, IN int32_t oid) = 500; ListEntry(IN int32_t id, - OUT struct prcheckentry *entry) = 512; + OUT prcheckentry *entry) = 512; DumpEntry(IN uint32_t pos, - OUT struct prdebugentry *entry) = 502; + OUT prdebugentry *entry) = 502; ChangeEntry(IN int32_t id, IN string name, diff --git a/rpc-api/vldb.xg b/rpc-api/vldb.xg index 326e209..cc8be08 100644 --- a/rpc-api/vldb.xg +++ b/rpc-api/vldb.xg @@ -52,7 +52,6 @@ const NMAXNSERVERS = 13; const MAX_NUMBER_OPCODES = 30; const MAXTYPES = 3; const MAXSERVERID = 30; -const HASHSIZE = 8191; const DEFAULTBULK = 10000; typedef opaque bulk; diff --git a/rpc-api/volumeserver.xg b/rpc-api/volumeserver.xg index 51cddef..28f7ea9 100644 --- a/rpc-api/volumeserver.xg +++ b/rpc-api/volumeserver.xg @@ -271,12 +271,12 @@ struct replica { struct destServer destserver; }; -#define AFS_MAX_DESTINATIONS 255 +const AFS_MAX_DESTINATIONS = 255; typedef replica manyDests; AFSVolCreateVolume(IN uint32_t partition, - IN string name, + IN string name<>, IN uint32_t type, IN uint32_t parent, INOUT uint32_t *volid, @@ -301,14 +301,14 @@ AFSVolRestore(IN uint32_t toTrans, AFSVolForward(IN int32_t fromTrans, IN int32_t fromData, - IN struct destServer *destination, + IN destServer *destination, IN uint32_t destTrans, - IN struct restoreCookie *cookie) = VOLFORWARD; + IN restoreCookie *cookie) = VOLFORWARD; AFSVolClone(IN uint32_t trans, IN uint32_t purgeVol, IN uint32_t newType, - IN string newName, + IN string newName<>, INOUT uint32_t *newVol) = VOLCLONE; AFSVolReClone(IN uint32_t tid, @@ -347,18 +347,18 @@ AFSVolSetIdsTypes(IN uint32_t tId, AFSVolSetDate(IN uint32_t tid, IN uint32_t newDate) = VOLSETDATE; -AFSVolListPartitions(OUT struct pIDs *partIDs) = VOLLISTPARTITIONS; +AFSVolListPartitions(OUT pIDs *partIDs) = VOLLISTPARTITIONS; AFSVolPartitionInfo(IN string name<>, - OUT struct diskPartition *partition) = VOLPARTITIONINFO; + OUT diskPartition *partition) = VOLPARTITIONINFO; AFSVolListVolumes(IN uint32_t partID, IN uint32_t flags, - OUT struct volEntries *resultEntries) = VOLLISTVOLUMES; + OUT volEntries *resultEntries) = VOLLISTVOLUMES; AFSVolListOneVolume(IN uint32_t partID, IN uint32_t volid, - OUT struct volEntries *resultEntries) = VOLLISTONEVOLUME; + OUT volEntries *resultEntries) = VOLLISTONEVOLUME; AFSVolGetNthVolume(IN uint32_t index, OUT uint32_t *volume, @@ -368,11 +368,11 @@ AFSVolMonitor(OUT transDebugEntries *result) = VOLMONITOR; AFSVolXListVolumes(IN uint32_t partID, IN uint32_t flags, - OUT struct xvolEntries *resultEntries) = VOLXLISTVOLUMES; + OUT xvolEntries *resultEntries) = VOLXLISTVOLUMES; AFSVolXListOneVolume(IN uint32_t partID, IN uint32_t volid, - OUT struct xvolEntries *resultEntries) = VOLXLISTONEVOL; + OUT xvolEntries *resultEntries) = VOLXLISTONEVOL; AFSVolSetInfo(IN uint32_t transid, IN volintInfo *volinfo) = VOLSETINFO; @@ -383,5 +383,5 @@ AFSVolForwardMultiple(IN int32_t fromTrans, IN int32_t fromData, IN manyDests *destinations, IN uint32_t spare0, - IN struct restoreCookie *cookie, + IN restoreCookie *cookie, OUT multi_results *results) = VOLFORWARDMULTIPLE; diff --git a/rxgen.h b/rxgen.h index 4aa8f50..a6fbdbd 100644 --- a/rxgen.h +++ b/rxgen.h @@ -13,7 +13,8 @@ #define _RXGEN_H #include "af_rxrpc.h" -#include "circ_buf.h" +#include +#include typedef uint32_t net_xdr_t; @@ -23,28 +24,147 @@ struct rx_connection { int fd; }; +#define RXGEN_BUFFER_SIZE 1024 + +struct rx_buf { + uint32_t magic; + unsigned short io_cursor; + //unsigned short enc_cursor; + uint8_t *buf; + struct rx_buf *next; +}; + +enum rx_call_state { + rx_call_cl_not_started, + rx_call_cl_encoding_params, + rx_call_cl_waiting_for_response, + rx_call_cl_decoding_response, + rx_call_cl_wait_for_no_MSG_MORE, + rx_call_cl_complete, + rx_call_sv_not_started, + rx_call_sv_waiting_for_opcode, + rx_call_sv_decoding_opcode, + rx_call_sv_decoding_params, + rx_call_sv_wait_for_no_MSG_MORE, + rx_call_sv_processing, + rx_call_sv_encoding_response, + rx_call_sv_response_encoded, + rx_call_sv_waiting_for_final_ack, + rx_call_sv_complete, + rx_call_remotely_aborted, + rx_call_locally_aborted, + rx_call_net_error, + rx_call_local_error, + rx_call_rejected_busy, +}; + struct rx_call { - int got_eor; + uint32_t magic; + struct rx_connection *conn; + enum rx_call_state state; + unsigned known_to_kernel : 1; + unsigned secured : 1; + unsigned more : 1; int error_code; uint32_t abort_code; - int phase; unsigned need_size; - unsigned need_bulk_count; - unsigned item_per_buf; - unsigned bulk_index; - - /* Circular buffer holding received data (power-of-two size) */ - unsigned head; - unsigned tail; - unsigned size; - net_xdr_t reply[]; + + /* Service routines */ + void (*processor)(struct rx_call *call); + void (*failed)(struct rx_call *call); + + /* String of buffers holding data */ + uint8_t *data_start; + uint8_t *data_cursor; + uint8_t *data_stop; + struct rx_buf *buffer_head; + struct rx_buf *buffer_tail; + unsigned data_count; + unsigned buffer_space; + + /* Decoding support */ + unsigned phase; /* Encode/decode phase */ + unsigned bulk_count; /* Number of items in bulk array */ + unsigned bulk_index; /* Index of item being processed */ + union { + void *bulk_item; /* Pointer to string/bytes/struct being processed */ + uint32_t bulk_u32; /* 8/16/32-bit integer being processed */ + uint64_t bulk_u64; /* 64-bit unsigned integer being processed */ + int64_t bulk_s64; /* 64-bit signed integer being processed */ + }; + int (*decoder)(struct rx_call *call); + void *decoder_private; }; -static inline size_t rxgen_call_buffer_space_to_end(struct rx_call *call) + +extern int rxrpc_enc_bytes(struct rx_call *call, const void *data, size_t size); +extern void rxrpc_enc_slow(struct rx_call *call, net_xdr_t data); +static inline void rxrpc_enc(struct rx_call *call, uint32_t data) +{ + uint32_t xdr_data = htonl(data); + if (__builtin_expect(call->data_cursor < call->data_stop, 1)) { + *(net_xdr_t *)call->data_cursor = xdr_data; + call->data_cursor += 4; + } else { + rxrpc_enc_slow(call, xdr_data); + } +} + +static inline void rxrpc_enc_align(struct rx_call *call) { - return CIRC_SPACE_TO_END(call->head, call->tail, call->size); + while ((unsigned long)call->data_cursor & 3) + *(call->data_cursor++) = 0; } +static inline int rxrpc_post_enc(struct rx_call *call) +{ + if (__builtin_expect(!call->error_code, 1)) { + size_t n = call->data_cursor - call->data_start; + call->data_start = call->data_cursor; + call->buffer_space -= n; + call->data_count += n; + return 0; + } else { + errno = call->error_code; + return -1; + } +} + +extern void rxrpc_dec_bytes(struct rx_call *call); +extern uint32_t rxrpc_dec_slow(struct rx_call *call); +static inline uint32_t rxrpc_dec(struct rx_call *call) +{ + if (__builtin_expect(call->data_cursor < call->data_stop, 1)) { + net_xdr_t x = *(net_xdr_t *)call->data_cursor; + call->data_cursor += sizeof(x); + return ntohl(x); + } else { + return rxrpc_dec_slow(call); + } +} + +static inline void rxrpc_dec_align(struct rx_call *call) +{ + unsigned long cursor = (unsigned long)call->data_cursor; + cursor = (cursor + 3) & ~3UL; + call->data_cursor = (uint8_t*)cursor; +} + +static inline int rxrpc_post_dec(struct rx_call *call) +{ + if (__builtin_expect(!call->error_code, 1)) { + size_t n = call->data_cursor - call->data_start; + call->data_start = call->data_cursor; + call->data_count -= n; + return 0; + } else { + errno = call->error_code; + return -1; + } +} + +extern int rxgen_dec_discard_excess(struct rx_call *call); + extern struct rx_connection *rx_new_connection(const struct sockaddr *sa, socklen_t salen, uint16_t service, @@ -54,12 +174,11 @@ extern struct rx_connection *rx_new_connection(const struct sockaddr *sa, extern void rx_close_connection(struct rx_connection *z_conn); -extern int rxrpc_send_request(struct rx_connection *z_conn, - struct rx_call *call, - struct iovec *request, - int request_ioc); +extern struct rx_call *rxrpc_alloc_call(struct rx_connection *z_conn, int incoming_call); +extern void rxrpc_abort_call(struct rx_call *call, uint32_t abort_code); +extern void rxrpc_terminate_call(struct rx_call *call, uint32_t abort_code); -extern int rxrpc_wait_for_sync_reply(struct rx_connection *z_conn, - struct rx_call *call); +extern int rxrpc_send_data(struct rx_call *call); +extern int rxrpc_run_sync_call(struct rx_call *call); #endif /* _RXGEN_H */ diff --git a/rxgen/emit_c_struct.pm b/rxgen/emit_c_struct.pm index 75bab2b..f5dbb44 100644 --- a/rxgen/emit_c_struct.pm +++ b/rxgen/emit_c_struct.pm @@ -27,12 +27,6 @@ sub emit_struct_encdec_decls ($) { sub emit_struct_encdec ($) { my ($struct) = @_; - # Dump the banner comment block - print RXHDR "\n"; - print RXHDR @{$struct->{banner}}; - print RXOUT "\n"; - print RXOUT @{$struct->{banner}}; - # Write out a C structure definition for this type print RXHDR "struct ", $struct->{type}, " {\n"; foreach my $m (@{$struct->{members}}) { @@ -54,8 +48,9 @@ sub emit_struct_encdec ($) { print RXHDR "};\n"; # Write an encoding function - print RXOUT "static __attribute__((unused))\n"; - print RXOUT "net_xdr_t *rxgen_encode_", $struct->{type}, "(net_xdr_t *xdr, const struct ", $struct->{type}, " *p)\n"; + print RXHDR "extern void rxgen_encode_", $struct->{type}, "(struct rx_call *call, const struct ", $struct->{type}, " *p);\n"; + + print RXOUT "void rxgen_encode_", $struct->{type}, "(struct rx_call *call, const struct ", $struct->{type}, " *p)\n"; print RXOUT "{\n"; foreach my $m (@{$struct->{members}}) { @@ -68,19 +63,19 @@ sub emit_struct_encdec ($) { foreach my $m (@{$struct->{members}}) { if ($m->{class} eq "basic") { if ($m->{type} !~ /64/) { - print RXOUT "\t*xdr++ = htonl(p->", $m->{name}, ");\n"; + print RXOUT "\trxrpc_enc(call, p->", $m->{name}, ");\n"; } else { die $m->{where}, ": No encoding for type '", $m->{type}, "'"; } } elsif ($m->{class} eq "struct") { - print RXOUT "\txdr = rxgen_encode_", $m->{type}, "(xdr, &p->", $m->{name}, ");\n"; + print RXOUT "\trxgen_encode_", $m->{type}, "(call, &p->", $m->{name}, ");\n"; } elsif ($m->{class} eq "array") { print RXOUT "\tfor (i = 0; i < ", $m->{dim}, "; i++)\n"; if ($m->{elem}->{class} eq "basic" && $m->{elem}->{type} !~ /64/) { - print RXOUT "\t\t*xdr++ = htonl(p->", $m->{name}, "[i]);\n"; + print RXOUT "\t\trxrpc_enc(call, p->", $m->{name}, "[i]);\n"; } elsif ($m->{elem}->{class} eq "struct") { - print RXOUT "\t\txdr = rxgen_encode_", $m->{elem}->{type}, - "(xdr, &p->", $m->{name}, "[i]);\n"; + print RXOUT "\t\trxgen_encode_", $m->{elem}->{type}, + "(call, &p->", $m->{name}, "[i]);\n"; } else { die $m->{where}, ": No encoding for array type '", $m->{elem}->{type}, "'"; } @@ -89,13 +84,13 @@ sub emit_struct_encdec ($) { } } - print RXOUT "\treturn xdr;\n"; print RXOUT "}\n"; print RXOUT "\n"; # Write a decoding function - print RXOUT "static __attribute__((unused))\n"; - print RXOUT "unsigned rxgen_decode_", $struct->{type}, "(struct ", $struct->{type}, " *p, const net_xdr_t *xdr, unsigned tail, unsigned mask)\n"; + print RXHDR "extern void rxgen_decode_", $struct->{type}, "(struct rx_call *call, struct ", $struct->{type}, " *p);\n"; + + print RXOUT "void rxgen_decode_", $struct->{type}, "(struct rx_call *call, struct ", $struct->{type}, " *p)\n"; print RXOUT "{\n"; foreach my $m (@{$struct->{members}}) { @@ -108,18 +103,18 @@ sub emit_struct_encdec ($) { foreach my $m (@{$struct->{members}}) { if ($m->{class} eq "basic") { if ($m->{type} !~ /64/) { - print RXOUT "\tp->", $m->{name}, " = ntohl(xdr[tail++ & mask]);\n"; + print RXOUT "\tp->", $m->{name}, " = rxrpc_dec(call);\n"; } else { die $m->{where}, "No decoding for type '$type'"; } } elsif ($m->{class} eq "struct") { - print RXOUT "\ttail = rxgen_decode_", $m->{type}, "(&p->", $m->{name}, ", xdr, tail, mask);\n"; + print RXOUT "\trxgen_decode_", $m->{type}, "(call, &p->", $m->{name}, ");\n"; } elsif ($m->{class} eq "array") { print RXOUT "\tfor (i = 0; i < ", $m->{dim}, "; i++)\n"; if ($m->{elem}->{class} eq "basic" && $m->{elem}->{type} !~ /64/) { - print RXOUT "\t\tp->", $m->{name}, "[i] = ntohl(xdr[tail++ & mask]);\n"; + print RXOUT "\t\tp->", $m->{name}, "[i] = rxrpc_dec(call);\n"; } elsif ($m->{elem}->{class} eq "struct") { - print RXOUT "\t\ttail = rxgen_decode_", $m->{elem}->{type}, "(&p->", $m->{name}, "[i], xdr, tail, mask);\n"; + print RXOUT "\t\trxgen_decode_", $m->{elem}->{type}, "(call, &p->", $m->{name}, "[i]);\n"; } else { die $m->{where}, "No decoding for array type '$type'"; } @@ -128,7 +123,6 @@ sub emit_struct_encdec ($) { } } - print RXOUT "\treturn tail;\n"; print RXOUT "}\n"; } diff --git a/rxgen/emit_c_sync_funcs.pm b/rxgen/emit_c_sync_funcs.pm index ff6068d..bf6deac 100644 --- a/rxgen/emit_c_sync_funcs.pm +++ b/rxgen/emit_c_sync_funcs.pm @@ -13,286 +13,369 @@ # Calculate the C function prototypes # ############################################################################### -sub calc_func_prototype($) +sub emit_func_prototype($) { my ($func) = @_; # Function prototype lists (we add commas and the closing bracket later) my @protos = ( "int " . $func->{name} . "(\n" ); - my @send_protos = ( "int rxgen_send_request_" . $func->{name} . "(\n" ); - my @recv_protos = ( "int rxgen_decode_reply_" . $func->{name} . "(\n" ); - push @protos, "\tstruct rx_connection *z_conn"; - push @send_protos, "\tstruct rx_connection *z_conn"; - push @send_protos, "\tstruct rx_call *call"; - push @recv_protos, "\tstruct rx_connection *z_conn"; - push @recv_protos, "\tstruct rx_call *call"; + my @send_request_protos = (); + my @send_response_protos = (); + my @recv_request_protos = (); + my @recv_response_protos = (); # Arguments to pass when sending a call or processing a reply my @send_args = (); my @recv_args = (); foreach my $p (@{$func->{params}}) { - my @lines = (); - my @args = (); + my @enclines = (); + my @declines = (); - $proto = "\t /*" . $p->{dir} . "*/ "; if ($p->{class} eq "array") { die $p->{where}, ": Array arg not supported"; } elsif ($p->{class} eq "bulk" && !($p->{elem}->{class} eq "string" || $p->{elem}->{class} eq "opaque") ) { - if ($p->{dir} eq "OUT") { - if ($p->{elem}->{class} eq "struct") { - $proto .= "int (*alloc__" . $p->{name} . ")(void *token, int index, "; - $proto .= "struct " . $p->{type} . " **p_object)"; - push @args, "alloc__" . $p->{name}; - } else { - $proto .= "int (*store__" . $p->{name} . ")(void *token, int index, "; - $proto .= $p->{type} . " *value)"; - push @args, "store__" . $p->{name}; - } - push @lines, $proto; - $proto = "\t\t void *token__" . $p->{name}; - push @args, "token__" . $p->{name}; + # Encode + if ($p->{elem}->{class} eq "struct") { + $proto = "int (*get__" . $p->{name} . ")(struct rx_call *call, void *token)"; } else { - if ($p->{elem}->{class} eq "struct") { - $proto .= "int (*get__" . $p->{name} . ")(void *token, int index, "; - $proto .= "const struct " . $p->{type} . " **object)"; - } else { - $proto .= "int (*get__" . $p->{name} . ")(void *token, int index, "; - $proto .= $p->{type} . " *object)"; - } - push @lines, $proto; - $proto = "\t\tvoid *token__" . $p->{name}; - push @lines, $proto; - $proto = "\t\tsize_t nr__" . $p->{name}; - push @args, "get__" . $p->{name}, "token__" . $p->{name}; - push @args, "nr__" . $p->{name}; + $proto = "int (*get__" . $p->{name} . ")(struct rx_call *call, void *token)"; } - } else { - if ($p->{class} eq "bulk" && $p->{elem}->{class} ne "string") { - $proto .= "size_t nr_" . $p->{name} . ", "; - push @args, "nr__" . $p->{name}; + push @enclines, $proto; + push @enclines, "void *token__" . $p->{name}; + push @enclines, "size_t nr__" . $p->{name}; + push @args, "get__" . $p->{name}; + push @args, "token__" . $p->{name}; + push @args, "nr__" . $p->{name}; + + # Decode + if ($p->{elem}->{class} eq "struct") { + $proto = "int (*alloc__" . $p->{name} . ")(struct rx_call *call, void **token)"; + push @args, "alloc__" . $p->{name}; + } else { + $proto = "int (*store__" . $p->{name} . ")(struct rx_call *call, void **token)"; + push @args, "store__" . $p->{name}; } - $proto .= "const " if ($p->{dir} eq "IN" && $p->{class} ne "basic"); + push @declines, $proto; + push @declines, "void *token__" . $p->{name}; + push @declines, "size_t nr__" . $p->{name}; + push @args, "token__" . $p->{name}; + } elsif ($p->{class} eq "bulk" && ($p->{elem}->{class} eq "string" || + $p->{elem}->{class} eq "opaque")) { + $proto = $p->{type} . " " . $p->{ptr} . $p->{name}; + push @enclines, "size_t nr__" . $p->{name}; + push @enclines, "const " . $proto; + + push @declines, "size_t nr__" . $p->{name}; + push @declines, "void *token__" . $p->{name}; + push @declines, "int (*alloc__" . $p->{name} . ")(struct rx_call *call, void **token)"; + push @args, "nr__" . $p->{name}; + push @args, $p->{name}; + push @args, "alloc__" . $p->{name}; + } else { + my $enc_const = ""; + $enc_const = "const " if ($p->{class} ne "basic"); + $proto = ""; $proto .= "struct " if ($p->{class} eq "struct"); - $proto .= $p->{type} . " " . $p->{ptr} . $p->{name}; + $proto .= $p->{type} . " "; + $proto .= $p->{ptr} if ($p->{class} ne "basic"); + $proto .= $p->{name}; + push @enclines, $enc_const . $proto; + push @declines, $proto; push @args, $p->{name}; } - push @lines, $proto; - push @protos, @lines; - push @send_protos, @lines unless ($p->{dir} eq "OUT"); - push @recv_protos, @lines unless ($p->{dir} eq "IN"); + push @send_request_protos, @enclines unless ($p->{dir} eq "OUT"); + push @recv_request_protos, @declines unless ($p->{dir} eq "OUT"); + push @send_response_protos, @enclines unless ($p->{dir} eq "IN"); + push @recv_response_protos, @declines unless ($p->{dir} eq "IN"); push @send_args, @args unless ($p->{dir} eq "OUT"); push @recv_args, @args unless ($p->{dir} eq "IN"); } - # Terminate each line with a comma, excepting the last, which we terminate - # with a closing bracket. - for (my $i = 1; $i < $#protos; $i++) { - $protos[$i] .= ",\n"; + print RXHDR "\n"; + print RXHDR "/*\n"; + print RXHDR " * ", $func->{name}, "\n"; + print RXHDR " */\n"; + + if (@recv_request_protos) { + print RXHDR "struct ", $func->{name}, "_request {\n"; + foreach my $p (@recv_request_protos) { + print RXHDR "\t$p;\n"; + } + print RXHDR "};\n"; } - $protos[$#protos] .= ")"; - for (my $i = 1; $i < $#send_protos; $i++) { - $send_protos[$i] .= ",\n"; + print RXHDR "\n"; + if (@recv_response_protos) { + print RXHDR "struct ", $func->{name}, "_response {\n"; + foreach my $p (@recv_response_protos) { + print RXHDR "\t$p;\n"; + } + print RXHDR "};\n"; } - $send_protos[$#send_protos] .= ")"; - for (my $i = 1; $i < $#recv_protos; $i++) { - $recv_protos[$i] .= ",\n"; - } - $recv_protos[$#recv_protos] .= ")"; + # # Terminate each line with a comma, excepting the last, which we terminate + # # with a closing bracket. + # for (my $i = 1; $i < $#protos; $i++) { + # $protos[$i] .= ",\n"; + # } + # $protos[$#protos] .= ")"; + + # for (my $i = 1; $i < $#send_protos; $i++) { + # $send_protos[$i] .= ",\n"; + # } + # $send_protos[$#send_protos] .= ")"; + + # for (my $i = 1; $i < $#recv_protos; $i++) { + # $recv_protos[$i] .= ",\n"; + # } + # $recv_protos[$#recv_protos] .= ")"; $func->{protos} = \@protos; - $func->{send_protos} = \@send_protos; - $func->{recv_protos} = \@recv_protos; + $func->{send_request_protos} = \@send_request_protos; + $func->{recv_request_protos} = \@recv_request_protos; + $func->{send_response_protos} = \@send_response_protos; + $func->{recv_response_protos} = \@recv_response_protos; $func->{send_args} = \@send_args; $func->{recv_args} = \@recv_args; } ############################################################################### # -# Emit a function to encode a request +# Emit a function to encode a block in a way that can be used from asynchronous +# code. # ############################################################################### -sub emit_func_send_request($) +sub emit_func_encode($$$$) { - my ($func) = @_; + my ($func, $side, $subname, $paramlist) = @_; + my @params = @{$paramlist}; + my $ptr; + + if ($side eq "client") { + my %op_id = ( + class => "opcode", + type => "uint32_t", + name => $func->{opcode}, + where => $func->{where}, + xdr_size => 4, + ); + unshift @params, \%op_id; + $ptr = "call->req."; + } else { + $ptr = "call->resp."; + } - # Count up how many chunks we're going to need to send (iovec components) - # and how much XDR buffer size we're going to need. - my $xdr_bufsize = 0; - my $need_tmp = 0; - my @iovs = (); - my $iov = [ "buf + 0" ]; - my $iov_size = 4; - foreach my $p (@{$func->{request}}) { - $iov = [ "buf + " . $xdr_bufsize / 4 ] if (!$iov); - - $p->{enc_ioc} = $#iovs + 1; - - if ($p->{class} eq "bulk") { - $iov_size += 4; # Element count - $xdr_bufsize += $iov_size; - push @{$iov}, "$iov_size"; - $iov_size = 0; - push @iovs, $iov; - $iov = 0; - - push @iovs, [ ($p->{elem}->{class} eq "string" || - $p->{elem}->{class} eq "opaque") ? "(void *)" . $p->{name} : "NULL", - "nr__" . $p->{name} . " * " . $p->{xdr_size} ]; - if ($p->{elem}->{xdr_size} & 3 != 0) { - # Allocate space to use as padding - $need_tmp = 1; - $xdr_bufsize += 4; - $iov = [ "buf + " . $xdr_bufsize / 4 ]; - $iov_size = 0; - } - } else { - $iov_size += $p->{xdr_size}; + # We marshal the data in a number of phases. Each phase marshals a chunk + # of data of a certain size. A phase's size might be dependent on a + # variable in the previous phase. Variable-sized bulk arrays are split + # across multiple phases, with the length being at the end of the first + # phase and the data in the second. + my @phases = (); + my $phase = 0; + my $have_bulk = 0; + + foreach my $p (@params) { + unless ($phase) { + $phase = { type => "flat", size => 0, params => [] }; + push @phases, $phase; } - } - if ($iov) { - $xdr_bufsize += $iov_size; - push @{$iov}, "$iov_size"; - push @iovs, $iov; - } + if ($p->{class} eq "opcode" || + $p->{class} eq "basic" || + $p->{class} eq "struct" + ) { + $phase->{size} += $p->{xdr_size}; + push @{$phase->{params}}, $p; + } elsif ($p->{class} eq "bulk") { + $have_bulk = 1 if ($have_bulk == 0); + if ($p->{elem}->{class} eq "string" || + $p->{elem}->{class} eq "opaque") { + $have_bulk = 2; + } - die if ($#iovs < 0); + # Bulk objects begin with an element count + $phase->{elem_count} = $phase->{size}; + $phase->{size} += 4; - # Function definition and arguments - foreach $proto (@{$func->{send_protos}}) { - print RXOUT $proto; + my %pseudoparam = ( + class => "basic", + type => "bulk_size", + name => $p->{name}, + elem => $p->{elem}, + where => $p->{where}, + xdr_size => $p->{xdr_size}, + ); + push @{$phase->{params}}, \%pseudoparam; + + # Create a new phase + $phase = { + type => "bulk", + name => $p->{name}, + elem => $p->{elem}, + params => [ $p ], + xdr_size => $p->{xdr_size}, + size => $p->{xdr_size}, + }; + push @phases, $phase; + + if ($p->{elem}->{class} eq "string" || + $p->{elem}->{class} eq "opaque") { + $phase->{size} = 4; + } else { + # We don't want to be sending one object at a time if they're + # really small. + my $n_buf = ($p->{xdr_size} < 1020) ? int(1020 / $p->{xdr_size}) : 1; + $n_buf *= $p->{xdr_size}; + $phase->{size} = $p->{xdr_size}; + } + $phase = 0; + } else { + die $p->{where}, "Encode array arg not supported"; + } } + + # Function definition (data gets passed in *call) print RXOUT "\n"; + print RXOUT "int ", $func->{name}, "_", $subname, "(\n"; + print RXOUT "\tstruct rx_connection *z_conn,\n"; + print RXOUT "\tstruct ", $func->{name}, "_", $side, "_call *call)\n"; print RXOUT "{\n"; - my @bulk_params = (); - foreach my $p (@{$func->{request}}) { - if ($p->{class} eq "bulk" && - $p->{elem}->{class} ne "string" && - $p->{elem}->{class} ne "opaque" - ) { - push @bulk_params, $p; - } + unless (@params) { + die if ($side eq "client"); + print RXOUT "\tcall->more = 0;\n"; + print RXOUT "\treturn 0;\n"; + print RXOUT "}\n"; + return; } # Local variables - foreach my $p (@{$func->{request}}) { - if ($p->{class} eq "bulk" && $p->{elem}->{class} eq "string") { - print RXOUT "\tsize_t nr__", $p->{name}, " = strlen(", $p->{name}, ");\n"; - } - } - print RXOUT "\tnet_xdr_t buf[", $xdr_bufsize / 4, "];\n"; - print RXOUT "\tstruct iovec iov[", $#iovs + 1, "] = {\n"; - for my $iov (@iovs) { - print RXOUT "\t\t{ ", join(", ", @{$iov}), " },\n"; - } - print RXOUT "\t};\n"; - - if ($func->{req_has_charptr} || @bulk_params) { - print RXOUT "\tsize_t tmp;\n"; - } - print RXOUT "\tint ret;\n"; + print RXOUT "\tunsigned phase = call->phase;\n"; - # If we have bulk data that isn't string/opaque, we need a buffer to encode into - if (@bulk_params) { - print RXOUT "\tsize_t bulk_size = 0;\n"; - print RXOUT "\tvoid *bulk_buffer = NULL;\n"; - print RXOUT "\tnet_xdr_t *xdr;\n"; - print RXOUT "\n"; - foreach my $p (@bulk_params) { - print RXOUT "\tbulk_size += nr__", $p->{name}, " * ", $p->{elem}->{xdr_size}, ";\n"; - } + # Deal with each phase + print RXOUT "\n"; + print RXOUT "select_phase:\n" if ($have_bulk); + print RXOUT "\tswitch (phase) {\n"; - print RXOUT "\tif (bulk_size > 0) {\n"; - print RXOUT "\t\tbulk_buffer = malloc(bulk_size);\n"; - print RXOUT "\t\tif (!bulk_buffer)\n"; - print RXOUT "\t\t\treturn -1;\n"; + print RXOUT "\tcase 0:\n"; + print RXOUT "\t\tcall->more = 1;\n"; + my $phase_goto_label = 0; + my $phix; + for ($phix = 1; $phix <= $#phases + 1; $phix++) { + $phase = $phases[$phix - 1]; print RXOUT "\n"; - print RXOUT "\t\tbulk_size = 0;\n"; - foreach my $p (@bulk_params) { - print RXOUT "\t\tiov[", $p->{enc_ioc} + 1, "].iov_base = bulk_buffer + bulk_size;\n"; - print RXOUT "\t\tbulk_size += nr__", $p->{name}, " * ", $p->{elem}->{xdr_size}, ";\n"; + print RXOUT "\t\t/* --- Phase ", $phix, " --- */\n"; + if ($phase_goto_label == $phix) { + print RXOUT "\tphase_", $phix, ":\n"; + $phase_goto_label = 0; } - print RXOUT "\t}\n"; - } + # Determine how big bulk objects are + if ($phase->{type} eq "bulk") { + my $p = $phase->{params}->[0]; + print RXOUT "\t\tcall->bulk_count = ", $ptr, "nr__", $p->{name}, ";\n"; + print RXOUT "\t\tif (call->bulk_count == 0)\n"; + print RXOUT "\t\t\tgoto phase_", $phix + 1, ";\n"; + $phase_goto_label = $phix + 1; + print RXOUT "\t\tcall->bulk_index = 0;\n"; + } - # Marshal the data - my $ix = 4; + # Entry point for a phase + print RXOUT "\t\tcall->phase = ", $phix, ";\n"; + print RXOUT "\tcase ", $phix, ":\n"; - print RXOUT "\n"; - #print RXOUT "\txdr = request;\n"; - print RXOUT "\tbuf[0] = htonl(", $func->{opcode}, ");\n"; - foreach my $p (@{$func->{request}}) { - if ($p->{class} eq "basic" && $p->{type} !~ /64/) { - print RXOUT "\tbuf[", $ix/4, "] = htonl(", $p->{name}, ");\n"; - $ix += 4; - } elsif ($p->{class} eq "basic" && $p->{type} =~ /64/) { - print RXOUT "\tbuf[", $ix/4, "] = htonl((unsigned long)", $p->{name}, ");\n"; - $ix += 4; - print RXOUT "\tbuf[", $ix/4, "] = htonl((unsigned long)(", $p->{name}, " >> 32));\n"; - $ix += 4; - } elsif ($p->{class} eq "struct") { - print RXOUT "\trxgen_encode_", $p->{type}, "(buf + ", $ix/4, ", ", $p->{name}, ");\n"; - $ix += $p->{xdr_size}; - } elsif ($p->{class} eq "bulk" && ($p->{elem}->{class} eq "string" || - $p->{elem}->{class} eq "opaque")) { - if (exists $p->{dim}) { - print RXOUT "\tif (nr__", $p->{name}, " > ", $p->{dim}, ") {\n"; - print RXOUT "\t\terrno = EINVAL;\n"; - print RXOUT "\t\treturn -1;\n"; - print RXOUT "\t};\n"; + # Marshal the data + foreach my $p (@{$phase->{params}}) { + if ($p->{type} eq "bulk_size") { + print RXOUT "\t\trxrpc_enc(call, ", $ptr, "nr__", $p->{name}, ");\n"; + $close_phase = 0; + next; } - print RXOUT "\tbuf[", $ix/4, "] = htonl(nr__", $p->{name}, ");\n"; - $ix += 4; - print RXOUT "\tbuf[", $ix/4, "] = 0; /* end-of-object padding */\n"; - $ix += 4; - print RXOUT "\ttmp = (4 - (nr__", $p->{name}, " & 3)) & 3;\n"; - print RXOUT "\tiov[", $p->{enc_ioc} + 2, "].iov_len += tmp;\n"; - print RXOUT "\tiov[", $p->{enc_ioc} + 2, "].iov_base -= tmp;\n"; - } elsif ($p->{class} eq "bulk") { - print RXOUT "\txdr = iov[", $p->{enc_ioc} + 1, "].iov_base;\n"; - print RXOUT "\tfor (tmp = 0; tmp < nr__", $p->{name}, "; tmp++) {\n"; - if ($p->{elem}->{class} eq "basic" && $p->{elem}->{type} !~ /64/) { - print RXOUT "\t\t", $p->{elem}->{type}, " x;\n"; - print RXOUT "\t\tget__", $p->{name}, "(token__", $p->{name}, ", tmp, &x);\n"; - print RXOUT "\t\t*xdr++ = htonl(x);\n"; - } elsif ($p->{elem}->{class} eq "struct") { - print RXOUT "\t\txdr = rxgen_encode_", $p->{elem}->{type}, "(xdr, &p->", $p->{name}, "[i]);\n"; + + if ($p->{class} eq "bulk" && $p->{elem}->{class} eq "basic") { + if ($p->{elem}->{xdr_size} == 4) { + print RXOUT "\t\tif (", $ptr, "get__", $p->{name}, "(call, &", $ptr, "token__", $p->{name}, ") < 0)\n"; + } else { + print RXOUT "\t\tif (", $ptr, "get__", $p->{name}, "(call, &", $ptr, "token__", $p->{name}, ") < 0)\n"; + } + print RXOUT "\t\t\treturn -1;\n"; + + if ($p->{elem}->{xdr_size} == 4) { + print RXOUT "\t\trxrpc_enc(call, call->bulk_u32);\n"; + } elsif ($p->{elem}->{xdr_size} == 8) { + print RXOUT "\t\trxrpc_enc(call, call->bulk_u64 >> 32)\n"; + print RXOUT "\t\trxrpc_enc(call, (uint32_t)call->bulk_u64)\n"; + } else { + die; + } + print RXOUT "\t\tcall->bulk_index++;\n"; + } elsif ($p->{class} eq "bulk" && $p->{elem}->{class} eq "struct") { + print RXOUT "\t\tif (", $ptr, "get__", $p->{name}, "(call, &", $ptr, "token__", $p->{name}, ") < 0)\n"; + print RXOUT "\t\t\treturn -1;\n"; + print RXOUT "\t\trxgen_encode_", $p->{type}, "(call, call->bulk_item);\n"; + print RXOUT "\t\tcall->bulk_index++;\n"; + } elsif ($p->{class} eq "bulk" && ($p->{elem}->{class} eq "string" || + $p->{elem}->{class} eq "opaque")) { + print RXOUT "\t\trxrpc_enc_bytes(call, ", $ptr, $p->{name}, ", call);\n"; + print RXOUT "\t\trxrpc_enc_align(call);\n"; + } elsif ($p->{class} eq "opcode") { + print RXOUT "\t\trxrpc_enc(call, ", $p->{name}, ");\n"; + } elsif ($p->{class} eq "basic" && $p->{xdr_size} == 4) { + print RXOUT "\t\trxrpc_enc(call, ", $ptr, $p->{name}, ");\n"; + } elsif ($p->{class} eq "basic" && $p->{xdr_size} == 8) { + print RXOUT "\t\trxrpc_enc(call, ", $ptr, $p->{name}, " >> 32);\n"; + print RXOUT "\t\trxrpc_enc(call, (uint32_t)", $ptr, $p->{name}, ");\n"; + } elsif ($p->{class} eq "struct") { + print RXOUT "\t\trxgen_encode_", $p->{type}, "(call, ", $ptr, $p->{name}, ");\n"; } else { - die $p->{where}, "No decoding for array type '$type'"; + die $p->{where}, ": Unsupported type in decode"; + } + + if ($p->{class} eq "bulk") { + print RXOUT "\t\tif (call->bulk_index < call->bulk_count) {\n"; + print RXOUT "\t\t\tphase = ", $phix, ";\n"; + print RXOUT "\t\t\tgoto select_phase;\n"; + print RXOUT "\t\t}\n"; } - print RXOUT "\t}"; - } else { - die $p->{where}, ": Unsupported param encoding"; } } - # Send the message print RXOUT "\n"; - print RXOUT "\tret = rxrpc_send_request(z_conn, call, iov, ", $#iovs + 1, ");\n"; - print RXOUT "\tfree(bulk_buffer);\n" if (@bulk_params); - print RXOUT "\treturn ret;\n"; + print RXOUT "\t\t/* --- Phase ", $phix, " --- */\n"; + if ($phase_goto_label == $phix) { + print RXOUT "\tphase_", $phix, ":\n"; + $phase_goto_label = 0; + } + print RXOUT "\t\tcall->phase = ", $phix, ";\n"; + print RXOUT "\tcase ", $phix, ":\n"; + print RXOUT "\t\tif (rxrpc_post_enc(call) < 0)\n"; + print RXOUT "\t\t\treturn -1;\n"; + print RXOUT "\t\tcall->more = 0;\n"; + print RXOUT "\t\tbreak;\n"; + print RXOUT "\t}\n"; + + print RXOUT "\treturn 0;\n"; print RXOUT "}\n"; } ############################################################################### # -# Emit a function to decode a reply in a way that can be used from asynchronous -# code. +# Emit a function to decode a block in a way that can be used from asynchronous +# code. The opcode is expected to have been removed from the incoming call on +# the server side. # ############################################################################### -sub emit_func_decode_reply($) +sub emit_func_decode($$$$) { - my ($func) = @_; + my ($func, $side, $subname, $paramlist) = @_; + my @params = @{$paramlist}; + my $ptr = "obj->"; # We fetch the data in a number of phases. Each phase receives a chunk of # data of a certain size. A phase's size might be dependent on a variable @@ -301,10 +384,9 @@ sub emit_func_decode_reply($) # the data in the second. my @phases = (); my $phase = 0; - my $buf_size = 16; my $have_bulk = 0; - foreach my $p (@{$func->{reply}}) { + foreach my $p (@params) { unless ($phase) { $phase = { type => "flat", size => 0, params => [] }; push @phases, $phase; @@ -313,16 +395,12 @@ sub emit_func_decode_reply($) if ($p->{class} eq "basic" || $p->{class} eq "struct" ) { - $p->{reply_offset} = $phase->{size}; $phase->{size} += $p->{xdr_size}; push @{$phase->{params}}, $p; } elsif ($p->{class} eq "bulk") { - die if ($p->{elem}->{class} eq "string" || - $p->{elem}->{class} eq "opaque"); $have_bulk = 1; # Bulk objects begin with an element count - $p->{reply_offset} = $phase->{size}; $phase->{elem_count} = $phase->{size}; $phase->{size} += 4; @@ -333,7 +411,7 @@ sub emit_func_decode_reply($) elem => $p->{elem}, where => $p->{where}, xdr_size => $p->{xdr_size}, - ); + ); push @{$phase->{params}}, \%pseudoparam; # Create a new phase @@ -345,11 +423,15 @@ sub emit_func_decode_reply($) }; push @phases, $phase; + if ($p->{elem}->{class} eq "string" || + $p->{elem}->{class} eq "opaque") { + $phase->{bytearray} = 1; + } + # We don't want to be asking recvmsg() for one object at a time if # they're really small. my $n_buf = ($p->{xdr_size} < 1020) ? int(1020 / $p->{xdr_size}) : 1; $n_buf *= $p->{xdr_size}; - $buf_size = $n_buf if ($buf_size < $n_buf); $phase->{size} = $p->{xdr_size}; $phase = 0; } else { @@ -357,295 +439,307 @@ sub emit_func_decode_reply($) } } - # Determine the size of the reply buffer. It has to be big enough to hold - # all of a flat phase or at least one element of a bulk variable array. - # - # We need to round up to the nearest power-of-2 so that the circular buffer - # algorithm can operate with bitwise-AND masking and we need to leave a - # blank slot so that the head and tail pointers don't collide when the - # buffer is full. - foreach my $phase (@phases) { - $buf_size = $phase->{size} if ($buf_size < $phase->{size}); - } - $buf_size++; - - my $i; - for ($i = 1; $i < $buf_size; $i *= 2) { - ; - } - $buf_size = $i; - $func->{call_buf_size} = $buf_size; - # Function definition and arguments print RXOUT "\n"; - foreach $proto (@{$func->{recv_protos}}) { - print RXOUT $proto; - } - print RXOUT "\n"; + print RXOUT "static int rxgen_decode_", $func->{name}, "_", $subname, "(struct rx_call *call)\n"; print RXOUT "{\n"; - # Local variables - print RXOUT "\tconst net_xdr_t *xdr = call->reply;\n"; - print RXOUT "\tunsigned head = call->head / 4;\n"; - print RXOUT "\tunsigned tail = call->tail / 4;\n"; - print RXOUT "\tunsigned size = call->size / 4;\n"; - print RXOUT "\tunsigned mask = size - 1;\n"; - - if ($have_bulk) { - print RXOUT "\tunion {\n"; - foreach my $p (@{$func->{reply}}) { - if ($p->{class} eq "bulk") { - if ($p->{elem}->{class} eq "basic") { - print RXOUT "\t\t", $p->{type}, " ", $p->{name}, ";\n"; - } else { - print RXOUT "\t\tstruct ", $p->{type}, " *", $p->{name}, ";\n"; - } - } - } - print RXOUT "\t} bulk;\n"; + unless (@params) { + print RXOUT "\treturn 0;\n"; + print RXOUT "}\n"; + return; } + # Local variables + print RXOUT "\tstruct ", $func->{name}, "_", $subname, " *obj = call->decoder_private;\n"; + print RXOUT "\tunsigned count;\n"; + print RXOUT "\tunsigned phase = call->phase;\n"; + # Deal with each phase print RXOUT "\n"; - print RXOUT "\tswitch (call->phase) {\n"; + print RXOUT "select_phase:\n" if ($have_bulk); + print RXOUT "\tcount = call->data_count;\n"; + print RXOUT "\tswitch (phase) {\n"; print RXOUT "\tcase 0:\n"; my $phase_goto_label = 0; - my $close_phase = 0; my $phix; for ($phix = 1; $phix <= $#phases + 1; $phix++) { + print RXOUT "\n"; + print RXOUT "\t\t/* --- Phase ", $phix, " --- */\n"; $phase = $phases[$phix - 1]; if ($phase_goto_label == $phix) { print RXOUT "\tphase_", $phix, ":\n"; $phase_goto_label = 0; } - print RXOUT "\t\tcall->need_size = ", $phase->{size}, ";\n" - unless ($phase->{type} eq "bulk"); + + # Determine how big bulk objects are + if ($phase->{type} eq "bulk") { + my $p = $phase->{params}->[0]; + print RXOUT "\t\tcall->bulk_count = ", $ptr, "nr__", $p->{name}, ";\n"; + print RXOUT "\t\tcall->bulk_index = UINT_MAX;\n"; + + if ($p->{elem}->{class} eq "basic") { + print RXOUT "\t\tif (", $ptr, "store__", $p->{name}, "(call, &", $ptr, "token__", $p->{name}, ") < 0)\n"; + } elsif ($p->{elem}->{class} eq "string" || + $p->{elem}->{class} eq "opaque") { + print RXOUT "\t\tif (", $ptr, "alloc__", $p->{name}, "(call, &", $ptr, "token__", $p->{name}, ") < 0)\n"; + } else { + print RXOUT "\t\tif (", $ptr, "alloc__", $p->{name}, "(call, &", $ptr, "token__", $p->{name}, ") < 0)\n"; + } + print RXOUT "\t\t\treturn -1;\n"; + print RXOUT "\t\tif (call->bulk_count == 0)\n"; + print RXOUT "\t\t\tgoto phase_", $phix + 1, ";\n"; + $phase_goto_label = $phix + 1; + print RXOUT "\t\tcall->bulk_index = 0;\n"; + } else { + print RXOUT "\t\tcall->need_size = ", $phase->{size}, ";\n" + } + + # Entry point for a phase print RXOUT "\t\tcall->phase = ", $phix, ";\n"; print RXOUT "\tcase ", $phix, ":\n"; - print RXOUT "\t\tdo {\n"; - $close_phase = 1; - print RXOUT "\t\t\tif (CIRC_CNT(head, tail, size) < ", $phase->{size}, " / 4)"; - if ($phase->{type} eq "bulk") { + print RXOUT "\t\tif (count < ", $phase->{size}, ")"; + if ($phase->{type} eq "bulk" && !exists($phase->{bytearray}) && + $phase->{xdr_size} <= 512) { print RXOUT " {\n"; - print RXOUT "\t\t\t\tcall->need_size = MIN(call->need_bulk_count, call->item_per_buf);\n"; - print RXOUT "\t\t\t\tcall->need_size *= ", $phase->{xdr_size}, ";\n"; + print RXOUT "\t\t\tunsigned n = call->bulk_count - call->bulk_index;\n"; + print RXOUT "\t\t\tn = MIN(n, ", int(1024 / $phase->{xdr_size}), ");\n"; + print RXOUT "\t\t\tcall->need_size = n * ", $phase->{xdr_size}, ";\n"; + print RXOUT "\t\t\treturn 1;\n"; + print RXOUT "\t\t}"; } else { print RXOUT "\n"; + print RXOUT "\t\t\treturn 1;\n"; } - print RXOUT "\t\t\t\tgoto need_more_data;\n"; - print RXOUT "\t\t\t}" if ($phase->{type} eq "bulk"); # Unmarshal the data print RXOUT "\n"; foreach my $p (@{$phase->{params}}) { if ($p->{type} eq "bulk_size") { - print RXOUT "\t\t\tcall->need_bulk_count = ntohl(xdr[tail++ & mask]);\n"; - print RXOUT "\t\t} while (0);\n" if ($close_phase); - $close_phase = 0; - - if ($p->{elem}->{class} eq "basic") { - print RXOUT "\t\tif (store__", $p->{name}, "(token__", $p->{name}, ", call->need_bulk_count, NULL) < 0)\n"; - } else { - print RXOUT "\t\tif (alloc__", $p->{name}, "(token__", $p->{name}, ", call->need_bulk_count, NULL) < 0)\n"; - } - print RXOUT "\t\t\treturn -1;\n"; - print RXOUT "\t\tif (call->need_bulk_count == 0)\n"; - print RXOUT "\t\t\tgoto phase_", $phix + 2, ";\n"; - $phase_goto_label = $phix + 2; - print RXOUT "\t\tcall->item_per_buf = size / (", $p->{xdr_size}, " / 4);\n"; - print RXOUT "\t\tcall->bulk_index = 0;\n"; + print RXOUT "\t\t", $ptr, "nr__", $p->{name}, " = rxrpc_dec(call);\n"; next; } if ($p->{class} eq "bulk" && $p->{elem}->{class} eq "basic") { if ($p->{elem}->{xdr_size} == 4) { - print RXOUT "\t\t\tbulk.", $p->{name}, " = ntohl(xdr[tail++ & mask]);\n"; + print RXOUT "\t\tcall->bulk_u32 = rxrpc_dec(call);\n"; + print RXOUT "\t\tif (", $ptr, "store__", $p->{name}, "(call, &", $ptr, "token__", $p->{name}, ") < 0)\n"; } elsif ($p->{elem}->{xdr_size} == 8) { - print RXOUT "\t\t\tbulk.", $p->{name}, " = (uint64_t)ntohl(xdr[tail++ & mask]) << 32\n"; - print RXOUT "\t\t\t\t | (uint64_t)ntohl(xdr[tail++ & mask]);\n"; + print RXOUT "\t\tcall->bulk_u64 = (uint64_t)rxrpc_dec(call) << 32;\n"; + print RXOUT "\t\tcall->bulk_u64 |= (uint64_t)rxrpc_dec(call);\n"; + print RXOUT "\t\tif (", $ptr, "store__", $p->{name}, "(call, &", $ptr, "token__", $p->{name}, ") < 0)\n"; } else { die; } - print RXOUT "\t\t\tif (store__", $p->{name}, "(token__", $p->{name}, ", call->bulk_index++, &bulk.", $p->{name}, ") < 0)\n"; - print RXOUT "\t\t\t\treturn -1;\n"; + print RXOUT "\t\t\treturn -1;\n"; + print RXOUT "\t\tcall->bulk_index++;\n"; } elsif ($p->{class} eq "bulk" && $p->{elem}->{class} eq "struct") { - print RXOUT "\t\t\tif (alloc__", $p->{name}, "(token__", $p->{name}, ", call->bulk_index++, &bulk.", $p->{name}, ") < 0)\n"; - print RXOUT "\t\t\t\treturn -1;\n"; - print RXOUT "\t\t\ttail = rxgen_decode_", $p->{type}, "(bulk.", $p->{name}, ", xdr, tail, mask);\n"; + print RXOUT "\t\tif (", $ptr, "alloc__", $p->{name}, "(call, &", $ptr, "token__", $p->{name}, ") < 0)\n"; + print RXOUT "\t\t\treturn -1;\n"; + print RXOUT "\t\trxgen_decode_", $p->{type}, "(call, call->bulk_item);\n"; + print RXOUT "\t\tcall->bulk_index++;\n"; + } elsif ($p->{class} eq "bulk" && ($p->{elem}->{class} eq "string" || + $p->{elem}->{class} eq "opaque")) { + print RXOUT "\t\trxrpc_dec_bytes(call);\n"; + print RXOUT "\t\trxrpc_dec_align(call);\n"; } elsif ($p->{class} eq "basic" && $p->{xdr_size} == 4) { - print RXOUT "\t\t\t*", $p->{name}, " = ntohl(xdr[tail++ & mask]);\n"; + print RXOUT "\t\t", $ptr, $p->{name}, " = rxrpc_dec(call);\n"; } elsif ($p->{class} eq "basic" && $p->{xdr_size} == 8) { - print RXOUT "\t\t\t*", $p->{name}, " = (uint64_t)ntohl(xdr[tail++ & mask]) << 32\n"; - print RXOUT "\t\t\t\t | (uint64_t)ntohl(xdr[tail++ & mask]);\n"; + print RXOUT "\t\t", $ptr, $p->{name}, " = (uint64_t)rxrpc_dec(call) << 32;\n"; + print RXOUT "\t\t", $ptr, $p->{name}, " |= (uint64_t)rxrpc_dec(call);\n"; } elsif ($p->{class} eq "struct") { - print RXOUT "\t\t\ttail = rxgen_decode_", $p->{type}, "(", $p->{name}, ", xdr, tail, mask);\n"; + print RXOUT "\t\trxgen_decode_", $p->{type}, "(call, ", $ptr, $p->{name}, ");\n"; } else { die $p->{where}, ": Unsupported type in decode"; } if ($p->{class} eq "bulk") { - print RXOUT "\t\t\tcall->need_bulk_count--;\n"; - print RXOUT "\t\t} while (call->need_bulk_count > 0);\n"; - $close_phase = 0; + print RXOUT "\t\tif (rxrpc_post_dec(call) < 0)\n"; + print RXOUT "\t\t\treturn -1;\n"; + print RXOUT "\t\tif (call->bulk_index < call->bulk_count) {\n"; + print RXOUT "\t\t\tphase = ", $phix, ";\n"; + print RXOUT "\t\t\tgoto select_phase;\n"; + print RXOUT "\t\t}\n"; } } - print RXOUT "\t\t} while (0);\n" if ($close_phase); + if ($phase->{type} ne "bulk") { + print RXOUT "\t\tif (rxrpc_post_dec(call) < 0)\n"; + print RXOUT "\t\t\treturn -1;\n"; + } } + print RXOUT "\n"; + print RXOUT "\t\t/* --- Phase ", $phix, " --- */\n"; if ($phase_goto_label == $phix) { print RXOUT "\tphase_", $phix, ":\n"; $phase_goto_label = 0; } print RXOUT "\t\tcall->phase = ", $phix, ";\n"; - print RXOUT "\tcase ", $phix, ":\n"; - print RXOUT "\t\tif (!call->got_eor) {\n"; - print RXOUT "\t\t\ttail = head;\n"; - print RXOUT "\t\t\tcall->need_size = size - 1;\n"; - print RXOUT "\t\t\tgoto need_more_data_2;\n"; - print RXOUT "\t\t}\n"; - print RXOUT "\t\tbreak;\n"; - print RXOUT "\t}\n"; - - print RXOUT "\n"; - print RXOUT "\tcall->tail = tail * 4;\n"; - print RXOUT "\treturn 0;\n"; - - print RXOUT "\n"; - print RXOUT "need_more_data:\n"; - print RXOUT "\tif (call->got_eor) {\n"; - print RXOUT "\t\terrno = EMSGSIZE;\n"; - print RXOUT "\t\treturn -1;\n"; + print RXOUT "\t\tcall->need_size = 0;\n"; + print RXOUT "\tdefault:\n"; + print RXOUT "\t\treturn 0;\n"; print RXOUT "\t}\n"; - print RXOUT "need_more_data_2:\n"; - print RXOUT "\tcall->tail = tail * 4;\n"; - print RXOUT "\treturn 1;\n"; - print RXOUT "}\n"; } ############################################################################### # -# Emit a dummy function to decode a replyless reply in a way that can be used -# from asynchronous code. +# Emit a function to encode and dispatch a request or a response # ############################################################################### -sub emit_func_dummy_decode_reply($) +sub emit_func_send($$) { - my ($func) = @_; + my ($func, $what) = @_; + my $params; + my $bad_ret; # Function definition and arguments - print RXOUT "\n"; - foreach $proto (@{$func->{recv_protos}}) { - print RXOUT $proto; + my @protos; + if ($what eq "request") { + @protos = @{$func->{send_request_protos}}; + $params = $func->{request}; + $bad_ret = "NULL"; + } else { + @protos = @{$func->{send_response_protos}}; + $params = $func->{response}; + $bad_ret = "-1"; } print RXOUT "\n"; + if ($what eq "request") { + print RXOUT "struct rx_call *", $func->{name} . "(\n"; + print RXOUT "\tstruct rx_connection *z_conn"; + } else { + print RXOUT "int respond_to_", $func->{name} . "(\n"; + print RXOUT "\tstruct rx_call *call"; + } + foreach $proto (@protos) { + print RXOUT ",\n\t", $proto; + } + if ($what eq "request" && @{$func->{response}}) { + print RXOUT ",\n"; + print RXOUT "\tstruct ", $func->{name}, "_response *response"; + } + print RXOUT ")\n"; print RXOUT "{\n"; - print RXOUT "\tcall->tail = call->head;\n"; - print RXOUT "\tcall->need_size = call->size - 1;\n"; - print RXOUT "\treturn call->got_eor ? 0 : 1;\n"; - print RXOUT "}\n"; -} - -############################################################################### -# -# Emit a function to allocate a call struct with appropriately sized bufferage. -# -############################################################################### -sub emit_func_alloc_call($) -{ - my ($func) = @_; + print RXOUT "\tstruct rx_call *call;\n" if ($what eq "request"); - my $buf_size = 16; + my @all_bulk_params = grep { $_->{class} eq "bulk"; } @{$params}; - $buf_size = $func->{call_buf_size} if (exists $func->{call_buf_size}); + my @bulk_params = grep { ($_->{elem}->{class} ne "string" && + $_->{elem}->{class} ne "opaque"); } @all_bulk_params; - # Emit a function to allocate an rxrpc_call struct for this call - print RXOUT "\n"; - print RXOUT "struct rx_call *rxgen_alloc_call_", $func->{name}, "(void)\n"; - print RXOUT "{\n"; - print RXOUT "\tstruct rx_call *call = malloc(sizeof(*call) + ", $buf_size, ");\n"; - print RXOUT "\tif (call) {\n"; - print RXOUT "\t\tmemset(call, 0, sizeof(*call));\n"; - print RXOUT "\t\tcall->size = ", $buf_size, ";\n"; - print RXOUT "\t}\n"; - print RXOUT "\treturn call;\n"; - print RXOUT "}\n"; -} + # Local variables + print RXOUT "\tint ret;\n"; -############################################################################### -# -# Emit a function to make a simple synchronous call -# -############################################################################### -sub emit_func_simple_sync_call($) -{ - my ($func) = @_; + # Check lengths + if (@all_bulk_params) { + print RXOUT "\n"; + print RXOUT "\tif ("; + my $first = 1; + foreach my $p (@all_bulk_params) { + if ($first) { + $first = 0; + } else { + print RXOUT " ||\n\t "; + } + if ($p->{elem}->{class} eq "string" || + $p->{elem}->{class} eq "opaque") { + print RXOUT "!", $p->{name}; + } else { + print RXOUT "!get__", $p->{name}; + } + if (exists($p->{dim})) { + print RXOUT " || nr__", $p->{name}, " > ", $p->{dim}; + } + } + print RXOUT ") {\n"; + print RXOUT "\t\terrno = EINVAL;\n"; + print RXOUT "\t\treturn ", $bad_ret, ";\n"; + print RXOUT "\t};\n"; + } - # Function declaration - print RXHDR "\n"; - print RXHDR "extern "; - foreach $proto (@{$func->{protos}}) { - print RXHDR $proto; + # Allocate call + if ($what eq "request") { + print RXOUT "\n"; + print RXOUT "\tcall = rxrpc_alloc_call(z_conn, 0);\n"; + print RXOUT "\tif (!call)\n"; + print RXOUT "\t\treturn ", $bad_ret, ";\n"; + print RXOUT "\tcall->decoder = rxgen_decode_", $func->{name}, "_response;\n"; + print RXOUT "\tcall->decoder_private = response;\n" if (@{$func->{response}}); } - print RXHDR ";\n"; - # Function definition and arguments - print RXOUT "\n"; - foreach $proto (@{$func->{protos}}) { - print RXOUT $proto; + # Marshal the data + print RXOUT "\n" if ($what eq "request" || @{$params}); + print RXOUT "\trxrpc_enc(call, ", $func->{opcode}, ");\n" if ($what eq "request"); + foreach my $p (@{$params}) { + if ($p->{class} eq "basic" && $p->{type} !~ /64/) { + print RXOUT "\trxrpc_enc(call, ", $p->{name}, ");\n"; + } elsif ($p->{class} eq "basic" && $p->{type} =~ /64/) { + print RXOUT "\trxrpc_enc(call, (uint32_t)", $p->{name}, ");\n"; + print RXOUT "\trxrpc_enc(call, (uint32_t)(", $p->{name}, " >> 32));\n"; + } elsif ($p->{class} eq "struct") { + print RXOUT "\trxgen_encode_", $p->{type}, "(call, ", $p->{name}, ");\n"; + } elsif ($p->{class} eq "bulk" && ($p->{elem}->{class} eq "string" || + $p->{elem}->{class} eq "opaque")) { + print RXOUT "\trxrpc_enc_bytes(call, ", $p->{name}, ", nr__", $p->{name}, ");\n"; + print RXOUT "\trxrpc_enc_align(call);\n"; + } elsif ($p->{class} eq "bulk") { + print RXOUT "\trxrpc_enc(call, nr__", $p->{name}, ");\n"; + print RXOUT "\tcall->bulk_count = nr__", $p->{name}, ";\n"; + print RXOUT "\tfor (call->bulk_index = 0; call->bulk_index < call->bulk_count; call->bulk_index++) {\n"; + if ($p->{elem}->{class} eq "struct") { + print RXOUT "\t\tstruct ", $p->{elem}->{type}, " x;\n"; + } else { + print RXOUT "\t\t", $p->{elem}->{type}, " x;\n"; + } + print RXOUT "\t\tcall->bulk_item = &x;\n"; + print RXOUT "\t\tif (get__", $p->{name}, "(call, token__", $p->{name}, ") < 0)\n"; + print RXOUT "\t\t\tgoto error;\n"; + if ($p->{elem}->{class} eq "basic" && $p->{elem}->{type} !~ /64/) { + if ($p->{type} !~ /^u/) { + print RXOUT "\t\trxrpc_enc(call, (u", $p->{type}, ")x);\n"; + } else { + print RXOUT "\t\trxrpc_enc(call, x);\n"; + } + } elsif ($p->{class} eq "basic" && $p->{type} =~ /64/) { + print RXOUT "\t\trxrpc_enc(call, (uint32_t)", $p->{name}, ");\n"; + print RXOUT "\t\trxrpc_enc(call, (uint32_t)(", $p->{name}, " >> 32));\n"; + } elsif ($p->{elem}->{class} eq "struct") { + print RXOUT "\t\trxgen_encode_", $p->{elem}->{type}, "(call, &x);\n"; + } else { + die $p->{where}, "No decoding for array type '$type'"; + } + print RXOUT "\t}\n"; + } else { + die $p->{where}, ": Unsupported param encoding"; + } } - print RXOUT "\n"; - # Function body, beginning with local variables - print RXOUT "{\n"; - print RXOUT "\tstruct rx_call *call;\n"; - print RXOUT "\tint ret;\n"; - print RXOUT "\n"; + print RXOUT "\tif (rxrpc_post_enc(call) < 0)\n"; + print RXOUT "\t\tgoto error;\n"; + print RXOUT "\tcall->more = 0;\n"; - # Allocate a call record and reply buffer - print RXOUT "\tcall = rxgen_alloc_call_", $func->{name}, "();\n"; - print RXOUT "\tif (!call)\n"; - print RXOUT "\t\treturn -1;\n"; + # Send the message print RXOUT "\n"; - - # Send the request - print RXOUT "\tret = rxgen_send_request_", $func->{name}, "("; - print RXOUT "z_conn, call"; - foreach (@{$func->{send_args}}) { - print RXOUT ", ", $_; - } - print RXOUT ");\n"; - print RXOUT "\tif (ret != 0) {\n"; - print RXOUT "\t\tfree(call);\n"; - print RXOUT "\t\treturn ret;\n"; - print RXOUT "\t}\n"; - - # Set up the reply buffer and try to parse the reply as it's received - print RXOUT "\tfor (;;) {\n"; - - print RXOUT "\t\tret = rxgen_decode_reply_", $func->{name}, "(z_conn, call"; - foreach my $a (@{$func->{recv_args}}) { - print RXOUT ", ", $a; + print RXOUT "\tret = rxrpc_send_data(call);\n"; + print RXOUT "\tif (ret < 0)\n"; + print RXOUT "\t\tgoto error;\n"; + if ($what eq "request") { + print RXOUT "\treturn call;\n"; + } else { + print RXOUT "\treturn 0;\n"; } - print RXOUT ");\n"; - print RXOUT "\t\tif (ret != 1)\n"; - print RXOUT "\t\t\tbreak;\n"; - # Receive reply data - print RXOUT "\t\tret = rxrpc_wait_for_sync_reply(z_conn, call);\n"; - print RXOUT "\t\tif (ret != 0)\n"; - print RXOUT "\t\t\tbreak;\n"; - - print RXOUT "\t}\n"; print RXOUT "\n"; - print RXOUT "\tfree(call);\n"; - print RXOUT "\treturn ret;\n"; + print RXOUT "error:\n"; + print RXOUT "\trxrpc_terminate_call(call, 0);\n"; + print RXOUT "\treturn ", $bad_ret, ";\n"; print RXOUT "}\n"; } diff --git a/rxgen/emit_py_module.pm b/rxgen/emit_py_module.pm index 54c0ea0..8830730 100644 --- a/rxgen/emit_py_module.pm +++ b/rxgen/emit_py_module.pm @@ -14,6 +14,15 @@ # ############################################################################### sub emit_py_module() { + # We want an exception we can raise when we get a remote abort + print PYHDR "extern PyObject *kafs_remote_abort;\n"; + + print PYOUT "\n"; + print PYOUT "/*\n"; + print PYOUT " * The remote-abort exception.\n"; + print PYOUT " */\n"; + print PYOUT "PyObject *kafs_remote_abort;\n"; + # Emit python structure wrapper static method table print PYOUT "\n"; print PYOUT "/*\n"; @@ -21,20 +30,13 @@ sub emit_py_module() { print PYOUT " */\n"; print PYOUT "static PyMethodDef module_methods[] = {\n"; - foreach my $struct (@structs) { - print PYOUT "\t{\"new_", $struct->{type}, "\", (PyCFunction)kafs_new_py_", $struct->{type}, ", METH_NOARGS,\n"; - print PYOUT "\t \"Create a new ", $struct->{type}, " record.\"\n"; - print PYOUT "\t},\n"; - } + print PYOUT "\t{\"rx_new_connection\", (PyCFunction)kafs_py_rx_new_connection, METH_VARARGS, \"\" },\n"; - foreach my $funcname (sort keys %func_names) { - my $func = $func_names{$funcname}; - print PYOUT "\t{\"", $func->{name}, "\", (PyCFunction)kafs_", $func->{name}, ", METH_VARARGS, \"\" },\n"; + foreach my $def (@py_func_defs) { + print PYOUT "\t{\"", $def->{name}, "\", (PyCFunction)", $def->{c_func}, ", METH_VARARGS,"; + print PYOUT " \"", $def->{doc}, "\" },\n"; } - print PYOUT "\t{\"rx_new_connection\", (PyCFunction)kafs_py_rx_new_connection, METH_VARARGS,\n"; - print PYOUT "\t\"\" },\n"; - print PYOUT "\t{}\n"; print PYOUT "};\n"; @@ -58,13 +60,13 @@ sub emit_py_module() { print PYOUT "\tPyObject *m;\n"; # Load types - if (@structs) { + if (@py_type_defs) { print PYOUT "\tif ("; print PYOUT "PyType_Ready(&py_rx_connectionType) < 0"; my $first = 0; - foreach my $struct (@structs) { + foreach my $def (@py_type_defs) { print PYOUT " ||\n\t " unless ($first); - print PYOUT "PyType_Ready(&py_", $struct->{type}, "Type) < 0"; + print PYOUT "PyType_Ready(&", $def->{c_type}, ") < 0"; $first = 0; } print PYOUT ")\n"; @@ -78,22 +80,26 @@ sub emit_py_module() { if (%constants) { print PYOUT "\n"; - foreach my $c (sort keys %constants) { + foreach my $c (sort grep /^[^0-9]/, keys %constants) { print PYOUT "\tPyModule_AddIntConstant(m, \"$c\", $c);\n"; } } - if (@structs) { + if (@py_type_defs) { print PYOUT "\n"; - foreach my $struct (@structs) { - print PYOUT "\tPy_INCREF(&py_", $struct->{type}, "Type);\n"; - print PYOUT "\tPyModule_AddObject(m, \"", $struct->{type}, "\", (PyObject *)&py_", $struct->{type}, "Type);\n"; + foreach my $def (@py_type_defs) { + print PYOUT "\tPy_INCREF(&", $def->{c_type}, ");\n"; + print PYOUT "\tPyModule_AddObject(m, \"", $def->{name}, "\", (PyObject *)&", $def->{c_type}, ");\n"; } - - print PYOUT "\n"; - print PYOUT "\treturn m;\n"; } + print PYOUT "\n"; + print PYOUT "\tkafs_remote_abort = PyErr_NewException(\"kafs.RemoteAbort\", NULL, NULL);\n"; + print PYOUT "\tif (!kafs_remote_abort)\n"; + print PYOUT "\t\treturn NULL;\n"; + + print PYOUT "\n"; + print PYOUT "\treturn m;\n"; print PYOUT "}\n"; } diff --git a/rxgen/emit_py_sync_funcs.pm b/rxgen/emit_py_sync_funcs.pm index 636f1b2..5735546 100644 --- a/rxgen/emit_py_sync_funcs.pm +++ b/rxgen/emit_py_sync_funcs.pm @@ -11,6 +11,158 @@ my %bulk_get_helpers = (); my %bulk_set_helpers = (); +############################################################################### +# +# Emit python objects to represent received parameter sets and received +# response sets for RPC calls. +# +############################################################################### +sub emit_py_func_param_object($$) { + my ($func, $set) = @_; + + my $struct_req = "py_" . $func->{name}. "_". $set; + my @basic = (); + my @complex = (); + my $params = (); + my $division = ""; + + push @py_type_defs, { + name => $func->{name} . "_" . $set, + c_type => $struct_req . "Type", + }; + + if ($set eq "request") { + $params = $func->{request}; + $division = "calls"; + } else { + $params = $func->{response}; + $division = "responses"; + } + + # Define a C structure to hold the python object header and the data. + print PYHDR "\n"; + print PYHDR "struct ", $struct_req, " {\n"; + print PYHDR "\tstruct py_rx_", $set, " common;\n"; + if (@{$params}) { + my $have_opaque = 0; + print PYHDR "\tstruct {\n"; + foreach my $p (@{$params}) { + if ($p->{class} eq "basic") { + push @basic, $p; + print PYHDR "\t\t", $p->{type}, "\t", $p->{name}, ";\n"; + } else { + push @complex, $p; + print PYHDR "\t\tPyObject\t*", $p->{name}, ";\n"; + } + $have_opaque = 1 if ($p->{class} eq "bulk" && $p->{elem}->{class} eq "opaque"); + } + print PYHDR "\t} x;\n"; + print PYHDR "\tPy_buffer dec_buf;\n" if ($have_opaque); + } + print PYHDR "};\n"; + + # We need to have a new function if the object is to be allocatable by the + # Python interpreter + print PYOUT "\n"; + print PYOUT "static PyObject *\n"; + print PYOUT $struct_req, "_new(PyTypeObject *subtype, PyObject *args, PyObject *kwds)\n"; + print PYOUT "{\n"; + print PYOUT "\tPyObject *obj;\n"; + print PYOUT "\n"; + print PYOUT "\tobj = subtype->tp_alloc(subtype, 1);\n"; + if (@{$params}) { + print PYOUT "\tif (obj) {\n"; + print PYOUT "\t\tstruct ", $struct_req, " *self = (struct ", $struct_req, " *)obj;\n"; + print PYOUT "\t\tmemset(&self->x, 0, sizeof(self->x));\n"; + print PYOUT "\t}\n"; + } + print PYOUT "\treturn obj;\n"; + print PYOUT "}\n"; + + # We have to have a deallocation function + print PYOUT "\n"; + print PYOUT "static void ", $struct_req, "_dealloc(struct ", $struct_req, " *self)\n"; + print PYOUT "{\n"; + foreach my $p (@complex) { + print PYOUT "\tPy_XDECREF(self->x.", $p->{name}, ");\n"; + } + print PYOUT "\tPy_TYPE(self)->tp_free((PyObject *)self);\n"; + print PYOUT "}\n"; + + # All elements are made directly accessible to the Python interpreter, + # either as integer types or as object types. + if (@{$params}) { + print PYOUT "\n"; + print PYOUT "static PyMemberDef ", $struct_req, "_members[] = {\n"; + foreach my $p (@{$params}) { + print PYOUT "\t{ \"", $p->{name}, "\", "; + if ($p->{class} eq "bulk") { print PYOUT "T_OBJECT_EX"; + } elsif ($p->{type} eq "char" ) { print PYOUT "T_CHAR"; + } elsif ($p->{type} eq "int8_t" ) { print PYOUT "T_BYTE"; + } elsif ($p->{type} eq "int16_t" ) { print PYOUT "T_SHORT"; + } elsif ($p->{type} eq "int32_t" ) { print PYOUT "T_INT"; + } elsif ($p->{type} eq "int64_t" ) { print PYOUT "T_LONGLONG"; + } elsif ($p->{type} eq "uint8_t" ) { print PYOUT "T_UBYTE"; + } elsif ($p->{type} eq "uint16_t") { print PYOUT "T_USHORT"; + } elsif ($p->{type} eq "uint32_t") { print PYOUT "T_UINT"; + } elsif ($p->{type} eq "uint64_t") { print PYOUT "T_ULONGLONG"; + } else { + print PYOUT "T_OBJECT_EX"; + } + print PYOUT ", offsetof(struct ", $struct_req, ", x.", $p->{name}, "), 0, \"\"},\n"; + } + print PYOUT "\t{}\n"; + print PYOUT "};\n"; + } + + # Emit the Python type definition + print PYOUT "\n"; + print PYOUT "static PyTypeObject ", $struct_req, "Type = {\n"; + print PYOUT "\tPyVarObject_HEAD_INIT(NULL, 0)\n"; + print PYOUT "\t\"kafs.", $func->{name}, "_", $set, "\",\t\t/*tp_name*/\n"; + print PYOUT "\tsizeof(struct ", $struct_req, "),\t/*tp_basicsize*/\n"; + print PYOUT "\t0,\t\t\t\t/*tp_itemsize*/\n"; + print PYOUT "\t(destructor)", $struct_req, "_dealloc, /*tp_dealloc*/\n"; + print PYOUT "\t0,\t\t\t\t/*tp_print*/\n"; + print PYOUT "\t0,\t\t\t\t/*tp_getattr*/\n"; + print PYOUT "\t0,\t\t\t\t/*tp_setattr*/\n"; + print PYOUT "\t0,\t\t\t\t/*tp_compare*/\n"; + print PYOUT "\t0,\t\t\t\t/*tp_repr*/\n"; + print PYOUT "\t0,\t\t\t\t/*tp_as_number*/\n"; + print PYOUT "\t0,\t\t\t\t/*tp_as_sequence*/\n"; + print PYOUT "\t0,\t\t\t\t/*tp_as_mapping*/\n"; + print PYOUT "\t0,\t\t\t\t/*tp_hash */\n"; + print PYOUT "\t0,\t\t\t\t/*tp_call */\n"; + print PYOUT "\t0,\t\t\t\t/*tp_str*/\n"; + print PYOUT "\t0,\t\t\t\t/*tp_getattro*/\n"; + print PYOUT "\t0,\t\t\t\t/*tp_setattro*/\n"; + print PYOUT "\t0,\t\t\t\t/*tp_as_buffer*/\n"; + print PYOUT "\tPy_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /*tp_flags*/\n"; + print PYOUT "\t\"\",\t\t\t\t/* tp_doc */\n"; + print PYOUT "\t0,\t\t\t\t/* tp_traverse */\n"; + print PYOUT "\t0,\t\t\t\t/* tp_clear */\n"; + print PYOUT "\t0,\t\t\t\t/* tp_richcompare */\n"; + print PYOUT "\t0,\t\t\t\t/* tp_weaklistoffset */\n"; + print PYOUT "\t0,\t\t\t\t/* tp_iter */\n"; + print PYOUT "\t0,\t\t\t\t/* tp_iternext */\n"; + print PYOUT "\t0,\t\t\t\t/* tp_methods */\n"; + if (@{$params}) { + print PYOUT "\t", $struct_req, "_members,\n"; + } else { + print PYOUT "\t0,\t\t\t\t/* tp_members */\n"; + } + print PYOUT "\t0,\t\t\t\t/* tp_getset */\n"; + print PYOUT "\t0,\t\t\t\t/* tp_base */\n"; + print PYOUT "\t0,\t\t\t\t/* tp_dict */\n"; + print PYOUT "\t0,\t\t\t\t/* tp_descr_get */\n"; + print PYOUT "\t0,\t\t\t\t/* tp_descr_set */\n"; + print PYOUT "\t0,\t\t\t\t/* tp_dictoffset */\n"; + print PYOUT "\t0,\t\t\t\t/* tp_init */\n"; + print PYOUT "\t0,\t\t\t\t/* tp_alloc */\n"; + print PYOUT "\t", $struct_req, "_new,\n"; + print PYOUT "};\n"; +} + ############################################################################### # # Emit functions to help deal with bulk lists @@ -24,108 +176,54 @@ sub emit_py_func_bulk_helper($) next if ($p->{class} ne "bulk"); next if ($p->{elem}->{class} eq "string" || $p->{elem}->{class} eq "opaque"); - # Data transmission - if ($p->{dir} eq "IN" && !exists $bulk_get_helpers{$p->{type}}) { + # Data encoding + if (!exists $bulk_get_helpers{$p->{type}}) { $bulk_get_helpers{$p->{type}} = 1; print PYOUT "\n"; - print PYOUT "static int py_get__", $p->{type}, "("; - if ($p->{elem}->{class} eq "basic") { - print PYOUT "void *token, int index, ", $p->{type}, " *object)\n"; - } elsif ($p->{elem}->{class} eq "struct") { - print PYOUT "void *token, int index, ", $p->{type}, " **object)\n"; - } else { - die $p->{where}, ": Unsupported type for bulk helper"; - } - + print PYOUT "static __attribute__((unused))\n"; + print PYOUT "int py_encode_bulk_", $p->{type}, "(struct rx_call *call, PyObject *list)\n"; print PYOUT "{\n"; - print PYOUT "\tPyObject *list = token;\n"; print PYOUT "\tPyObject *item;\n"; + print PYOUT "\tunsigned count, i;\n"; print PYOUT "\n"; - print PYOUT "\titem = PyList_GetItem(list, index);\n"; - print PYOUT "\tif (!item)\n"; - print PYOUT "\t\treturn -1;\n"; - + print PYOUT "\tcount = PyList_Size(list);\n"; + print PYOUT "\trxrpc_enc(call, count);\n"; print PYOUT "\n"; - if ($p->{elem}->{class} eq "basic") { - print PYOUT "\tif (!PyLong_Check(item)) {\n"; - print PYOUT "\t\tPyErr_SetString(PyExc_TypeError, \"Expected list of ", $p->{type}, "\");\n"; - print PYOUT "\t\treturn -1;\n"; - print PYOUT "\t}\n"; - } else { - print PYOUT "\tif (py_premarshal_", $p->{type}, "(item))\n"; - print PYOUT "\t\treturn -1;\n"; - } - - if ($p->{elem}->{class} eq "basic") { - if ($p->{elem}->{type} eq "int64_t") { - print PYOUT "\t*object = PyLong_AsLongLong(item);\n"; - } elsif ($p->{elem}->{type} eq "uint64_t") { - print PYOUT "\t*object = PyLong_AsUnsignedLongLong(item);\n"; - } elsif ($p->{elem}->{type} =~ /^int/) { - print PYOUT "\t*object = PyLong_AsLong(item);\n"; - } elsif ($p->{elem}->{type} =~ /^uint|^char/) { - print PYOUT "\t*object = PyLong_AsUnsignedLong(item);\n"; - } - } else { - print PYOUT "\t*object = &((struct py_", $p->{type}, " *)item)->x;\n"; - } - print PYOUT "\treturn 0;\n"; - print PYOUT "}\n"; - } - - # Data reception - if ($p->{dir} eq "OUT" && !exists $bulk_set_helpers{$p->{type}}) { - $bulk_set_helpers{$p->{type}} = 1; + print PYOUT "\tfor (i = 0; i < count; i++) {\n"; + print PYOUT "\t\titem = PyList_GetItem(list, i);\n"; + print PYOUT "\t\tif (!item)\n"; + print PYOUT "\t\t\treturn -1;\n"; print PYOUT "\n"; if ($p->{elem}->{class} eq "basic") { - print PYOUT "static int py_store__", $p->{type}, "("; - print PYOUT "void *token, int index, ", $p->{type}, " *object)\n"; - } elsif ($p->{elem}->{class} eq "struct") { - print PYOUT "static int py_alloc__", $p->{type}, "("; - print PYOUT "void *token, int index, struct ", $p->{type}, " **object)\n"; + print PYOUT "\t\tif (!PyLong_Check(item)) {\n"; + print PYOUT "\t\t\tPyErr_SetString(PyExc_TypeError, \"Expected list of ", $p->{type}, "\");\n"; + print PYOUT "\t\t\treturn -1;\n"; + print PYOUT "\t\t}\n"; } else { - die $p->{where}, ": Unsupported type for bulk helper"; + print PYOUT "\t\tif (py_premarshal_", $p->{type}, "(item))\n"; + print PYOUT "\t\t\treturn -1;\n"; } - print PYOUT "{\n"; - print PYOUT "\tPyObject *list = token;\n"; - print PYOUT "\tPyObject *item;\n"; - - # Bulk array size indication if !object - print PYOUT "\n"; - print PYOUT "\tif (!object)\n"; - print PYOUT "\t\treturn 0;\n"; - - print PYOUT "\n"; if ($p->{elem}->{class} eq "basic") { if ($p->{elem}->{type} eq "int64_t") { - print PYOUT "\titem = PyLong_FromLongLong(*object);\n"; + print PYOUT "\t\tuint64_t x = PyLong_AsLongLong(item);\n"; + print PYOUT "\t\trxrpc_enc(call, x >> 32);\n"; + print PYOUT "\t\trxrpc_enc(call, x);\n"; } elsif ($p->{elem}->{type} eq "uint64_t") { - print PYOUT "\titem = PyLong_FromUnsignedLongLong(*object);\n"; + print PYOUT "\t\tuint64_t x = PyLong_AsUnsignedLongLong(item);\n"; + print PYOUT "\t\trxrpc_enc(call, x >> 32);\n"; + print PYOUT "\t\trxrpc_enc(call, x);\n"; } elsif ($p->{elem}->{type} =~ /^int/) { - print PYOUT "\titem = PyLong_FromLong(*object);\n"; + print PYOUT "\t\trxrpc_enc(call, PyLong_AsLong(item));\n"; } elsif ($p->{elem}->{type} =~ /^uint|^char/) { - print PYOUT "\titem = PyLong_FromUnsignedLong(*object);\n"; + print PYOUT "\t\trxrpc_enc(call, PyLong_AsUnsignedLong(item));\n"; } } else { - print PYOUT "\titem = kafs_new_py_", $p->{type}, "(NULL, NULL);\n"; + print PYOUT "\t\trxgen_encode_", $p->{type}, "(call, &((struct py_", $p->{type}, " *)item)->x);\n"; } - print PYOUT "\tif (!item)\n"; - print PYOUT "\t\treturn -1;\n"; - - print PYOUT "\n"; - print PYOUT "\tif (PyList_Insert(list, index, item) < 0) {\n"; - print PYOUT "\t\tPy_DECREF(item);\n"; - print PYOUT "\t\treturn -1;\n"; print PYOUT "\t}\n"; - - if ($p->{elem}->{class} eq "struct") { - print PYOUT "\n"; - print PYOUT "\t*object = &((struct py_", $p->{type}, " *)item)->x;\n"; - } - print PYOUT "\treturn 0;\n"; print PYOUT "}\n"; } @@ -141,20 +239,26 @@ sub emit_py_func_simple_sync_call($) { my ($func) = @_; + push @py_func_defs, { + name => $func->{name}, + c_func => "kafs_" . $func->{name}, + doc => "", + }; + print PYOUT "\n"; print PYOUT "PyObject *\n"; print PYOUT "kafs_", $func->{name}, "(PyObject *_self, PyObject *args)\n"; print PYOUT "{\n"; # Local variable declarations representing parameters to send - my $need_tmp = 0; + print PYOUT "\tstruct rx_call *call;\n"; print PYOUT "\tstruct py_rx_connection *z_conn;\n"; - foreach my $p (@{$func->{params}}) { - if ($p->{class} eq "bulk" && $p->{elem}->{class} eq "string") { - die $p->{where}, ": String output args not supported" unless ($p->{dir} eq "IN"); - print PYOUT "\tconst char *param_", $p->{name}, ";\n"; + print PYOUT "\tstruct py_", $func->{name}, "_response *response;\n"; + foreach my $p (@{$func->{request}}) { + if ($p->{class} eq "bulk" && ($p->{elem}->{class} eq "string" || + $p->{elem}->{class} eq "opaque")) { + print PYOUT "\tPy_buffer param_", $p->{name}, ";\n"; } elsif ($p->{class} eq "basic") { - $need_tmp = 1 unless ($p->{dir} eq "IN"); print PYOUT "\t", $p->{type}, " param_", $p->{name}, ";\n"; } elsif ($p->{class} eq "struct") { die $p->{where}, ": INOUT struct args not supported" if ($p->{dir} eq "INOUT"); @@ -168,13 +272,6 @@ sub emit_py_func_simple_sync_call($) } } - # Replies are passed back into lists provided by the caller. INOUT variable - # input values must already occupy the lists. - foreach my $p (@{$func->{reply}}) { - print PYOUT "\tPyObject *reply_", $p->{name}, ";\n"; - } - - print PYOUT "\tPyObject *tmp = NULL;\n" if ($need_tmp); print PYOUT "\tPyObject *res = NULL;\n"; print PYOUT "\tint ret;\n"; @@ -183,7 +280,7 @@ sub emit_py_func_simple_sync_call($) print PYOUT "\n"; print PYOUT "\tif (!PyArg_ParseTuple(args, \"O!"; - foreach my $p (@{$func->{params}}) { + foreach my $p (@{$func->{request}}) { if ($p->{dir} ne "IN") { print PYOUT "O!"; } elsif ($p->{type} eq "int8_t") { print PYOUT "B"; } elsif ($p->{type} eq "int16_t") { print PYOUT "h"; @@ -195,7 +292,12 @@ sub emit_py_func_simple_sync_call($) } elsif ($p->{type} eq "uint64_t") { print PYOUT "K"; } elsif ($p->{class} eq "struct") { print PYOUT "O!"; } elsif ($p->{class} eq "bulk" && $p->{elem}->{class} eq "string") { - print PYOUT "s"; + print PYOUT "s*"; + } elsif ($p->{class} eq "bulk" && $p->{elem}->{class} eq "opaque") { + print PYOUT "z*"; + } elsif ($p->{class} eq "bulk" && ($p->{elem}->{class} eq "basic" || + $p->{elem}->{class} eq "struct")) { + print PYOUT "O!"; } else { die $p->{where}, ": No py parse for param"; } @@ -204,14 +306,13 @@ sub emit_py_func_simple_sync_call($) print PYOUT "\",\n"; print PYOUT "\t\t\t &py_rx_connectionType, &z_conn"; - foreach my $p (@{$func->{params}}) { + foreach my $p (@{$func->{request}}) { print PYOUT ",\n"; - print PYOUT "\t\t\t /*", $p->{dir}, "*/ "; - if ($p->{dir} ne "IN") { - print PYOUT "&PyList_Type, &reply_", $p->{name}; - } elsif ($p->{class} eq "bulk" && ($p->{elem}->{class} eq "string" || - $p->{elem}->{class} eq "opaque") || - $p->{class} eq "basic") { + print PYOUT "\t\t\t "; + if ($p->{class} eq "bulk" && ($p->{elem}->{class} eq "string" || + $p->{elem}->{class} eq "opaque")) { + print PYOUT "¶m_", $p->{name}; + } elsif ($p->{class} eq "basic") { print PYOUT "¶m_", $p->{name}; } elsif ($p->{class} eq "struct") { print PYOUT "&py_", $p->{type}, "Type, ¶m_", $p->{name}; @@ -224,123 +325,326 @@ sub emit_py_func_simple_sync_call($) print PYOUT "))\n"; print PYOUT "\t\treturn NULL;\n"; - # Allocate reply buffer objects - if (@{$func->{reply}}) { - print PYOUT "\n"; - foreach my $p (@{$func->{reply}}) { - if ($p->{class} eq "struct") { - print PYOUT "\tparam_", $p->{name}, " = (struct py_", $p->{type}, " *)kafs_new_py_", $p->{type}, "(NULL, NULL);\n"; - print PYOUT "\tif (!param_", $p->{name}, ")\n"; - print PYOUT "\t\tgoto error_alloc_", $p->{name}, ";\n"; - } + print PYOUT "\n"; + print PYOUT "\tcall = rxrpc_alloc_call(z_conn->x, 0);\n"; + print PYOUT "\tif (!call)\n"; + print PYOUT "\t\treturn PyExc_MemoryError;\n"; + + # Marshal the arguments + print PYOUT "\n"; + print PYOUT "\trxrpc_enc(call, ", $func->{opcode}, ");\n"; + foreach my $p (@{$func->{request}}) { + if ($p->{class} eq "bulk" && ($p->{elem}->{class} eq "basic" || + $p->{elem}->{class} eq "struct")) { + print PYOUT "\tif (py_encode_bulk_", $p->{type}, "(call, param_", $p->{name}, ") < 0)\n"; + print PYOUT "\t\tgoto error;\n"; + } elsif ($p->{class} eq "bulk" && ($p->{elem}->{class} eq "string" || + $p->{elem}->{class} eq "opaque")) { + print PYOUT "\tif (py_enc_buffer(call, ¶m_", $p->{name}, ") < 0) {\n"; + print PYOUT "\t\trxrpc_terminate_call(call, EINVAL);\n"; + print PYOUT "\t\treturn NULL;\n"; + print PYOUT "\t}\n"; + } elsif ($p->{class} eq "basic" && $p->{xdr_size} == 4) { + print PYOUT "\trxrpc_enc(call, param_", $p->{name}, ");\n"; + } elsif ($p->{class} eq "basic" && $p->{xdr_size} == 8) { + print PYOUT "\trxrpc_enc(call, param_", $p->{name}, " >> 32);\n"; + print PYOUT "\trxrpc_enc(call, param_", $p->{name}, ");\n"; + } elsif ($p->{class} eq "struct") { + print PYOUT "\tif (py_premarshal_", $p->{type}, "((PyObject *)param_", $p->{name}, ")) {\n"; + print PYOUT "\t\trxrpc_terminate_call(call, EINVAL);\n"; + print PYOUT "\t\treturn NULL;\n"; + print PYOUT "\t}\n"; + print PYOUT "\trxgen_encode_", $p->{type}, "(call, ¶m_", $p->{name}, "->x);\n"; + } else { + die $p->{where}, ": Unsupported type in decode"; } } + print PYOUT "\tif (rxrpc_post_enc(call) < 0)\n"; + print PYOUT "\t\tgoto error_no_res;\n"; + print PYOUT "\tcall->more = 0;\n"; + + # Allocate a reply object + print PYOUT "\n"; + print PYOUT "\tres = _PyObject_New(&py_", $func->{name}, "_responseType);\n"; + print PYOUT "\tresponse = (struct py_", $func->{name}, "_response *)res;\n"; + print PYOUT "\tif (!response)\n"; + print PYOUT "\t\tgoto enomem;\n"; + print PYOUT "\tmemset(&response->x, 0, sizeof(response->x));\n" + if (@{$func->{response}}); + print PYOUT "\tcall->decoder = py_", $func->{name}, "_decode_response;\n"; + print PYOUT "\tcall->decoder_private = response;\n"; + # Make the call print PYOUT "\n"; - print PYOUT "\tret = ", $func->{name}, "(\n"; - print PYOUT "\t\tz_conn->x"; + print PYOUT "\tret = rxrpc_send_data(call);\n"; + print PYOUT "\tif (ret == -1)\n"; + print PYOUT "\t\tgoto error;\n"; - foreach my $p (@{$func->{params}}) { - print PYOUT ",\n"; - if ($p->{class} eq "bulk" && $p->{elem}->{class} eq "string" || - $p->{class} eq "basic") { - if ($p->{dir} eq "IN") { - print PYOUT "\t\tparam_", $p->{name}; - } else { - print PYOUT "\t\t¶m_", $p->{name}; - } - } elsif ($p->{class} eq "struct") { - print PYOUT "\t\t¶m_", $p->{name}, "->x"; + # Wait for the reply + # + # If we're dealing with a split function or are in asynchronous mode, we + # need to return the call here. + # + print PYOUT "\n"; + print PYOUT "\tret = rxrpc_run_sync_call(call);\n"; + print PYOUT "\tif (ret == -1)\n"; + print PYOUT "\t\tgoto error;\n"; + + # Successful return + print PYOUT "\n"; + print PYOUT "\trxrpc_terminate_call(call, 0);\n"; + print PYOUT "\treturn res;\n"; + + # Error cleanups + print PYOUT "\n"; + print PYOUT "error:\n"; + print PYOUT "\tPy_XDECREF(res);\n"; + print PYOUT "error_no_res:\n"; + print PYOUT "\tif (errno == ENOMEM)\n"; + print PYOUT "enomem:\n"; + print PYOUT "\t\tres = PyExc_MemoryError;\n"; + print PYOUT "\telse if (errno == ECONNABORTED)\n"; + print PYOUT "\t\tres = PyErr_Format(kafs_remote_abort, \"Aborted %u\", call->abort_code);\n"; + #print PYOUT "\t\tres = PyLong_FromLong(call->abort_code);\n"; + print PYOUT "\telse\n"; + print PYOUT "\t\tres = PyErr_SetFromErrno(PyExc_IOError);\n"; + print PYOUT "\trxrpc_terminate_call(call, ENOMEM);\n"; + print PYOUT "\treturn res;\n"; + + # End the function + print PYOUT "}\n"; +} + +############################################################################### +# +# Emit a function to decode a block into a python object in a way that can be +# used from asynchronous code. The opcode is expected to have been removed +# from the incoming call on the server side. +# +############################################################################### +sub emit_py_func_decode($$$$) +{ + my ($func, $side, $subname, $paramlist) = @_; + my @params = @{$paramlist}; + my $ptr; + + $ptr = "obj->"; + + # We fetch the data in a number of phases. Each phase receives a chunk of + # data of a certain size. A phase's size might be dependent on a variable + # in the previous phase. Variable-sized bulk arrays are split across + # multiple phases, with the length being at the end of the first phase and + # the data in the second. + my @phases = (); + my $phase = 0; + my $have_bulk = 0; + my $want_item = 0; + + foreach my $p (@params) { + unless ($phase) { + $phase = { type => "flat", size => 0, params => [] }; + push @phases, $phase; + } + + if ($p->{class} eq "basic" || + $p->{class} eq "struct" + ) { + $phase->{size} += $p->{xdr_size}; + push @{$phase->{params}}, $p; } elsif ($p->{class} eq "bulk") { - if ($p->{dir} eq "IN") { - print PYOUT "\t\tpy_get__", $p->{type}, ", param_", $p->{name}; - print PYOUT ", PyList_Size(param_", $p->{name}, ")"; - } elsif ($p->{elem}->{class} eq "basic") { - print PYOUT "\t\tpy_store__", $p->{type}, ", reply_", $p->{name}; - } elsif ($p->{elem}->{class} eq "struct") { - print PYOUT "\t\tpy_alloc__", $p->{type}, ", reply_", $p->{name}; + $have_bulk = 1; + + # Bulk objects begin with an element count + $phase->{elem_count} = $phase->{size}; + $phase->{size} += 4; + + my %pseudoparam = ( + class => "basic", + type => "bulk_size", + name => $p->{name}, + elem => $p->{elem}, + where => $p->{where}, + xdr_size => $p->{xdr_size}, + ); + push @{$phase->{params}}, \%pseudoparam; + + # Create a new phase + $phase = { + type => "bulk", + name => $p->{name}, + params => [ $p ], + xdr_size => $p->{xdr_size}, + }; + push @phases, $phase; + + if ($p->{elem}->{class} eq "string" || + $p->{elem}->{class} eq "opaque") { + ; } else { - die; + $want_item = 1; } + + # We don't want to be asking recvmsg() for one object at a time if + # they're really small. + $phase->{size} = $p->{xdr_size}; + $phase = 0; } else { - die $p->{where}, ": Unsupported type \"", $p->{type}, "\""; + die $p->{where}, "Reply array not supported"; } } - print PYOUT ");\n"; - print PYOUT "\tif (ret != 0) {\n"; - print PYOUT "\t\tif (ret == -1 && errno == ENOMEM)\n"; - print PYOUT "\t\t\tres = PyExc_MemoryError;\n"; - print PYOUT "\t\telse if (ret == -1)\n"; - print PYOUT "\t\t\tres = PyErr_SetFromErrno(PyExc_IOError);\n"; - print PYOUT "\t\telse\n"; - print PYOUT "\t\t\tres = PyLong_FromLong(ret);\n"; - print PYOUT "\t\tgoto error;\n"; - print PYOUT "\t}\n"; - # Pass back any replies - if (@{$func->{reply}}) { + # Function definition and arguments + print PYOUT "\n"; + print PYOUT "int py_", $func->{name}, "_decode_", $subname, "(struct rx_call *call)\n"; + print PYOUT "{\n"; + + unless (@params) { + print PYOUT "\treturn 0;\n"; + print PYOUT "}\n"; + return; + } + + # Local variables + print PYOUT "\tstruct py_", $func->{name}, "_", $subname, " *obj = call->decoder_private;\n"; + print PYOUT "\tPyObject *item;\n" if ($want_item); + print PYOUT "\tunsigned phase = call->phase;\n"; + print PYOUT "\tunsigned count;\n"; + + # Deal with each phase + print PYOUT "\n"; + print PYOUT "select_phase:\n" if ($have_bulk); + print PYOUT "\tcount = call->data_count;\n"; + print PYOUT "\tswitch (phase) {\n"; + + print PYOUT "\tcase 0:\n"; + + my $phase_goto_label = 0; + my $phix; + for ($phix = 1; $phix <= $#phases + 1; $phix++) { print PYOUT "\n"; - foreach my $p (@{$func->{reply}}) { - my $set_null = 0; - my $var = "tmp"; + print PYOUT "\t\t/* --- Phase ", $phix, " --- */\n"; + $phase = $phases[$phix - 1]; + if ($phase_goto_label == $phix) { + print PYOUT "\tphase_", $phix, ":\n"; + $phase_goto_label = 0; + } - if ($p->{class} eq "basic") { - if ($p->{type} eq "int64_t") { - print PYOUT "\ttmp = PyLong_FromLongLong(param_", $p->{name}, ");\n"; - } elsif ($p->{type} eq "uint64_t") { - print PYOUT "\ttmp = PyLong_FromUnsignedLongLong(param_", $p->{name}, ");\n"; - } elsif ($p->{type} !~ /^u/) { - print PYOUT "\ttmp = PyLong_FromLong(param_", $p->{name}, ");\n"; + # Determine how big bulk objects are + if ($phase->{type} eq "bulk") { + my $p = $phase->{params}->[0]; + if ($p->{elem}->{class} eq "basic" || + $p->{elem}->{class} eq "struct") { + print PYOUT "\t\tobj->x.", $p->{name}, " = PyList_New(call->bulk_count);\n"; + print PYOUT "\t\tif (!obj->x.", $p->{name}, ")\n"; + print PYOUT "\t\t\treturn -1;\n"; + } elsif ($p->{elem}->{class} eq "string") { + print PYOUT "\t\tobj->x.", $p->{name}, " = PyUnicode_New(call->bulk_count, 255);\n"; + print PYOUT "\t\tif (!obj->x.", $p->{name}, ")\n"; + print PYOUT "\t\t\treturn -1;\n"; + print PYOUT "\t\tcall->bulk_item = obj->x.", $p->{name}, ";\n"; + } elsif ($p->{elem}->{class} eq "opaque") { + print PYOUT "\t\tobj->x.", $p->{name}, " = PyByteArray_FromStringAndSize(\"\", 0);\n"; + print PYOUT "\t\tif (!obj->x.", $p->{name}, ")\n"; + print PYOUT "\t\t\treturn -1;\n"; + print PYOUT "\t\tif (PyByteArray_Resize(obj->x.", $p->{name}, ", call->bulk_count) == -1)\n"; + print PYOUT "\t\t\treturn -1;\n"; + } else { + die; + } + + print PYOUT "\t\tif (call->bulk_count == 0)\n"; + print PYOUT "\t\t\tgoto phase_", $phix + 1, ";\n"; + $phase_goto_label = $phix + 1; + print PYOUT "\t\tcall->bulk_index = 0;\n"; + } + + # Entry point for a phase + print PYOUT "\t\tcall->need_size = ", $phase->{size}, ";\n"; + print PYOUT "\t\tcall->phase = ", $phix, ";\n"; + print PYOUT "\tcase ", $phix, ":\n"; + + print PYOUT "\t\tif (count < ", $phase->{size}, ")\n"; + print PYOUT "\t\t\treturn 1;\n"; + + # Unmarshal the data + print PYOUT "\n"; + foreach my $p (@{$phase->{params}}) { + if ($p->{type} eq "bulk_size") { + print PYOUT "\t\tcall->bulk_count = rxrpc_dec(call);\n"; + next; + } + + if ($p->{class} eq "bulk" && ($p->{elem}->{class} eq "basic" || + $p->{elem}->{class} eq "struct") + ) { + if ($p->{elem}->{class} eq "struct") { + print PYOUT "\t\titem = py_decode_", $p->{type}, "(call);\n"; + } elsif ($p->{elem}->{xdr_size} == 4 && $p->{type} =~ /^u/) { + print PYOUT "\t\titem = PyLong_FromUnsignedLong((", $p->{type}, ")rxrpc_dec(call));\n"; + } elsif ($p->{elem}->{xdr_size} == 4) { + print PYOUT "\t\titem = PyLong_FromLong((", $p->{type}, ")rxrpc_dec(call));\n"; + } elsif ($p->{elem}->{xdr_size} == 8 && $p->{type} =~ /^u/) { + print PYOUT "\t\tcall->bulk_u64 = (uint64_t)rxrpc_dec(call) << 32;\n"; + print PYOUT "\t\tcall->bulk_u64 |= (uint64_t)rxrpc_dec(call);\n"; + print PYOUT "\t\titem = PyLong_FromUnsignedLongLong(call->bulk_u64);\n"; + } elsif ($p->{elem}->{xdr_size} == 8) { + print PYOUT "\t\tcall->bulk_s64 = (int64_t)rxrpc_dec(call) << 32;\n"; + print PYOUT "\t\tcall->bulk_s64 |= (int64_t)rxrpc_dec(call);\n"; + print PYOUT "\t\titem = PyLong_FromLongLong(call->bulk_s64);\n"; } else { - print PYOUT "\ttmp = PyLong_FromUnsignedLong(param_", $p->{name}, ");\n"; + die; } + print PYOUT "\t\tif (!item)\n"; + print PYOUT "\t\t\treturn -1;\n"; + print PYOUT "\t\tif (PyList_SetItem(obj->x.", $p->{name}, ", call->bulk_index, item) < 0)\n"; + print PYOUT "\t\t\treturn -1;\n"; + print PYOUT "\t\tcall->bulk_index++;\n"; + + } elsif ($p->{class} eq "bulk" && $p->{elem}->{class} eq "string") { + print PYOUT "\t\tpy_dec_string(call);\n"; + } elsif ($p->{class} eq "bulk" && $p->{elem}->{class} eq "opaque") { + print PYOUT "\t\tpy_dec_opaque(call, obj->x.", $p->{name}, ");\n"; + } elsif ($p->{class} eq "basic" && $p->{xdr_size} == 4) { + print PYOUT "\t\tobj->x.", $p->{name}, " = (", $p->{type}, ")rxrpc_dec(call);\n"; + } elsif ($p->{class} eq "basic" && $p->{xdr_size} == 8) { + print PYOUT "\t\tobj->x.", $p->{name}, " = (", $p->{type}, ")rxrpc_dec(call) << 32;\n"; + print PYOUT "\t\tobj->x.", $p->{name}, " |= (", $p->{type}, ")rxrpc_dec(call) << 32;\n"; } elsif ($p->{class} eq "struct") { - $var = "(PyObject *)param_" . $p->{name}; - $set_null = 1; - } elsif ($p->{class} eq "bulk") { - # All done in the helper func - next; + print PYOUT "\t\tobj->x.", $p->{name}, " = py_decode_", $p->{type}, "(call);\n"; } else { - die $p->{where}, ": Unsupported class \"", $p->{class}, "\""; + die $p->{where}, ": Unsupported type in decode"; } - if ($var eq "tmp") { - print PYOUT "\tif (!tmp)\n"; - print PYOUT "\t\tgoto error;\n"; + if ($p->{class} eq "bulk") { + print PYOUT "\t\tif (rxrpc_post_dec(call) < 0)\n"; + print PYOUT "\t\t\treturn -1;\n"; + print PYOUT "\t\tif (call->bulk_index < call->bulk_count) {\n"; + print PYOUT "\t\t\tphase = ", $phix, ";\n"; + print PYOUT "\t\t\tgoto select_phase;\n"; + print PYOUT "\t\t}\n"; } - - print PYOUT "\tif (PyList_Insert(reply_", $p->{name}, ", 0, ", $var, ") == -1)\n"; - print PYOUT "\t\tgoto error", $need_tmp ? "_tmp" : "", ";\n"; - print PYOUT "\tparam_", $p->{name}, " = NULL;\n" if ($set_null); } + if ($phase->{type} ne "bulk") { + print PYOUT "\t\tif (rxrpc_post_dec(call) < 0)\n"; + print PYOUT "\t\t\treturn -1;\n"; + } } - # Successful return print PYOUT "\n"; - print PYOUT "\treturn PyLong_FromLong(0);\n"; - - # Error cleanups - print PYOUT "\n"; - if ($need_tmp) { - print PYOUT "error_tmp:\n"; - print PYOUT "\tPy_DECREF(tmp);\n"; - } - print PYOUT "error:\n"; - if (@{$func->{reply}}) { - foreach my $p (reverse @{$func->{reply}}) { - if ($p->{class} eq "struct") { - print PYOUT "\tPy_XDECREF(param_", $p->{name}, ");\n"; - print PYOUT "error_alloc_", $p->{name}, ":\n"; - } - } + print PYOUT "\t\t/* --- Phase ", $phix, " --- */\n"; + if ($phase_goto_label == $phix) { + print PYOUT "\tphase_", $phix, ":\n"; + $phase_goto_label = 0; } + print PYOUT "\t\tcall->phase = ", $phix, ";\n"; + print PYOUT "\t\tcall->need_size = 0;\n"; + print PYOUT "\tdefault:\n"; + print PYOUT "\t\treturn 0;\n"; + print PYOUT "\t}\n"; - print PYOUT "\treturn res;\n"; - - # End the function print PYOUT "}\n"; } diff --git a/rxgen/emit_py_types.pm b/rxgen/emit_py_types.pm index 3be828a..3a6e0cf 100644 --- a/rxgen/emit_py_types.pm +++ b/rxgen/emit_py_types.pm @@ -27,11 +27,10 @@ sub emit_py_type_wrapper_decls($) { sub emit_py_type_wrapper($) { my ($struct) = @_; - # Dump the banner comment block - print PYHDR "\n"; - print PYHDR @{$struct->{banner}}; - print PYOUT "\n"; - print PYOUT @{$struct->{banner}}; + push @py_type_defs, { + name => $struct->{type}, + c_type => "py_" . $struct->{type} . "Type", + }; # Divide the struct members into single ints, single structs, char arrays # (strings) and other arrays @@ -49,7 +48,7 @@ sub emit_py_type_wrapper($) { } elsif ($m->{class} eq "array") { push @arrays, $m; } else { - die; + die $m->{where}, ": Unsupported struct member type"; } } @@ -73,6 +72,15 @@ sub emit_py_type_wrapper($) { } print PYHDR "};\n"; + # We need to have a new function if the object is to be allocatable by the + # Python interpreter + print PYOUT "\n"; + print PYOUT "static PyObject *\n"; + print PYOUT "py_", $struct->{type}, "_new(PyTypeObject *subtype, PyObject *args, PyObject *kwds)\n"; + print PYOUT "{\n"; + print PYOUT "\treturn subtype->tp_alloc(subtype, 1);\n;"; + print PYOUT "}\n"; + # We have to have a deallocation function print PYOUT "\n"; print PYOUT "static void\n"; @@ -86,7 +94,7 @@ sub emit_py_type_wrapper($) { # Any integer non-array elements are made directly accessible to the Python # interpreter - if ($#single_ints + $#char_arrays > -2) { + if (@single_ints) { print PYOUT "\n"; print PYOUT "static PyMemberDef py_", $struct->{type}, "_members[] = {\n"; foreach my $m (@single_ints) { @@ -150,6 +158,15 @@ sub emit_py_type_wrapper($) { } elsif ($m->{elem}->{type} eq "uint32_t") { print PYOUT "\t\t\treturn py_rxgen_get_uint32(&self->x.", $m->{name}, ", ", $m->{dim}, ",\n"; print PYOUT "\t\t\t\t\t\t &self->c.", $m->{name}, ");\n"; + } elsif ($m->{elem}->{type} eq "int8_t") { + print PYOUT "\t\t\treturn py_rxgen_get_int8(&self->x.", $m->{name}, ", ", $m->{dim}, ",\n"; + print PYOUT "\t\t\t\t\t\t &self->c.", $m->{name}, ");\n"; + } elsif ($m->{elem}->{type} eq "int16_t") { + print PYOUT "\t\t\treturn py_rxgen_get_int16(&self->x.", $m->{name}, ", ", $m->{dim}, ",\n"; + print PYOUT "\t\t\t\t\t\t &self->c.", $m->{name}, ");\n"; + } elsif ($m->{elem}->{type} eq "int32_t") { + print PYOUT "\t\t\treturn py_rxgen_get_int32(&self->x.", $m->{name}, ", ", $m->{dim}, ",\n"; + print PYOUT "\t\t\t\t\t\t &self->c.", $m->{name}, ");\n"; } else { die $m->{where}, ": Unsupported array type \"", $m->{elem}->{type}, "\""; } @@ -254,7 +271,7 @@ sub emit_py_type_wrapper($) { print PYOUT "\t0,\t\t\t\t/* tp_dictoffset */\n"; print PYOUT "\t0,\t\t\t\t/* tp_init */\n"; print PYOUT "\t0,\t\t\t\t/* tp_alloc */\n"; - print PYOUT "\t0,\t\t\t\t/* tp_new */\n"; + print PYOUT "\tpy_", $struct->{type}, "_new,\n"; print PYOUT "};\n"; # Emit a function to allocate such a type @@ -279,7 +296,8 @@ sub emit_py_type_wrapper($) { print PYHDR "extern PyObject *py_data_to_", $struct->{type}, "(const void *);\n"; print PYOUT "\n"; - print PYOUT "PyObject *py_data_to_", $struct->{type}, "(const void *data) {\n"; + print PYOUT "PyObject *py_data_to_", $struct->{type}, "(const void *data)\n"; + print PYOUT "{\n"; print PYOUT "\tPyObject *obj = _PyObject_New(&py_", $struct->{type}, "Type);\n"; print PYOUT "\tstruct py_", $struct->{type}, " *self = (struct py_", $struct->{type}, " *)obj;\n"; print PYOUT "\tif (!obj)\n"; @@ -291,6 +309,21 @@ sub emit_py_type_wrapper($) { print PYOUT "\treturn obj;\n"; print PYOUT "}\n"; + # Emit a function to unmarshal on object of this type. + print PYOUT "\n"; + print PYOUT "PyObject *py_decode_", $struct->{type}, "(struct rx_call *call)\n"; + print PYOUT "{\n"; + print PYOUT "\tPyObject *obj = _PyObject_New(&py_", $struct->{type}, "Type);\n"; + print PYOUT "\tstruct py_", $struct->{type}, " *self = (struct py_", $struct->{type}, " *)obj;\n"; + print PYOUT "\tif (!obj)\n"; + print PYOUT "\t\treturn PyExc_MemoryError;\n"; + print PYOUT "\trxgen_decode_", $struct->{type}, "(call, &self->x);\n"; + if ($#single_structs + $#arrays > -2) { + print PYOUT "\tmemset(&self->c, 0, sizeof(self->c));\n"; + } + print PYOUT "\treturn obj;\n"; + print PYOUT "}\n"; + # Emit a function to premarshal such a type. This folds the contents of # the cached Python objects into their raw fields. # @@ -342,6 +375,12 @@ sub emit_py_type_wrapper($) { print PYOUT "py_rxgen_premarshal_uint16(&self->x.", $m->{name}, ", ", $m->{dim}, ", self->c.", $m->{name}, ") < 0"; } elsif ($m->{elem}->{type} eq "uint32_t") { print PYOUT "py_rxgen_premarshal_uint32(&self->x.", $m->{name}, ", ", $m->{dim}, ", self->c.", $m->{name}, ") < 0"; + } elsif ($m->{elem}->{type} eq "int8_t") { + print PYOUT "py_rxgen_premarshal_int8(&self->x.", $m->{name}, ", ", $m->{dim}, ", self->c.", $m->{name}, ") < 0"; + } elsif ($m->{elem}->{type} eq "int16_t") { + print PYOUT "py_rxgen_premarshal_int16(&self->x.", $m->{name}, ", ", $m->{dim}, ", self->c.", $m->{name}, ") < 0"; + } elsif ($m->{elem}->{type} eq "int32_t") { + print PYOUT "py_rxgen_premarshal_int32(&self->x.", $m->{name}, ", ", $m->{dim}, ", self->c.", $m->{name}, ") < 0"; } else { die $m->{where}, ": Unsupported array type \"", $m->{elem}->{type}, "\""; } @@ -351,6 +390,11 @@ sub emit_py_type_wrapper($) { print PYOUT "\t\treturn -1;\n"; print PYOUT "\treturn 0;\n"; print PYOUT "}\n"; + } else { + print PYHDR "static inline int py_premarshal_", $struct->{type}, "(PyObject *_self)\n"; + print PYHDR "{\n"; + print PYHDR "\treturn 0;\n"; + print PYHDR "}\n"; } } diff --git a/rxgen/rxgen.pl b/rxgen/rxgen.pl index ae58c8d..1c33221 100755 --- a/rxgen/rxgen.pl +++ b/rxgen/rxgen.pl @@ -31,18 +31,20 @@ our %struct_sizes = (); # Structure sizes our @funcs = (); # Functions in declaration order our %func_names = (); # Function name uniquifier our %constants = (); # Constants +our %packages = (); # Packages our @abort_codes = (); # Abort codes +our @py_type_defs = (); # Python type definitions +our @py_func_defs = (); # Python function definitions # # Divide the lines from the files up into typed collections # -my @comment = (); -my $pkg = ""; -my $banner = 0; +my $pkg = 0; my $struct = 0; my $func = 0; my $cpp_exclude = 0; my $error_codes = 0; +my $comment_discard = 0; my @files = @ARGV; my $file = ""; @@ -82,8 +84,8 @@ our %types = ( "uint16_t" => { class => "basic", type => "uint16_t", xdr_size => 4, multi => 0, }, "uint32_t" => { class => "basic", type => "uint32_t", xdr_size => 4, multi => 0, }, "uint64_t" => { class => "basic", type => "uint64_t", xdr_size => 8, multi => 0, }, - "string" => { class => "string", type => "char", xdr_size => 1, multi => 0, }, - "opaque" => { class => "opaque", type => "void", xdr_size => 1, multi => 0, }, + "string" => { class => "string", type => "char", xdr_size => 4, multi => 0, }, + "opaque" => { class => "opaque", type => "void", xdr_size => 4, multi => 0, }, ); sub look_up_type($) @@ -110,8 +112,9 @@ sub define_typedef($$$) my %combined = %{$type}; - if ($flags->{class} eq "bulk" || - $flags->{class} eq "array") { + if (exists $flags->{class} && + ($flags->{class} eq "bulk" || + $flags->{class} eq "array")) { die $where, ": Typedef'ing array/bulk as array/bulk not supported\n" if ($type->{multi}); $combined{multi} = 1; @@ -121,6 +124,8 @@ sub define_typedef($$$) $combined{xdr_size} *= $combined{dim} if ($flags->{class} eq "array"); } + die if (exists $combined{dim} && $combined{dim} eq -1); + define_type($new_type, \%combined); } @@ -134,6 +139,7 @@ sub parse_xg($) { $file = $filename; open my $APIHDR, "<$filename" || die $filename; while (my $line = <$APIHDR>) { + my $pre_comment = ""; $where = $file . ':' . $. ; # Detect #if 0/#endif pairs to exclude parts @@ -151,70 +157,100 @@ sub parse_xg($) { next if $cpp_exclude; - # Gather comments for later attachment to subsequent structs and funcs - if ($line =~ m@^/[*]\s*$@) { - die $where, ": Embedded comment\n" if $banner; - $banner = 1; - @comment = ( $line ); + chomp($line); + + # Extract error codes + if ($line eq "/* Error codes */") { + $error_codes = 1; next; } - if ($banner) { - die $where, ": Commentless terminator\n" if ($line !~ /^ [*]/); - push @comment, $line; - $banner = 0 if ($line =~ m!^ [*]/!); - next; + + $error_codes = 0 if ($line eq ""); + + # Discard comments +find_comment_terminator: + if ($comment_discard) { + # Find the terminator for a comment we're discarding + if ($line =~ m@.*[*]/(.*)@) { + $line = $pre_comment . $1; + $comment_discard = 0; + } else { + $line = $pre_comment; + goto discarded_comments if ($line); + next; + } } - chomp($line); + if ($line =~ m@(.*)/[*](.*)@) { + $pre_comment = $1; + $line = $2; + $comment_discard = 1; + goto find_comment_terminator; + } - @comment = () if ($line eq ""); +discarded_comments: + # Remove leading/trailing whitespace and distil interior whitespace + # down to a single space. Also remove whitespace next to symbols + # (excluding underscores) and remove blank lines. + $line =~ s/^\s+//; + $line =~ s/\s+$//; + $line =~ s/\s+/\t/g; # Convert all whitespace to single tabs as an intermediate step + # Convert any tab surrounded by two numbers/symbols into a space + $line =~ s!([a-zA-Z0-9_])\t([a-zA-Z0-9_])!$1 $2!g; + # Discard any remaining tabs (have an adjacent symbol) + $line =~ s!\t!!g; + next if (!$line); + + #print "'$line'\n"; # Complain about #defines die $where, ": Use const not #define" if ($line =~ /^#define/); # Extract package prefix if ($line =~ /^package\s+([A-Za-z_][A-Za-z0-9_]*)/) { - $pkg = $1; - next; - } - - # Extract error codes - if ($line eq "/* Error codes */") { - $error_codes = 1; + my $prefix = $1; + my $name = $prefix; + $name =~ s/_$//; + $pkg = { + name => $name, + prefix => $prefix, + }; + $packages{$prefix} = $pkg; next; } - $error_codes = 0 if ($line eq ""); - ####################################################################### # Extract constants # - if ($line =~ /^const\s+([A-Za-z0-9_]+)\s*=\s*(.*);/) { + if ($line =~ /^const ([A-Za-z0-9_]+)=(.*);/) { my $c = $1; my $v = $2; - die $where, ": Duplicate constant $c" if (exists $constants{$c}); - $v =~ s/^\s+//; - $v =~ s/\s+$//; - $constants{$c} = $v; + die $where, ": Duplicate constant $c (original at ", $constants{$c}->{where}, ": )" + if (exists $constants{$c}); + $v =~ s/^ //; + $v =~ s/ $//; + $constants{$c} = { name => $c, + val => $v, + where => $where, + }; push @abort_codes, $c if $error_codes; - @comment = (); next; } ####################################################################### # Extract typedefs # - if ($line =~ /^typedef\s+([a-zA-Z_][a-zA-Z0-9_]*)\s+([a-zA-Z_][a-zA-Z0-9_]*)\s*;/) { + if ($line =~ /^typedef ([a-zA-Z_][a-zA-Z0-9_]*) ([a-zA-Z_][a-zA-Z0-9_]*);/) { define_typedef($2, $1, { }); next; } - if ($line =~ /^typedef\s+([a-zA-Z_][a-zA-Z0-9_]*)\s+([a-zA-Z_][a-zA-Z0-9_]*)\s*<\s*>\s*;/) { + if ($line =~ /^typedef ([a-zA-Z_][a-zA-Z0-9_]*) ([a-zA-Z_][a-zA-Z0-9_]*)<>;/) { define_typedef($2, $1, { class => "bulk" }); next; } - if ($line =~ /^typedef\s+([a-zA-Z_][a-zA-Z0-9_]*)\s+([a-zA-Z_][a-zA-Z0-9_]*)\s*<\s*([a-zA-Z0-9_]+)\s*>\s*;/) { + if ($line =~ /^typedef ([a-zA-Z_][a-zA-Z0-9_]*) ([a-zA-Z_][a-zA-Z0-9_]*)<([a-zA-Z0-9_]+)>;/) { define_typedef($2, $1, { class => "bulk", dim => $3 }); next; } @@ -222,18 +258,16 @@ sub parse_xg($) { ####################################################################### # Extract structures # - if ($line =~ /^struct\s+([a-zA-Z_][a-zA-Z0-9_]*) {/) { + if ($line =~ /^struct ([a-zA-Z_][a-zA-Z0-9_]*){/) { my %type = ( class => "struct", type => $1, members => [], - banner => \@comment, xdr_size => 0, ); define_type($1, \%type); push @structs, \%type; $struct = \%type; - @comment = (); next; } @@ -244,10 +278,7 @@ sub parse_xg($) { # Extract structure members if ($struct) { - # Strip trailing comments - $line =~ s@\s*/[*][^*]*[*]/$@@; - - if ($line =~ /([a-zA-Z_][a-zA-Z0-9_]*)\s+([a-zA-Z_][a-zA-Z0-9_]*)\s*;/) { + if ($line =~ /([a-zA-Z_][a-zA-Z0-9_]*) ([a-zA-Z_][a-zA-Z0-9_]*);/) { my %member = %{look_up_type($1)}; die $where, ": Don't support bulk constructs in structs\n" if ($member{class} eq "bulk"); @@ -256,7 +287,7 @@ sub parse_xg($) { push $struct->{members}, \%member; $struct->{xdr_size} += $member{xdr_size}; #print "nonarray $2\n"; - } elsif ($line =~ /([a-zA-Z_][a-zA-Z0-9_]*)\s+([a-zA-Z_][a-zA-Z0-9_]*)\s*\[\s*([^]]+)\s*\]\s*;/) { + } elsif ($line =~ /([a-zA-Z_][a-zA-Z0-9_]*) ([a-zA-Z_][a-zA-Z0-9_]*)\[([^]]+)\];/) { my $element = look_up_type($1); die $where, ": Don't support arrays of bulk constructs or arrays\n" if ($element->{multi}); @@ -269,12 +300,14 @@ sub parse_xg($) { $member{where} = $where; if ($member{dim} =~ /^[0-9]+$/) { - $constants{$member{dim}} = $member{dim}; + $constants{$member{dim}} = { + val => $member{dim}, + }; } elsif (exists $constants{$member{dim}}) { } else { die $where, ": No constant for [", $member{dim}, "]\n" } - $member{xdr_size} = $constants{$member{dim}} * $element->{xdr_size}; + $member{xdr_size} = $constants{$member{dim}}->{val} * $element->{xdr_size}; push $struct->{members}, \%member; $struct->{xdr_size} += $member{xdr_size}; #print "array $2\n"; @@ -286,59 +319,63 @@ sub parse_xg($) { ####################################################################### # Extract functions # - if (!$func && $line =~ /^([a-zA-Z_][a-zA-Z0-9_]*)\s*\(\s*(.*)$/) { + if (!$func && $line =~ /^([a-zA-Z_][a-zA-Z0-9_]*)\((.*)$/) { #print "func $1\n"; + my $name = $1; + die $where, ": No package set" unless $pkg; + my $func_name = $pkg->{prefix} . $name; my %function = ( - name => $pkg . $1, - banner => \@comment, + pkg => $pkg, + rawname => $name, + name => $func_name, params => [], where => $where, + split => 0, + multi => 0, ); - die $where, ": Duplicate function name '$1'\n" - if (exists($func_names{$1})); - $func_names{$1} = \%function; + die $where, ": Duplicate function name '$func_name'\n" + if (exists($func_names{$func_name})); + $func_names{$func_name} = \%function; push @funcs, \%function; $func = \%function; - @comment = (); $line = $2; } # Extract function parameters if ($func) { + parse_param: my $dir = ""; my $term = 0; my $bulk_dim = 0; - $dir = $1 if ($line =~ s@^\s*(IN|OUT|INOUT)\s+@@); + # Split parameters that are on the same line and divide the last + # parameter from the function closure + my $clause = $line; + if ($line =~ /^([^,)]*),(.*)$/) { + $clause = $1; + $line = $2; + } elsif ($line =~ /^([^)]*)([)].*)$/) { + $clause = $1; + $line = $2; + } + + #print "CLAUSE: '", $clause, "'\n"; + + $dir = $1 if ($clause =~ s@^(IN|OUT|INOUT) @@); - if ($line =~ s@<\s*>@@) { + if ($clause =~ s@<>@@) { $bulk_dim = -1; - } elsif ($line =~ s@<\s*([0-9]+)\s*>@@) { + } elsif ($clause =~ s@<([0-9]+)>@@) { $bulk_dim = $1; - $constants{$bulk_dim} = $bulk_dim; - } elsif ($line =~ s@<\s*([a-zA-Z_][a-zA-Z0-9_]*)\s*>@@) { + $constants{$bulk_dim} = { + val => $bulk_dim, + }; + } elsif ($clause =~ s@<([a-zA-Z_][a-zA-Z0-9_]*)>@@) { die $where, ": No constant for $1\n" unless exists $constants{$1}; $bulk_dim = $1; } - # Strip trailing comments - $line =~ s@\s*/[*][^*]*[*]/$@@; - $line =~ s/\s+$//; - - if ($line =~ s/[)]\s*=\s*([a-zA-Z0-9_]*)\s*;$//) { - $term = 1; - $func->{opcode} = $1; - } elsif ($line =~ s/,$//) { - $term = 0; - } else { - die $where, ": Unexpected line termination '$line'"; - } - - $line =~ s/\s+$//; - - #print "\"", $line, "\"\n"; - - if ($line =~ /([a-zA-Z_][a-zA-Z0-9_]*)\s+([*]*)([a-zA-Z_][a-zA-Z0-9_]*)\s*/) { + if ($clause =~ /([a-zA-Z_][a-zA-Z0-9_]*)([*]*| )([a-zA-Z_][a-zA-Z0-9_]*)/) { die $where, ": No parameter direction specified\n" unless $dir; my $type = look_up_type($1); @@ -357,7 +394,7 @@ sub parse_xg($) { if ($param{class} eq "bulk"); $param{class} = "bulk"; $param{elem} = $type; - $param{dim} = $bulk_dim; + $param{dim} = $bulk_dim unless $bulk_dim eq -1; } $param{ptr} = "*" if ($type->{class} eq "string"); @@ -365,10 +402,29 @@ sub parse_xg($) { #print "- ", $1, " ", $param{name}, " ISA ", $param{class}, ".", $param{type}, " ", $param{dir}, "\n"; push $func->{params}, \%param; - } elsif ($line eq "") { - # Parameterless function + } elsif ($clause eq "") { + # No parameter here } else { - die $where, ": Unhandled RPC call parameter '$line'"; + die $where, ": Unhandled RPC call parameter '$clause'"; + } + + next unless ($line); + goto parse_param unless ($line =~ /^[)]/); + + # Parse the function termination + if ($line =~ s/[)]=([a-zA-Z0-9_]*);$//) { + $term = 1; + $func->{opcode} = $1; + } elsif ($line =~ s/[)]split=([a-zA-Z0-9_]*);$//) { + $func->{split} = 1; + $term = 1; + $func->{opcode} = $1; + } elsif ($line =~ s/[)]multi=([a-zA-Z0-9_]*);$//) { + $func->{multi} = 1; + $term = 1; + $func->{opcode} = $1; + } else { + die $where, ": Unexpected line termination '$line'"; } if ($term) { @@ -379,6 +435,7 @@ sub parse_xg($) { } close($APIHDR); + $pkg = 0; } foreach my $file (@files) { @@ -419,6 +476,7 @@ open PYHDR, ">afs_py.h" || die "afs_py.h"; print PYHDR "/* AUTOGENERATED */\n"; print PYHDR "#include \n"; print PYHDR "#include \"afs_xg.h\"\n"; +print PYHDR "#include \"py_rxgen.h\"\n"; open PYOUT, ">afs_py.c" || die "afs_py.c"; print PYOUT "/* AUTOGENERATED */\n"; @@ -426,13 +484,12 @@ print PYOUT "#include \n"; print PYOUT "#include \"structmember.h\"\n"; print PYOUT "#include \"afs_py.h\"\n"; print PYOUT "#include \n"; -print PYOUT "#include \"py_rxgen.h\"\n"; print PYOUT "\n"; # Declare constants print RXHDR "\n"; foreach my $c (sort keys %constants) { - print RXHDR "#define $c ", $constants{$c}, "\n" unless ($c =~ /^[0-9]/) + print RXHDR "#define $c ", $constants{$c}->{val}, "\n" unless ($c =~ /^[0-9]/) } # Declare types @@ -454,94 +511,59 @@ foreach my $s (@structs) { ############################################################################### foreach $func (@funcs) { # Dump the banner comment block - print RXOUT "\n"; - if (@{$func->{banner}}) { - print RXOUT @{$func->{banner}}; - print PYOUT "\n"; - print PYOUT @{$func->{banner}}; - } else { - print RXOUT "/*\n"; - print RXOUT " * RPC Call ", $func->{name}, "\n"; - print RXOUT " */\n"; - } + print RXOUT "/*\n"; + print RXOUT " * RPC Call ", $func->{name}, "\n"; + print RXOUT " */\n"; # Find the Operation ID die "Operation ID unspecified for ", $func->{name}, "\n" unless exists $func->{opcode}; - # Filter the parameters into request and reply + # Filter the parameters into request and response my @request = (); - my @reply = (); - my $request_size = 4; - my $reply_size = 0; - my $req_has_charptr = 0; - my $req_size_is_indefinite = 0; - my $rep_size_is_indefinite = 0; + my @response = (); foreach my $p (@{$func->{params}}) { #print RXOUT $dir, " ", $type, " ", $name, "\n"; - my $buffer_size = 0; - my $indefinite = 0; if ($p->{class} eq "basic") { - $buffer_size = $p->{xdr_size}; - + ; } elsif ($p->{class} eq "struct") { - die unless (exists $p->{xdr_size}); - $buffer_size = $p->{xdr_size}; - + die unless (exists $p->{xdr_size}); } elsif ($p->{class} eq "bulk") { die $p->{where}, ": No element type" unless (exists $p->{elem}); die $p->{where}, ": Element has no XDR size" unless (exists $p->{elem}->{xdr_size}); if (exists $p->{dim} && $p->{elem}->{xdr_size} > 0) { - $buffer_size = $p->{elem}->{xdr_size}; - $buffer_size *= $constants{$p->{dim}}; - $buffer_size = (4 + $buffer_size + 3) & ~3; - $req_has_charptr = 1 if ($p->{elem}->{class} eq "string"); - } else { - $buffer_size = 4; - $indefinite = 1; + die $where, ": Missing constant ", $p->{dim} unless exists $constants{$p->{dim}}; } } else { - die $p->where, ": Unsupported param class \"", $p->{class}, "\""; + die $p->{where}, ": Unsupported param class \"", $p->{class}, "\""; } if ($p->{dir} eq "IN") { push @request, $p; - $request_size += $buffer_size; - $req_size_is_indefinite |= $indefinite; } elsif ($p->{dir} eq "OUT") { - push @reply, $p; - $reply_size += $buffer_size; - $rep_size_is_indefinite |= $indefinite; + push @response, $p; } elsif ($p->{dir} eq "INOUT") { - push @reply, $p; + push @response, $p; push @request, $p; - $request_size += $buffer_size; - $reply_size += $buffer_size; - $req_size_is_indefinite |= $indefinite; - $rep_size_is_indefinite |= $indefinite; } } $func->{request} = \@request; - $func->{request_size} = $request_size unless ($req_size_is_indefinite); - $func->{req_has_charptr} = $req_has_charptr; - $func->{reply} = \@reply; - $func->{reply_size} = $reply_size unless ($rep_size_is_indefinite); - - #print RXOUT "/* req max size: $request_size */\n"; - #print RXOUT "/* rep max size: $reply_size */\n"; - calc_func_prototype($func); - emit_func_send_request($func); - if (@reply) { - emit_func_decode_reply($func); - } else { - emit_func_dummy_decode_reply($func); - } - emit_func_alloc_call($func); - emit_func_simple_sync_call($func); + $func->{response} = \@response; + + emit_func_prototype($func); + emit_func_decode($func, "client", "response", \@response); + emit_func_send($func, "request"); + #emit_func_decode($func, "server", "request", \@request); + #emit_func_send($func, "response"); + + emit_py_func_param_object($func, "request"); + emit_py_func_param_object($func, "response"); emit_py_func_bulk_helper($func); + emit_py_func_decode($func, "client", "response", \@response); + emit_py_func_decode($func, "server", "request", \@request); emit_py_func_simple_sync_call($func); } diff --git a/vl-test.py b/vl-test.py index afa575f..3e32fd2 100755 --- a/vl-test.py +++ b/vl-test.py @@ -13,8 +13,8 @@ import sys; import getopt; -import kafs; import dns.resolver; +import kafs; cell = "grand.central.org"; volumes = [ "root.cell" ]; @@ -28,38 +28,33 @@ for o, a in opts: if args: volumes = args; -print("cell:", cell); +print("-- Find VL servers for cell:", cell, "--"); # Find a list of Volume Location servers to contact -vladdrs = []; +vladdrs = set(); try: - srv_list = dns.resolver.query("_afs3-vlserver._udp." + cell, "SRV"); - for record in srv_list: - A_recs = dns.resolver.query(record.target, 'A'); - for addr in A_recs: - vladdrs.append(addr.address); - print(record.target, ":", vladdrs); + for SRV in dns.resolver.query("_afs3-vlserver._udp." + cell, "SRV"): + for A in dns.resolver.query(SRV.target, 'A'): + vladdrs.add(A.address); + print("SRV:", SRV.target, ":", vladdrs); except dns.resolver.NXDOMAIN: print("Couldn't find any SRV records"); except dns.resolver.NoAnswer: print("Couldn't find any SRV records"); try: - afsdb_list = dns.resolver.query(cell, "AFSDB"); - for record in afsdb_list: - A_recs = dns.resolver.query(record.hostname, 'A'); - for addr in A_recs: - vladdrs.append(addr.address); - print(record.hostname, ":", vladdrs); + for AFSDB in dns.resolver.query(cell, "AFSDB"): + for A in dns.resolver.query(AFSDB.hostname, 'A'): + vladdrs.add(A.address); + print("AFSDB:", AFSDB.hostname, ":", vladdrs); except dns.resolver.NoAnswer: print("Couldn't find any AFSDB records"); if not vladdrs: raise RuntimeError("Couldn't find any VL server addresses"); -print("vladdrs:", vladdrs); - # Go through the list of VLDB servers until one answers a probe request +print("-- Probe for live VLDB servers --"); for vlserver in vladdrs: print("Trying", vlserver); @@ -67,8 +62,7 @@ for vlserver in vladdrs: try: ret = kafs.VL_Probe(z_conn); - if ret == 0: - break; + break; except ConnectionRefusedError: pass; del z_conn; @@ -77,34 +71,29 @@ if not z_conn: raise RuntimeError("Couldn't connect to a server"); # Look up each of the volumes in the list +print("-- Look up the named volumes --"); for vol in volumes: - vldblist = []; - ret = kafs.VL_GetEntryByName(z_conn, vol, vldblist); - if ret: - raise RuntimeError("Abort occurred {:d}".format(ret)); + ret = kafs.VL_GetEntryByName(z_conn, vol); + vldb = ret.entry; servers = set(); - for i in vldblist: - print("[", i.name, "]"); - print("\tnum\t", i.nServers); - print("\ttype\t", i.volumeType); - print("\tvid\t", i.volumeId); - print("\tflags\t {:x}".format(i.flags)); - for j in i.serverNumber: - if j: - servers.add(j); + print("[", vldb.name, "]"); + print("\tnum\t", vldb.nServers); + print("\ttype\t", vldb.volumeType); + print("\tvid\t", vldb.volumeId); + print("\tflags\t {:x}".format(vldb.flags)); + for j in vldb.serverNumber: + if j: + servers.add(j); # Pick an arbitrary server serving that volume and find out what volumes # that server serves - attributes = kafs.new_VldbListByAttributes(); + attributes = kafs.VldbListByAttributes(); attributes.Mask = kafs.VLLIST_SERVER; attributes.server = servers.pop(); - nentries = []; - blkentries = []; - ret = kafs.VL_ListAttributes(z_conn, attributes, nentries, blkentries) - if ret: - raise RuntimeError("Abort occurred {:d}".format(ret)); + ret = kafs.VL_ListAttributes(z_conn, attributes) + blkentries = ret.blkentries; for i in blkentries: print("->", i.name); -- 2.49.0