From c9585d937f0e1f9323a4a9bc85b2967980cd4902 Mon Sep 17 00:00:00 2001 From: Casey Bodley Date: Thu, 10 Mar 2011 12:37:28 -0500 Subject: [PATCH] pnfs: readwrite uses pnfs_layout_state nfs41_lock_stateid_arg() is now called only once in handle_read()/handle_write(), and pnfs_read()/pnfs_write() no longer depend on nfs41_open_state Signed-off-by: Casey Bodley --- daemon/open.c | 2 +- daemon/pnfs.h | 15 ++++++--- daemon/pnfs_device.c | 2 +- daemon/pnfs_io.c | 75 +++++++++++++++++++++++++------------------- daemon/readwrite.c | 62 +++++++++++++++++++----------------- 5 files changed, 87 insertions(+), 69 deletions(-) diff --git a/daemon/open.c b/daemon/open.c index 98e207e..986814a 100644 --- a/daemon/open.c +++ b/daemon/open.c @@ -576,7 +576,7 @@ static int handle_close(nfs41_upcall *upcall) /* return associated file layouts if necessary */ if (state->type == NF4REG) - pnfs_open_state_close(state->session, state, args->remove); + pnfs_layout_state_close(state->session, state, args->remove); if (args->remove) { nfs41_component *name = &state->file.name; diff --git a/daemon/pnfs.h b/daemon/pnfs.h index 9dffdee..f942202 100644 --- a/daemon/pnfs.h +++ b/daemon/pnfs.h @@ -207,10 +207,13 @@ typedef struct __pnfs_layout_recall { /* io */ +struct __stateid_arg; typedef struct __pnfs_io_pattern { struct __pnfs_io_thread *threads; struct __nfs41_root *root; - struct __nfs41_open_state *state; + nfs41_path_fh *meta_file; + const struct __stateid_arg *stateid; + pnfs_layout_state *state; pnfs_file_layout *layout; unsigned char *buffer; uint64_t offset_start; @@ -351,8 +354,9 @@ __inline uint32_t data_server_index( enum pnfs_status pnfs_read( IN struct __nfs41_root *root, IN struct __nfs41_session *session, - IN struct __nfs41_open_state *state, - IN pnfs_file_layout *layout, + IN nfs41_path_fh *meta_file, + IN const struct __stateid_arg *stateid, + IN pnfs_layout_state *layout, IN uint64_t offset, IN uint64_t length, OUT unsigned char *buffer_out, @@ -361,8 +365,9 @@ enum pnfs_status pnfs_read( enum pnfs_status pnfs_write( IN struct __nfs41_root *root, IN struct __nfs41_session *session, - IN struct __nfs41_open_state *state, - IN pnfs_file_layout *layout, + IN nfs41_path_fh *meta_file, + IN const struct __stateid_arg *stateid, + IN pnfs_layout_state *layout, IN uint64_t offset, IN uint64_t length, IN unsigned char *buffer, diff --git a/daemon/pnfs_device.c b/daemon/pnfs_device.c index 0713112..1363871 100644 --- a/daemon/pnfs_device.c +++ b/daemon/pnfs_device.c @@ -341,7 +341,7 @@ static enum pnfs_status get_sparse_fh( } else if (filehandle_count == 1) { *file_out = &layout->filehandles.arr[0]; } else if (filehandle_count == 0) { - *file_out = &pattern->state->file; + *file_out = pattern->meta_file; } else { eprintf("invalid sparse layout! has %u file handles " "and %u servers\n", filehandle_count, server_count); diff --git a/daemon/pnfs_io.c b/daemon/pnfs_io.c index 9d3c8e1..dc23f48 100644 --- a/daemon/pnfs_io.c +++ b/daemon/pnfs_io.c @@ -43,8 +43,9 @@ static uint32_t io_unit_count( static enum pnfs_status pattern_init( IN pnfs_io_pattern *pattern, IN nfs41_root *root, - IN nfs41_open_state *state, - IN pnfs_file_layout *layout, + IN nfs41_path_fh *meta_file, + IN const stateid_arg *stateid, + IN pnfs_layout_state *state, IN unsigned char *buffer, IN uint64_t offset, IN uint64_t length, @@ -58,25 +59,27 @@ static enum pnfs_status pattern_init( enum pnfs_status status; /* take a reference on the layout so we don't return it during io */ - status = pnfs_layout_io_start(layout); + status = pnfs_layout_io_start(state); if (status) goto out; #ifdef PNFS_THREAD_BY_SERVER - pattern->count = layout->device->servers.count; + pattern->count = state->layout->device->servers.count; #else - pattern->count = io_unit_count(layout, length); + pattern->count = io_unit_count(state->layout, length); #endif pattern->threads = calloc(pattern->count, sizeof(pnfs_io_thread)); if (pattern->threads == NULL) { status = PNFSERR_RESOURCES; - pnfs_layout_io_finished(pattern->layout); + pnfs_layout_io_finished(state); goto out; } pattern->root = root; + pattern->meta_file = meta_file; + pattern->stateid = stateid; pattern->state = state; - pattern->layout = layout; + pattern->layout = state->layout; pattern->buffer = buffer; pattern->offset_start = offset; pattern->offset_end = offset + length; @@ -115,7 +118,7 @@ static void pattern_free( IN pnfs_io_pattern *pattern) { /* inform the layout that our io is finished */ - pnfs_layout_io_finished(pattern->layout); + pnfs_layout_io_finished(pattern->state); free(pattern->threads); } @@ -124,17 +127,17 @@ static enum pnfs_status thread_next_unit( OUT pnfs_io_unit *io) { pnfs_io_pattern *pattern = thread->pattern; - pnfs_file_layout *layout = pattern->layout; + pnfs_layout_state *state = pattern->state; enum pnfs_status status = PNFS_SUCCESS; - AcquireSRWLockShared(&layout->layout.lock); + AcquireSRWLockShared(&state->lock); /* stop io if the layout is recalled */ - if (layout->layout.status & PNFS_LAYOUT_CHANGED) { + if (state->status & PNFS_LAYOUT_CHANGED) { status = PNFSERR_LAYOUT_CHANGED; goto out_unlock; } - if (layout->layout.status & PNFS_LAYOUT_RECALLED) { + if (state->status & PNFS_LAYOUT_RECALLED) { status = PNFSERR_LAYOUT_RECALLED; goto out_unlock; } @@ -157,7 +160,7 @@ static enum pnfs_status thread_next_unit( thread->offset += io->length; } out_unlock: - ReleaseSRWLockShared(&layout->layout.lock); + ReleaseSRWLockShared(&state->lock); return status; } @@ -259,7 +262,7 @@ static uint64_t pattern_bytes_transferred( static enum pnfs_status map_ds_error( IN enum nfsstat4 nfsstat, - IN pnfs_layout *layout) + IN pnfs_layout_state *state) { switch (nfsstat) { case NO_ERROR: @@ -274,12 +277,12 @@ static enum pnfs_status map_ds_error( case NFS4ERR_PNFS_NO_LAYOUT: dprintf(IOLVL, "data server fencing detected!\n"); - AcquireSRWLockExclusive(&layout->lock); + AcquireSRWLockExclusive(&state->lock); /* flag the layout for return once io is finished */ - layout->status |= PNFS_LAYOUT_RECALLED | PNFS_LAYOUT_CHANGED; + state->status |= PNFS_LAYOUT_RECALLED | PNFS_LAYOUT_CHANGED; /* reset GRANTED so we know not to try LAYOUTRETURN */ - layout->status &= ~PNFS_LAYOUT_GRANTED; - ReleaseSRWLockExclusive(&layout->lock); + state->status &= ~PNFS_LAYOUT_GRANTED; + ReleaseSRWLockExclusive(&state->lock); /* return CHANGED to prevent any further use of the layout */ return PNFSERR_LAYOUT_CHANGED; @@ -320,7 +323,7 @@ static uint32_t WINAPI file_layout_read_thread(void *args) goto out; } - nfs41_lock_stateid_arg(pattern->state, &stateid); + memcpy(&stateid, pattern->stateid, sizeof(stateid)); stateid.stateid.seqid = 0; total_read = 0; @@ -334,7 +337,7 @@ static uint32_t WINAPI file_layout_read_thread(void *args) if (nfsstat) { eprintf("nfs41_read() failed with %s\n", nfs_error_string(nfsstat)); - status = map_ds_error(nfsstat, &pattern->layout->layout); + status = map_ds_error(nfsstat, pattern->state); break; } @@ -389,7 +392,7 @@ static uint32_t WINAPI file_layout_write_thread(void *args) goto out; } - nfs41_lock_stateid_arg(pattern->state, &stateid); + memcpy(&stateid, pattern->stateid, sizeof(stateid)); stateid.stateid.seqid = 0; retry_write: @@ -408,7 +411,7 @@ retry_write: if (nfsstat) { eprintf("nfs41_write() failed with %s\n", nfs_error_string(nfsstat)); - status = map_ds_error(nfsstat, &layout->layout); + status = map_ds_error(nfsstat, pattern->state); break; } if (!verify_write(&verf, &thread->stable)) @@ -446,7 +449,7 @@ retry_write: if (nfsstat == NFS4_OK) thread->stable = DATA_SYNC4; else - status = map_ds_error(nfsstat, &pattern->layout->layout); + status = map_ds_error(nfsstat, pattern->state); out: dprintf(IOLVL, "<-- file_layout_write_thread(%u) returning %s\n", thread->id, pnfs_error_string(status)); @@ -457,8 +460,9 @@ out: enum pnfs_status pnfs_read( IN nfs41_root *root, IN nfs41_session *session, - IN nfs41_open_state *state, - IN pnfs_file_layout *layout, + IN nfs41_path_fh *meta_file, + IN const stateid_arg *stateid, + IN pnfs_layout_state *layout, IN uint64_t offset, IN uint64_t length, OUT unsigned char *buffer_out, @@ -471,7 +475,7 @@ enum pnfs_status pnfs_read( *len_out = 0; - status = pattern_init(&pattern, root, state, layout, + status = pattern_init(&pattern, root, meta_file, stateid, layout, buffer_out, offset, length, session->lease_time); if (status) { eprintf("pattern_init() failed with %s\n", @@ -496,8 +500,9 @@ out: enum pnfs_status pnfs_write( IN nfs41_root *root, IN nfs41_session *session, - IN nfs41_open_state *state, - IN pnfs_file_layout *layout, + IN nfs41_path_fh *meta_file, + IN const stateid_arg *stateid, + IN pnfs_layout_state *layout, IN uint64_t offset, IN uint64_t length, IN unsigned char *buffer, @@ -512,7 +517,7 @@ enum pnfs_status pnfs_write( *len_out = 0; - status = pattern_init(&pattern, root, state, layout, + status = pattern_init(&pattern, root, meta_file, stateid, layout, buffer, offset, length, session->lease_time); if (status) { eprintf("pattern_init() failed with %s\n", @@ -533,7 +538,7 @@ enum pnfs_status pnfs_write( /* not all data was committed, so commit to metadata server */ dprintf(1, "sending COMMIT to meta server for offset=%d and len=%d\n", offset, *len_out); - nfsstat = nfs41_commit(session, &state->file, offset, *len_out, 1); + nfsstat = nfs41_commit(session, meta_file, offset, *len_out, 1); if (nfsstat) { dprintf(IOLVL, "nfs41_commit() failed with %s\n", nfs_error_string(nfsstat)); @@ -541,11 +546,15 @@ enum pnfs_status pnfs_write( } } else if (stable == DATA_SYNC4) { /* send LAYOUTCOMMIT to sync the metadata */ + stateid4 layout_stateid; uint64_t new_last_offset = offset + *len_out - 1; - nfsstat = pnfs_rpc_layoutcommit(session, &state->file, - &pattern.layout->layout.state, offset, *len_out, - &new_last_offset, NULL); + AcquireSRWLockShared(&layout->lock); + memcpy(&layout_stateid, &layout->stateid, sizeof(layout_stateid)); + ReleaseSRWLockShared(&layout->lock); + + nfsstat = pnfs_rpc_layoutcommit(session, meta_file, + &layout_stateid, offset, *len_out, &new_last_offset, NULL); if (nfsstat) { dprintf(IOLVL, "pnfs_rpc_layoutcommit() failed with %s\n", nfs_error_string(nfsstat)); diff --git a/daemon/readwrite.c b/daemon/readwrite.c index 2051420..577035e 100644 --- a/daemon/readwrite.c +++ b/daemon/readwrite.c @@ -62,20 +62,18 @@ out: /* NFS41_READ */ static int read_from_mds( IN nfs41_session *session, - IN nfs41_open_state *state, + IN nfs41_path_fh *file, + IN stateid_arg *stateid, IN uint64_t offset, IN uint32_t length, OUT unsigned char *buffer, OUT ULONG *len_out) { - stateid_arg stateid; int status = 0; bool_t eof; unsigned char *p = buffer; ULONG to_rcv = length, reloffset = 0, len = 0; - const uint32_t maxreadsize = max_read_size(session, &state->file.fh); - - nfs41_lock_stateid_arg(state, &stateid); + const uint32_t maxreadsize = max_read_size(session, &file->fh); if (to_rcv > maxreadsize) dprintf(1, "handle_nfs41_read: reading %d in chunks of %d\n", @@ -84,11 +82,11 @@ static int read_from_mds( while(to_rcv > 0) { uint32_t bytes_read = 0, chunk = min(to_rcv, maxreadsize); - status = nfs41_read(session, &state->file, &stateid, + status = nfs41_read(session, file, stateid, offset + reloffset, chunk, p, &bytes_read, &eof); if (status == NFS4ERR_OPENMODE && !len) { - stateid.type = STATEID_SPECIAL; - memcpy(&stateid.stateid, &special_read_stateid, sizeof(stateid4)); + stateid->type = STATEID_SPECIAL; + memcpy(&stateid->stateid, &special_read_stateid, sizeof(stateid4)); continue; } else if (status && !len) { status = nfs_to_windows_error(status, ERROR_NET_WRITE_FAULT); @@ -117,24 +115,25 @@ out: static int read_from_pnfs( IN nfs41_root *root, IN nfs41_open_state *state, + IN const stateid_arg *stateid, IN uint64_t offset, IN uint32_t length, OUT unsigned char *buffer, OUT ULONG *len_out) { - pnfs_file_layout *layout; + pnfs_layout_state *layout; enum pnfs_status pnfsstat; int status = NO_ERROR; - pnfsstat = pnfs_open_state_layout(state->session->client->layouts, + pnfsstat = pnfs_layout_state_open(state->session->client->layouts, state->session, state, PNFS_IOMODE_READ, offset, length, &layout); if (pnfsstat) { status = ERROR_NOT_SUPPORTED; goto out; } - pnfsstat = pnfs_read(root, state->session, state, - layout, offset, length, buffer, len_out); + pnfsstat = pnfs_read(root, state->session, &state->file, + stateid, layout, offset, length, buffer, len_out); switch (pnfsstat) { case PNFS_SUCCESS: break; @@ -152,11 +151,14 @@ out: static int handle_read(nfs41_upcall *upcall) { readwrite_upcall_args *args = &upcall->args.rw; + stateid_arg stateid; ULONG pnfs_bytes_read = 0; int status = NO_ERROR; + nfs41_lock_stateid_arg(args->state, &stateid); + #ifdef PNFS_ENABLE_READ - status = read_from_pnfs(args->root, args->state, + status = read_from_pnfs(args->root, args->state, &stateid, args->offset, args->len, args->buffer, &args->out_len); if (status == NO_ERROR || status == ERROR_HANDLE_EOF) @@ -172,8 +174,8 @@ static int handle_read(nfs41_upcall *upcall) } #endif - status = read_from_mds(args->state->session, args->state, - args->offset, args->len, args->buffer, &args->out_len); + status = read_from_mds(args->state->session, &args->state->file, + &stateid, args->offset, args->len, args->buffer, &args->out_len); args->out_len += pnfs_bytes_read; out: @@ -184,22 +186,20 @@ out: /* NFS41_WRITE */ static int write_to_mds( IN nfs41_session *session, - IN nfs41_open_state *state, + IN nfs41_path_fh *file, + IN stateid_arg *stateid, IN uint64_t offset, IN uint32_t length, IN unsigned char *buffer, OUT ULONG *len_out) { - stateid_arg stateid; nfs41_write_verf verf; enum stable_how4 stable, committed; unsigned char *p; - const uint32_t maxwritesize = max_write_size(session, &state->file.fh); + const uint32_t maxwritesize = max_write_size(session, &file->fh); uint32_t to_send, reloffset, len; int status = 0; - nfs41_lock_stateid_arg(state, &stateid); - retry_write: p = buffer; to_send = length; @@ -215,7 +215,7 @@ retry_write: while(to_send > 0) { uint32_t bytes_written = 0, chunk = min(to_send, maxwritesize); - status = nfs41_write(session, &state->file, &stateid, p, chunk, + status = nfs41_write(session, file, stateid, p, chunk, offset + reloffset, stable, &bytes_written, &verf); if (status && !len) goto out; @@ -232,7 +232,7 @@ retry_write: } if (committed == UNSTABLE4) { dprintf(1, "sending COMMIT for offset=%d and len=%d\n", offset, len); - status = nfs41_commit(session, &state->file, offset, len, 1); + status = nfs41_commit(session, file, offset, len, 1); } out: *len_out = len; @@ -242,24 +242,25 @@ out: static int write_to_pnfs( IN nfs41_root *root, IN nfs41_open_state *state, + IN const stateid_arg *stateid, IN uint64_t offset, IN uint32_t length, IN unsigned char *buffer, OUT ULONG *len_out) { - pnfs_file_layout *layout; + pnfs_layout_state *layout; enum pnfs_status pnfsstat; int status = NO_ERROR; - pnfsstat = pnfs_open_state_layout(state->session->client->layouts, + pnfsstat = pnfs_layout_state_open(state->session->client->layouts, state->session, state, PNFS_IOMODE_RW, offset, length, &layout); if (pnfsstat) { status = ERROR_NOT_SUPPORTED; goto out; } - pnfsstat = pnfs_write(root, state->session, state, - layout, offset, length, buffer, len_out); + pnfsstat = pnfs_write(root, state->session, &state->file, + stateid, layout, offset, length, buffer, len_out); if (pnfsstat) { status = ERROR_WRITE_FAULT; goto out; @@ -271,11 +272,14 @@ out: static int handle_write(nfs41_upcall *upcall) { readwrite_upcall_args *args = &upcall->args.rw; + stateid_arg stateid; ULONG pnfs_bytes_written = 0; int status; + nfs41_lock_stateid_arg(args->state, &stateid); + #ifdef PNFS_ENABLE_WRITE - status = write_to_pnfs(args->root, args->state, + status = write_to_pnfs(args->root, args->state, &stateid, args->offset, args->len, args->buffer, &args->out_len); if (status == NO_ERROR) @@ -291,8 +295,8 @@ static int handle_write(nfs41_upcall *upcall) } #endif - status = write_to_mds(args->state->session, args->state, - args->offset, args->len, args->buffer, &args->out_len); + status = write_to_mds(args->state->session, &args->state->file, + &stateid, args->offset, args->len, args->buffer, &args->out_len); args->out_len += pnfs_bytes_written; out: