From 02216cbf28cab0eda0cd4052e5830fe0c25138f5 Mon Sep 17 00:00:00 2001 From: Casey Bodley Date: Wed, 6 Jul 2011 13:41:59 -0400 Subject: [PATCH] deleg: return delegation on CB_RECALL moved thread creation from callback_server.c to nfs41_delegation_recall() in delegation.c nfs41_delegation_recall() first searches for the delegation, and returns NFS4ERR_BADHANDLE if not found. otherwise, it spawns a thread to handle the recall and returns NFS4_OK delegation_return() calls nfs41_delegation_to_open() for each nfs41_open_state associated with the delegation nfs41_delegation_to_open() sends OPEN with CLAIM_DELEGATE_CUR to reclaim an open stateid, and uses a condition variable in nfs41_open_state to prevent multiple threads from attempting reclaim Signed-off-by: Casey Bodley --- daemon/callback_server.c | 64 +---------- daemon/delegation.c | 241 ++++++++++++++++++++++++++++++++++++++- daemon/delegation.h | 12 ++ daemon/nfs41.h | 6 +- 4 files changed, 258 insertions(+), 65 deletions(-) diff --git a/daemon/callback_server.c b/daemon/callback_server.c index ec79e67..4026d81 100644 --- a/daemon/callback_server.c +++ b/daemon/callback_server.c @@ -24,6 +24,7 @@ #include #include "nfs41_ops.h" +#include "delegation.h" #include "nfs41_callback.h" #include "daemon_debug.h" @@ -177,71 +178,14 @@ out: } /* OP_CB_RECALL */ -typedef struct _nfs41_cb_recall { - nfs41_rpc_clnt *rpc_clnt; - struct cb_recall_args *args; -} nfs41_cb_recall; - -static unsigned int WINAPI _handle_cb_recall(void *args) -{ - nfs41_cb_recall *cb_args = (nfs41_cb_recall *)args; - nfs41_path_fh path_fh; - - dprintf(1, "_handle_cb_recall: start\n"); - print_hexbuf(3, (unsigned char *)"_handle_cb_recall: fh ", - cb_args->args->fh.fh, cb_args->args->fh.len); - print_hexbuf(3, (unsigned char *)"_handle_cb_recall: stateid ", - cb_args->args->stateid.other, NFS4_STATEID_OTHER); - ZeroMemory(&path_fh, sizeof(nfs41_path_fh)); - memcpy(&path_fh.fh, &cb_args->args->fh, sizeof(nfs41_fh)); - path_fh.fh.superblock = NULL; - path_fh.path = NULL; - path_fh.name.len = 0; - dprintf(1, "_handle_cb_recall: sending nfs41_delegreturn\n"); - nfs41_delegreturn(cb_args->rpc_clnt->client->session, &path_fh, - &cb_args->args->stateid); - nfs41_root_deref(cb_args->rpc_clnt->client->root); - free(cb_args->args); - free(cb_args); - dprintf(1, "_handle_cb_recall: end\n"); - return 1; -} - static enum_t handle_cb_recall( IN nfs41_rpc_clnt *rpc_clnt, IN struct cb_recall_args *args, OUT struct cb_recall_res *res) { - nfs41_cb_recall *cb_args; - uintptr_t status; - res->status = NFS4_OK; - - dprintf(CBSLVL, "OP_CB_RECALL\n"); - cb_args = calloc(1, sizeof(nfs41_cb_recall)); - if (cb_args == NULL) { - res->status = NFS4ERR_SERVERFAULT; - goto out; - } - cb_args->rpc_clnt = rpc_clnt; - cb_args->args = calloc(1, sizeof(struct cb_recall_args)); - if (cb_args->args == NULL) { - free(cb_args); - res->status = NFS4ERR_SERVERFAULT; - goto out; - } - memcpy(cb_args->args, args, sizeof(struct cb_recall_args)); - status = _beginthreadex(NULL, 0, _handle_cb_recall, cb_args, 0, NULL); - if (status == -1L || !status) { - eprintf("_beginthreadex failed to start for _handle_cb_recall %d", - status); - free(cb_args->args); - free(cb_args); - res->status = NFS4ERR_SERVERFAULT; - goto out; - } - nfs41_root_ref(rpc_clnt->client->root); - -out: + /* return the delegation asynchronously */ + res->status = nfs41_delegation_recall(rpc_clnt->client, + &args->fh, &args->stateid, args->truncate); return res->status; } diff --git a/daemon/delegation.c b/daemon/delegation.c index 1a94882..bb86ed4 100644 --- a/daemon/delegation.c +++ b/daemon/delegation.c @@ -46,8 +46,11 @@ static int delegation_create( } memcpy(&state->state, delegation, sizeof(open_delegation4)); - fh_copy(&state->fh, &file->fh); + abs_path_copy(&state->path, file->path); + path_fh_init(&state->file, &state->path); + fh_copy(&state->file.fh, &file->fh); list_init(&state->client_entry); + InitializeSRWLock(&state->lock); state->ref_count = 1; *deleg_out = state; out: @@ -58,19 +61,98 @@ void nfs41_delegation_ref( IN nfs41_delegation_state *state) { const LONG count = InterlockedIncrement(&state->ref_count); - dprintf(DGLVL, "nfs41_delegation_ref() count %d\n", count); + dprintf(DGLVL, "nfs41_delegation_ref(%s) count %d\n", + state->path.path, count); } void nfs41_delegation_deref( IN nfs41_delegation_state *state) { const LONG count = InterlockedDecrement(&state->ref_count); - dprintf(DGLVL, "nfs41_delegation_deref() count %d\n", count); + dprintf(DGLVL, "nfs41_delegation_deref(%s) count %d\n", + state->path.path, count); if (count == 0) free(state); } +/* delegation return */ +#define open_entry(pos) list_container(pos, nfs41_open_state, client_entry) + +static int open_deleg_cmp(const struct list_entry *entry, const void *value) +{ + nfs41_open_state *open = open_entry(entry); + int result = -1; + + /* open must match the delegation and have no open stateid */ + AcquireSRWLockShared(&open->lock); + if (open->delegation.state != value) goto out; + if (open->do_close) goto out; + result = 0; +out: + ReleaseSRWLockShared(&open->lock); + return result; +} + +/* find the first open that needs recovery */ +static nfs41_open_state* deleg_open_find( + IN struct client_state *state, + IN const nfs41_delegation_state *deleg) +{ + struct list_entry *entry; + nfs41_open_state *open = NULL; + + EnterCriticalSection(&state->lock); + entry = list_search(&state->opens, deleg, open_deleg_cmp); + if (entry) { + open = open_entry(entry); + nfs41_open_state_ref(open); /* return a reference */ + } + LeaveCriticalSection(&state->lock); + return open; +} + +#pragma warning (disable : 4706) /* assignment within conditional expression */ + +static void delegation_return( + IN nfs41_client *client, + IN nfs41_delegation_state *deleg, + IN bool_t truncate) +{ + struct list_entry *entry; + nfs41_open_state *open; + + /* recover opens associated with the delegation */ + while (open = deleg_open_find(&client->state, deleg)) { + nfs41_delegation_to_open(open, TRUE); + nfs41_open_state_deref(open); + } + + /* TODO: flush data and metadata before returning delegation */ + + nfs41_delegreturn(client->session, &deleg->file, &deleg->state.stateid); + + /* remove from the client's list */ + EnterCriticalSection(&client->state.lock); + list_remove(&deleg->client_entry); + + list_for_each(entry, &client->state.opens) { + open = open_entry(entry); + AcquireSRWLockExclusive(&open->lock); + if (open->delegation.state == deleg) { + /* drop the delegation reference */ + nfs41_delegation_deref(open->delegation.state); + open->delegation.state = NULL; + } + ReleaseSRWLockExclusive(&open->lock); + } + LeaveCriticalSection(&client->state.lock); + + /* release the client's reference */ + nfs41_delegation_deref(deleg); +} + + /* open delegation */ int nfs41_delegation_granted( IN nfs41_session *session, @@ -112,7 +194,7 @@ out_return: /* return the delegation on failure */ static int deleg_fh_cmp(const struct list_entry *entry, const void *value) { - const nfs41_fh *lhs = &deleg_entry(entry)->fh; + const nfs41_fh *lhs = &deleg_entry(entry)->file.fh; const nfs41_fh *rhs = (const nfs41_fh*)value; if (lhs->superblock != rhs->superblock) return -1; if (lhs->fileid != rhs->fileid) return -1; @@ -202,6 +284,157 @@ out_deleg: goto out; } +int nfs41_delegation_to_open( + IN nfs41_open_state *open, + IN bool_t try_recovery) +{ + open_delegation4 ignore; + open_claim4 claim; + stateid4 deleg_stateid, open_stateid = { 0 }; + int status = NFS4_OK; + + AcquireSRWLockExclusive(&open->lock); + if (open->delegation.state == NULL) /* no delegation to reclaim */ + goto out_unlock; + + if (open->do_close) /* already have an open stateid */ + goto out_unlock; + + /* if another thread is reclaiming the open stateid, + * wait for it to finish before returning success */ + if (open->delegation.reclaim) { + do { + SleepConditionVariableSRW(&open->delegation.cond, &open->lock, + INFINITE, 0); + } while (open->delegation.reclaim); + goto out_unlock; + } + open->delegation.reclaim = 1; + + AcquireSRWLockShared(&open->delegation.state->lock); + memcpy(&deleg_stateid, &open->delegation.state->state.stateid, + sizeof(stateid4)); + ReleaseSRWLockShared(&open->delegation.state->lock); + + ReleaseSRWLockExclusive(&open->lock); + + /* send OPEN with CLAIM_DELEGATE_CUR */ + claim.claim = CLAIM_DELEGATE_CUR; + claim.u.deleg_cur.delegate_stateid = &deleg_stateid; + claim.u.deleg_cur.name = &open->file.name; + + status = nfs41_rpc_open(open->session, &open->parent, &open->file, + &open->owner, &claim, open->share_access, open->share_deny, + OPEN4_NOCREATE, 0, 0, try_recovery, &open_stateid, &ignore, NULL); + if (status) + eprintf("nfs41_delegation_to_open(%p) failed with %s\n", + open, nfs_error_string(status)); + + AcquireSRWLockExclusive(&open->lock); + /* save the new open stateid */ + memcpy(&open->stateid, &open_stateid, sizeof(stateid4)); + open->delegation.reclaim = 0; + open->do_close = 1; + + /* signal anyone waiting on the open stateid */ + WakeAllConditionVariable(&open->delegation.cond); +out_unlock: + ReleaseSRWLockExclusive(&open->lock); + return status; +} + + +/* asynchronous delegation recall */ +struct recall_thread_args { + nfs41_client *client; + nfs41_delegation_state *delegation; + bool_t truncate; +}; + +static unsigned int WINAPI delegation_recall_thread(void *args) +{ + struct recall_thread_args *recall = (struct recall_thread_args*)args; + + delegation_return(recall->client, recall->delegation, recall->truncate); + + /* clean up thread arguments */ + nfs41_delegation_deref(recall->delegation); + nfs41_root_deref(recall->client->root); + free(recall); + return 0; +} + +static int deleg_stateid_cmp(const struct list_entry *entry, const void *value) +{ + const stateid4 *lhs = &deleg_entry(entry)->state.stateid; + const stateid4 *rhs = (const stateid4*)value; + return memcmp(lhs->other, rhs->other, NFS4_STATEID_OTHER); +} + +int nfs41_delegation_recall( + IN nfs41_client *client, + IN nfs41_fh *fh, + IN const stateid4 *stateid, + IN bool_t truncate) +{ + nfs41_delegation_state *deleg; + struct recall_thread_args *args; + int status; + + dprintf(2, "--> nfs41_delegation_recall()\n"); + + /* search for the delegation by stateid instead of filehandle; + * deleg_fh_cmp() relies on a proper superblock and fileid, + * which we don't get with CB_RECALL */ + status = delegation_find(client, stateid, deleg_stateid_cmp, &deleg); + if (status) + goto out; + + /* start the delegation recall, or fail if it's already started */ + AcquireSRWLockShared(&deleg->lock); + if (deleg->state.recalled == 0) + deleg->state.recalled = 1; + else + status = NFS4ERR_BADHANDLE; + ReleaseSRWLockShared(&deleg->lock); + if (status) + goto out_deleg; + + /* TODO: return NFS4_OK if the delegation is already being returned */ + + /* allocate thread arguments */ + args = calloc(1, sizeof(struct recall_thread_args)); + if (args == NULL) { + status = NFS4ERR_SERVERFAULT; + eprintf("nfs41_delegation_recall() failed to allocate arguments\n"); + goto out_deleg; + } + + /* hold a reference on the root */ + nfs41_root_ref(client->root); + args->client = client; + args->delegation = deleg; + args->truncate = truncate; + + /* the callback thread can't make rpc calls, so spawn a separate thread */ + if (_beginthreadex(NULL, 0, delegation_recall_thread, args, 0, NULL) == 0) { + status = NFS4ERR_SERVERFAULT; + eprintf("nfs41_delegation_recall() failed to start thread\n"); + goto out_args; + } +out: + dprintf(DGLVL, "<-- nfs41_delegation_recall() returning %s\n", + nfs_error_string(status)); + return status; + +out_args: + free(args); + nfs41_root_deref(client->root); +out_deleg: + nfs41_delegation_deref(deleg); + goto out; +} + void nfs41_client_delegation_free( IN nfs41_client *client) { diff --git a/daemon/delegation.h b/daemon/delegation.h index d29a9a3..b493cf5 100644 --- a/daemon/delegation.h +++ b/daemon/delegation.h @@ -54,4 +54,16 @@ int nfs41_delegate_open( OUT nfs41_delegation_state **deleg_out, OUT nfs41_file_info *info); +int nfs41_delegation_to_open( + IN nfs41_open_state *open, + IN bool_t try_recovery); + + +/* asynchronous delegation recall */ +int nfs41_delegation_recall( + IN nfs41_client *client, + IN nfs41_fh *fh, + IN const stateid4 *stateid, + IN bool_t truncate); + #endif /* DELEGATION_H */ diff --git a/daemon/nfs41.h b/daemon/nfs41.h index 3124b2e..433aa3d 100644 --- a/daemon/nfs41.h +++ b/daemon/nfs41.h @@ -82,9 +82,11 @@ typedef struct __nfs41_server { typedef struct __nfs41_delegation_state { open_delegation4 state; - nfs41_fh fh; + nfs41_abs_path path; + nfs41_path_fh file; struct list_entry client_entry; /* entry in nfs41_client.delegations */ LONG ref_count; + SRWLOCK lock; } nfs41_delegation_state; typedef struct __nfs41_lock_state { @@ -119,6 +121,8 @@ typedef struct __nfs41_open_state { struct { nfs41_delegation_state *state; + bool_t reclaim; + CONDITION_VARIABLE cond; } delegation; struct { /* list of open lock state for recovery */