evin/ms-nfs41-client
3
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

recovery: remember byte-range locks and reclaim during recovery

Browse source 
nfs41_open_state maintains a list of outstanding byte-range locks by calling open_lock_add() and open_lock_remove() in lock.c

during client state recovery, after reclaiming each OPEN stateid, send LOCK requests with reclaim=TRUE for each lock it owns, and update the open's lock stateid with the result

added 'bool_t reclaim' argument to nfs41_lock(); when set, compound_encode_send_decode() is called with try_recovery=FALSE to avoid recursive recovery

Signed-off-by: Casey Bodley <cbodley@citi.umich.edu>
This commit is contained in:
Casey Bodley 2010-11-18 15:11:36 -05:00 • committed by unknown
parent 1906610544
commit 222c1bf020
7 changed files with 141 additions and 56 deletions
Download patch file Download diff file Expand all files Collapse all files

115
daemon/lock.c
View file

@ -37,53 +37,84 @@ void nfs41_lock_stateid_arg(
IN nfs41_open_state *state, IN nfs41_open_state *state,
OUT stateid_arg *arg) OUT stateid_arg *arg)
{ {
AcquireSRWLockShared(&state->last_lock.lock); AcquireSRWLockShared(&state->lock);
if (state->last_lock.initialized) { if (state->locks.stateid.seqid) {
/* use lock stateid where available */ /* use lock stateid where available */
memcpy(&arg->stateid, &state->last_lock.stateid, sizeof(stateid4)); memcpy(&arg->stateid, &state->locks.stateid, sizeof(stateid4));
arg->type = STATEID_LOCK; arg->type = STATEID_LOCK;
ReleaseSRWLockShared(&state->last_lock.lock); arg->open = state;
} else { } else {
ReleaseSRWLockShared(&state->last_lock.lock);
/* fall back on open stateid */ /* fall back on open stateid */
nfs41_open_stateid_arg(state, arg); nfs41_open_stateid_arg(state, arg);
} }
ReleaseSRWLockShared(&state->lock);
} }
static void update_last_lock_state( /* expects the caller to hold an exclusive lock on nfs41_open_state.lock */
OUT nfs41_lock_state *lock_state, static void update_lock_state(
IN stateid4 *stateid) OUT nfs41_open_state *state,
IN const stateid4 *stateid)
{ {
/* update the lock state if the seqid is more recent */ if (state->locks.stateid.seqid == 0) {
AcquireSRWLockShared(&lock_state->lock); /* if it's a new lock stateid, copy it in */
if (stateid->seqid > lock_state->stateid.seqid) { memcpy(&state->locks.stateid, stateid, sizeof(stateid4));
ReleaseSRWLockShared(&lock_state->lock); } else if (stateid->seqid > state->locks.stateid.seqid) {
/* update the seqid if it's more recent */
AcquireSRWLockExclusive(&lock_state->lock); state->locks.stateid.seqid = stateid->seqid;
if (stateid->seqid > lock_state->stateid.seqid) {
if (lock_state->initialized) {
/* if the lock state already existed, update the seqid only;
* assume that stateid->other remains unchanged */
dprintf(LKLVL, "update_last_lock_state: setting seqid=%u "
"(was %u)\n", stateid->seqid, lock_state->stateid.seqid);
lock_state->stateid.seqid = stateid->seqid;
} else {
/* copy the entire stateid and mark as initialized */
dprintf(LKLVL, "update_last_lock_state: stateid "
"initialized with seqid=%u\n", stateid->seqid);
memcpy(&lock_state->stateid, stateid, sizeof(stateid4));
lock_state->initialized = 1;
}
}
ReleaseSRWLockExclusive(&lock_state->lock);
} else {
dprintf(LKLVL, "update_last_lock_state: discarding seqid=%u "
"(already %u)\n", stateid->seqid, lock_state->stateid.seqid);
ReleaseSRWLockShared(&lock_state->lock);
} }
} }
static int open_lock_add(
IN nfs41_open_state *state,
IN const stateid4 *stateid,
IN uint64_t offset,
IN uint64_t length,
IN uint32_t type)
{
nfs41_lock_state *lock;
int status = NO_ERROR;
AcquireSRWLockExclusive(&state->lock);
update_lock_state(state, stateid);
lock = malloc(sizeof(nfs41_lock_state));
if (lock == NULL) {
status = GetLastError();
goto out;
}
lock->offset = offset;
lock->length = length;
lock->type = type;
list_add_tail(&state->locks.list, &lock->open_entry);
out:
ReleaseSRWLockExclusive(&state->lock);
return status;
}
static void open_lock_remove(
IN nfs41_open_state *state,
IN const stateid4 *stateid,
IN uint64_t offset,
IN uint64_t length)
{
struct list_entry *entry;
nfs41_lock_state *lock;
AcquireSRWLockExclusive(&state->lock);
update_lock_state(state, stateid);
list_for_each(entry, &state->locks.list) {
lock = list_container(entry, nfs41_lock_state, open_entry);
if (lock->offset == offset && lock->length == length) {
list_remove(entry);
free(lock);
break;
}
}
ReleaseSRWLockExclusive(&state->lock);
}
/* NFS41_LOCK */ /* NFS41_LOCK */
static int parse_lock(unsigned char *buffer, uint32_t length, nfs41_upcall *upcall) static int parse_lock(unsigned char *buffer, uint32_t length, nfs41_upcall *upcall)
@ -132,7 +163,7 @@ static int handle_lock(nfs41_upcall *upcall)
nfs41_lock_stateid_arg(state, &stateid); nfs41_lock_stateid_arg(state, &stateid);
status = nfs41_lock(state->session, &state->file, &state->owner, status = nfs41_lock(state->session, &state->file, &state->owner,
type, args->offset, args->length, &stateid); type, args->offset, args->length, 0, &stateid);
if (status) { if (status) {
dprintf(LKLVL, "nfs41_lock failed with %s\n", dprintf(LKLVL, "nfs41_lock failed with %s\n",
nfs_error_string(status)); nfs_error_string(status));
@ -140,7 +171,9 @@ static int handle_lock(nfs41_upcall *upcall)
goto out; goto out;
} }
update_last_lock_state(&state->last_lock, &stateid.stateid); /* ignore errors from open_lock_add(); they just mean we
* won't be able to recover the lock after reboot */
open_lock_add(state, &stateid.stateid, args->offset, args->length, type);
out: out:
return status; return status;
} }
@ -168,7 +201,7 @@ static void cancel_lock(IN nfs41_upcall *upcall)
goto out; goto out;
} }
update_last_lock_state(&state->last_lock, &stateid.stateid); open_lock_remove(state, &stateid.stateid, args->offset, args->length);
out: out:
dprintf(1, "<-- cancel_lock() returning %d\n", status); dprintf(1, "<-- cancel_lock() returning %d\n", status);
} }
@ -208,7 +241,7 @@ static int handle_unlock(nfs41_upcall *upcall)
uint32_t buf_len = args->buf_len; uint32_t buf_len = args->buf_len;
uint64_t offset; uint64_t offset;
uint64_t length; uint64_t length;
int status; int status = NO_ERROR;
nfs41_lock_stateid_arg(state, &stateid); nfs41_lock_stateid_arg(state, &stateid);
if (stateid.type != STATEID_LOCK) { if (stateid.type != STATEID_LOCK) {
@ -217,7 +250,6 @@ static int handle_unlock(nfs41_upcall *upcall)
goto out; goto out;
} }
status = NO_ERROR;
for (i = 0; i < args->count; i++) { for (i = 0; i < args->count; i++) {
if (safe_read(&buf, &buf_len, &offset, sizeof(LONGLONG))) break; if (safe_read(&buf, &buf_len, &offset, sizeof(LONGLONG))) break;
if (safe_read(&buf, &buf_len, &length, sizeof(LONGLONG))) break; if (safe_read(&buf, &buf_len, &length, sizeof(LONGLONG))) break;
@ -225,6 +257,7 @@ static int handle_unlock(nfs41_upcall *upcall)
status = nfs41_unlock(state->session, status = nfs41_unlock(state->session,
&state->file, offset, length, &stateid); &state->file, offset, length, &stateid);
if (status == NFS4_OK) { if (status == NFS4_OK) {
open_lock_remove(state, &stateid.stateid, offset, length);
nsuccess++; nsuccess++;
} else { } else {
dprintf(LKLVL, "nfs41_unlock failed with %s\n", dprintf(LKLVL, "nfs41_unlock failed with %s\n",
@ -234,7 +267,7 @@ static int handle_unlock(nfs41_upcall *upcall)
} }
if (nsuccess) { if (nsuccess) {
update_last_lock_state(&state->last_lock, &stateid.stateid); update_lock_state(state, &stateid.stateid);
status = NO_ERROR; status = NO_ERROR;
} }
out: out:

13
daemon/nfs41.h
View file

@ -68,9 +68,10 @@ typedef struct __nfs41_server {
} nfs41_server; } nfs41_server;
typedef struct __nfs41_lock_state { typedef struct __nfs41_lock_state {
bool_t initialized; struct list_entry open_entry; /* entry in nfs41_open_state.locks */
stateid4 stateid; uint64_t offset;
SRWLOCK lock; uint64_t length;
uint32_t type;
} nfs41_lock_state; } nfs41_lock_state;
/* nfs41_open_state reference counting: /* nfs41_open_state reference counting:
@ -89,13 +90,17 @@ typedef struct __nfs41_open_state {
bool_t do_close; bool_t do_close;
stateid4 stateid; stateid4 stateid;
state_owner4 owner; state_owner4 owner;
nfs41_lock_state last_lock;
struct __pnfs_file_layout *layout; struct __pnfs_file_layout *layout;
struct list_entry client_entry; /* entry in nfs41_client.opens */ struct list_entry client_entry; /* entry in nfs41_client.opens */
SRWLOCK lock; SRWLOCK lock;
LONG ref_count; LONG ref_count;
uint32_t share_access; uint32_t share_access;
uint32_t share_deny; uint32_t share_deny;
struct { /* list of open lock state for recovery */
stateid4 stateid;
struct list_entry list;
} locks;
} nfs41_open_state; } nfs41_open_state;
typedef struct __nfs41_rpc_clnt { typedef struct __nfs41_rpc_clnt {

41
daemon/nfs41_compound.c
View file

@ -127,18 +127,39 @@ static int recover_open(
IN nfs41_session *session, IN nfs41_session *session,
IN nfs41_open_state *open) IN nfs41_open_state *open)
{ {
stateid4 stateid; stateid_arg stateid;
struct list_entry *entry;
nfs41_lock_state *lock;
int status; int status;
/* reclaim the open stateid */ /* reclaim the open stateid */
status = nfs41_open_reclaim(session, &open->parent, &open->file, status = nfs41_open_reclaim(session, &open->parent, &open->file,
&open->owner, open->share_access, open->share_deny, &stateid); &open->owner, open->share_access, open->share_deny, &stateid.stateid);
if (status) if (status)
goto out; goto out;
AcquireSRWLockExclusive(&open->lock); AcquireSRWLockExclusive(&open->lock);
/* update the open stateid */ /* update the open stateid */
memcpy(&open->stateid, &stateid, sizeof(stateid4)); memcpy(&open->stateid, &stateid.stateid, sizeof(stateid4));
stateid.type = STATEID_OPEN;
stateid.open = open;
/* recover any locks for this open */
list_for_each(entry, &open->locks.list) {
lock = list_container(entry, nfs41_lock_state, open_entry);
status = nfs41_lock(session, &open->file, &open->owner,
lock->type, lock->offset, lock->length, TRUE, &stateid);
if (status == NFS4ERR_BADSESSION)
break;
}
if (status != NFS4ERR_BADSESSION) {
/* if we got a lock stateid back, save the lock with the open */
if (stateid.type == STATEID_LOCK)
memcpy(&open->locks.stateid, &stateid.stateid, sizeof(stateid4));
}
ReleaseSRWLockExclusive(&open->lock); ReleaseSRWLockExclusive(&open->lock);
out: out:
return status; return status;
@ -295,6 +316,7 @@ restart_recovery:
nfs_argop4 *argop = &compound->args.argarray[ nfs_argop4 *argop = &compound->args.argarray[
compound->res.resarray_count-1]; compound->res.resarray_count-1];
stateid_arg *stateid = NULL; stateid_arg *stateid = NULL;
stateid4 *source = NULL;
if (argop->op == OP_CLOSE) { if (argop->op == OP_CLOSE) {
nfs41_op_close_args *close = (nfs41_op_close_args*)argop->arg; nfs41_op_close_args *close = (nfs41_op_close_args*)argop->arg;
@ -327,6 +349,7 @@ restart_recovery:
switch (stateid->type) { switch (stateid->type) {
case STATEID_OPEN: case STATEID_OPEN:
case STATEID_LOCK:
/* if there's recovery in progress, wait for it to finish */ /* if there's recovery in progress, wait for it to finish */
EnterCriticalSection(&session->client->recovery.lock); EnterCriticalSection(&session->client->recovery.lock);
while (session->client->recovery.in_recovery) while (session->client->recovery.in_recovery)
@ -334,14 +357,20 @@ restart_recovery:
&session->client->recovery.lock, INFINITE); &session->client->recovery.lock, INFINITE);
LeaveCriticalSection(&session->client->recovery.lock); LeaveCriticalSection(&session->client->recovery.lock);
/* if the open stateid is different, update and retry */ if (stateid->type == STATEID_OPEN)
source = &stateid->open->stateid;
else
source = &stateid->open->locks.stateid;
/* if the source stateid is different, update and retry */
AcquireSRWLockShared(&stateid->open->lock); AcquireSRWLockShared(&stateid->open->lock);
if (memcmp(&stateid->stateid, &stateid->open->stateid, sizeof(stateid4))) { if (memcmp(&stateid->stateid, source, sizeof(stateid4))) {
memcpy(&stateid->stateid, &stateid->open->stateid, sizeof(stateid4)); memcpy(&stateid->stateid, source, sizeof(stateid4));
retry = TRUE; retry = TRUE;
} }
ReleaseSRWLockShared(&stateid->open->lock); ReleaseSRWLockShared(&stateid->open->lock);
break; break;
default: default:
eprintf("%s returned %s: can't recover stateid type %u\n", eprintf("%s returned %s: can't recover stateid type %u\n",
nfs_opnum_to_string(argop->op), nfs_opnum_to_string(argop->op),

10
daemon/nfs41_ops.c
View file

@ -866,6 +866,7 @@ int nfs41_lock(
IN uint32_t type, IN uint32_t type,
IN uint64_t offset, IN uint64_t offset,
IN uint64_t length, IN uint64_t length,
IN bool_t reclaim,
IN OUT stateid_arg *stateid) IN OUT stateid_arg *stateid)
{ {
int status; int status;
@ -892,7 +893,7 @@ int nfs41_lock(
compound_add_op(&compound, OP_LOCK, &lock_args, &lock_res); compound_add_op(&compound, OP_LOCK, &lock_args, &lock_res);
lock_args.locktype = type; lock_args.locktype = type;
lock_args.reclaim = 0; lock_args.reclaim = reclaim;
lock_args.offset = offset; lock_args.offset = offset;
lock_args.length = length; lock_args.length = length;
if (stateid->type == STATEID_LOCK) { if (stateid->type == STATEID_LOCK) {
@ -909,11 +910,14 @@ int nfs41_lock(
lock_res.u.resok4.lock_stateid = &stateid->stateid; lock_res.u.resok4.lock_stateid = &stateid->stateid;
lock_res.u.denied.owner.owner_len = NFS4_OPAQUE_LIMIT; lock_res.u.denied.owner.owner_len = NFS4_OPAQUE_LIMIT;
status = compound_encode_send_decode(session, &compound, TRUE); status = compound_encode_send_decode(session, &compound, !reclaim);
if (status) if (status)
goto out; goto out;
compound_error(status = compound.res.status); if (compound_error(status = compound.res.status))
goto out;
stateid->type = STATEID_LOCK; /* returning a lock stateid */
out: out:
return status; return status;
} }

1
daemon/nfs41_ops.h
View file

@ -975,6 +975,7 @@ int nfs41_lock(
IN uint32_t type, IN uint32_t type,
IN uint64_t offset, IN uint64_t offset,
IN uint64_t length, IN uint64_t length,
IN bool_t reclaim,
IN OUT stateid_arg *stateid); IN OUT stateid_arg *stateid);
int nfs41_unlock( int nfs41_unlock(

15
daemon/open.c
View file

@ -62,6 +62,7 @@ static int create_open_state(
state->owner.owner_len = (uint32_t)strlen( state->owner.owner_len = (uint32_t)strlen(
(const char*)state->owner.owner); (const char*)state->owner.owner);
state->ref_count = 1; state->ref_count = 1;
list_init(&state->locks.list);
*state_out = state; *state_out = state;
status = NO_ERROR; status = NO_ERROR;
@ -73,6 +74,18 @@ out_free:
goto out; goto out;
} }
static void open_state_free(
IN nfs41_open_state *state)
{
struct list_entry *entry, *tmp;
/* free associated lock state */
list_for_each_tmp(entry, tmp, &state->locks.list)
free(list_container(entry, nfs41_lock_state, open_entry));
free(state);
}
/* open state reference counting */ /* open state reference counting */
void nfs41_open_state_ref( void nfs41_open_state_ref(
IN nfs41_open_state *state) IN nfs41_open_state *state)
@ -91,7 +104,7 @@ void nfs41_open_state_deref(
dprintf(2, "nfs41_open_state_deref(%s) count %d\n", dprintf(2, "nfs41_open_state_deref(%s) count %d\n",
state->path.path, count); state->path.path, count);
if (count == 0) if (count == 0)
free(state); open_state_free(state);
} }
void nfs41_open_stateid_arg( void nfs41_open_stateid_arg(

2
daemon/pnfs_layout.c
View file

@ -551,7 +551,7 @@ enum pnfs_status pnfs_open_state_layout(
goto out; goto out;
} }
nfs41_open_stateid_arg(state, &stateid); nfs41_lock_stateid_arg(state, &stateid);
/* make sure the layout can satisfy this request */ /* make sure the layout can satisfy this request */
status = file_layout_get(layout, session, &state->file, status = file_layout_get(layout, session, &state->file,