From 9cd97445670e8878f13b2804511689afdee8f3f9 Mon Sep 17 00:00:00 2001 From: Casey Bodley Date: Thu, 2 Dec 2010 10:32:15 -0500 Subject: [PATCH] pnfs: revoke device info on bulk layout recall 20.3. CB_LAYOUTRECALL "LAYOUTRECALL4_FSID and LAYOUTRECALL4_ALL specify that all the storage device ID to storage device address mappings in the affected file system(s) are also recalled." pnfs_file_layout_recall() now takes a nfs41_client instead of just the pnfs_file_layout_list, because both the layout list and device list are accessible from nfs41_client. for bulk recalls, calls new function pnfs_file_device_list_invalidate(). each device with layout_count=0 is removed and freed, and devices in use are flagged as REVOKED and freed when layout_count->0 layout_recall_return() now takes a pnfs_file_layout instead of pnfs_layout for access to pnfs_file_layout.device. pnfs_layout_io_start() and pnfs_layout_io_finish() do the same, because pnfs_layout_io_finish() calls layout_recall_return(). layout_recall_return() calls pnfs_file_device_put() to release its reference on the device Signed-off-by: Casey Bodley --- daemon/callback_server.c | 2 +- daemon/pnfs.h | 9 +++-- daemon/pnfs_device.c | 31 ++++++++++++++++ daemon/pnfs_io.c | 6 ++-- daemon/pnfs_layout.c | 78 +++++++++++++++++++++++----------------- 5 files changed, 86 insertions(+), 40 deletions(-) diff --git a/daemon/callback_server.c b/daemon/callback_server.c index 62fd788..cc89f1c 100644 --- a/daemon/callback_server.c +++ b/daemon/callback_server.c @@ -50,7 +50,7 @@ static enum_t handle_cb_layoutrecall( pnfs_iomode_string(args->iomode), args->recall.type, nfs_error_string(res->status)); - status = pnfs_file_layout_recall(rpc_clnt->client->layouts, args); + status = pnfs_file_layout_recall(rpc_clnt->client, args); if (status) eprintf("pnfs_file_layout_recall() failed with %s\n", pnfs_error_string(status)); diff --git a/daemon/pnfs.h b/daemon/pnfs.h index d23a13f..6e826b3 100644 --- a/daemon/pnfs.h +++ b/daemon/pnfs.h @@ -258,14 +258,14 @@ void pnfs_open_state_close( IN bool_t remove); enum pnfs_status pnfs_file_layout_recall( - IN struct pnfs_file_layout_list *layouts, + IN struct __nfs41_client *client, IN const struct cb_layoutrecall_args *recall); enum pnfs_status pnfs_layout_io_start( - IN pnfs_layout *layout); + IN pnfs_file_layout *layout); void pnfs_layout_io_finished( - IN pnfs_layout *layout); + IN pnfs_file_layout *layout); __inline int is_dense( @@ -294,6 +294,9 @@ enum pnfs_status pnfs_file_device_list_create( void pnfs_file_device_list_free( IN struct pnfs_file_device_list *devices); +void pnfs_file_device_list_invalidate( + IN struct pnfs_file_device_list *devices); + enum pnfs_status pnfs_file_device_get( IN struct __nfs41_session *session, IN struct pnfs_file_device_list *devices, diff --git a/daemon/pnfs_device.c b/daemon/pnfs_device.c index 60f49f7..0713112 100644 --- a/daemon/pnfs_device.c +++ b/daemon/pnfs_device.c @@ -156,6 +156,37 @@ void pnfs_file_device_list_free( free(devices); } +void pnfs_file_device_list_invalidate( + IN struct pnfs_file_device_list *devices) +{ + struct list_entry *entry, *tmp; + pnfs_file_device *device; + + dprintf(FDLVL, "--> pnfs_file_device_list_invalidate()\n"); + + EnterCriticalSection(&devices->lock); + + list_for_each_tmp(entry, tmp, &devices->head) { + device = device_entry(entry); + EnterCriticalSection(&device->device.lock); + /* if there are layouts still using the device, flag it + * as revoked and clean up on last reference */ + if (device->device.layout_count) { + device->device.status |= PNFS_DEVICE_REVOKED; + LeaveCriticalSection(&device->device.lock); + } else { + LeaveCriticalSection(&device->device.lock); + /* no layouts are using it, so it's safe to free */ + list_remove(entry); + file_device_free(device); + } + } + + LeaveCriticalSection(&devices->lock); + + dprintf(FDLVL, "<-- pnfs_file_device_list_invalidate()\n"); +} + /* pnfs_file_device */ enum pnfs_status pnfs_file_device_get( diff --git a/daemon/pnfs_io.c b/daemon/pnfs_io.c index a02df08..b952154 100644 --- a/daemon/pnfs_io.c +++ b/daemon/pnfs_io.c @@ -58,7 +58,7 @@ static enum pnfs_status pattern_init( enum pnfs_status status; /* take a reference on the layout so we don't return it during io */ - status = pnfs_layout_io_start(&layout->layout); + status = pnfs_layout_io_start(layout); if (status) goto out; @@ -70,7 +70,7 @@ static enum pnfs_status pattern_init( pattern->threads = calloc(pattern->count, sizeof(pnfs_io_thread)); if (pattern->threads == NULL) { status = PNFSERR_RESOURCES; - pnfs_layout_io_finished(&pattern->layout->layout); + pnfs_layout_io_finished(pattern->layout); goto out; } @@ -115,7 +115,7 @@ static void pattern_free( IN pnfs_io_pattern *pattern) { /* inform the layout that our io is finished */ - pnfs_layout_io_finished(&pattern->layout->layout); + pnfs_layout_io_finished(pattern->layout); free(pattern->threads); } diff --git a/daemon/pnfs_layout.c b/daemon/pnfs_layout.c index 95ea2a6..b3b9fb7 100644 --- a/daemon/pnfs_layout.c +++ b/daemon/pnfs_layout.c @@ -609,14 +609,20 @@ void pnfs_open_state_close( /* expects the caller to have an exclusive lock */ static enum pnfs_status layout_recall_return( - IN pnfs_layout *layout) + IN pnfs_file_layout *layout) { dprintf(FLLVL, "layout_recall_return() 'forgetting' layout\n"); + /* release our reference on the device */ + if (layout->device) { + pnfs_file_device_put(layout->device); + layout->device = NULL; + } + /* since we're forgetful, we don't actually return the layout; * just zero the stateid since it won't be valid anymore */ - ZeroMemory(&layout->state, sizeof(layout->state)); - layout->status = 0; + ZeroMemory(&layout->layout.state, sizeof(layout->layout.state)); + layout->layout.status = 0; return PNFS_SUCCESS; } @@ -632,7 +638,7 @@ static enum pnfs_status file_layout_recall( if (layout->layout.io_count == 0) { /* if there is no pending io, return the layout now */ - status = layout_recall_return(&layout->layout); + status = layout_recall_return(layout); } else { /* flag the layout as recalled so it can be returned after io */ layout->layout.status |= PNFS_LAYOUT_RECALLED; @@ -649,7 +655,7 @@ static enum pnfs_status file_layout_recall( } static enum pnfs_status file_layout_recall_file( - IN struct pnfs_file_layout_list *layouts, + IN nfs41_client *client, IN const struct cb_layoutrecall_args *recall) { struct list_entry *entry; @@ -657,13 +663,13 @@ static enum pnfs_status file_layout_recall_file( dprintf(FLLVL, "--> file_layout_recall_file()\n"); - EnterCriticalSection(&layouts->lock); + EnterCriticalSection(&client->layouts->lock); - status = layout_entry_find(layouts, &recall->recall.args.file.fh, &entry); + status = layout_entry_find(client->layouts, &recall->recall.args.file.fh, &entry); if (status == PNFS_SUCCESS) status = file_layout_recall(layout_entry(entry), recall); - LeaveCriticalSection(&layouts->lock); + LeaveCriticalSection(&client->layouts->lock); dprintf(FLLVL, "<-- file_layout_recall_file() returning %s\n", pnfs_error_string(status)); @@ -679,7 +685,7 @@ static bool_t fsid_matches( } static enum pnfs_status file_layout_recall_fsid( - IN struct pnfs_file_layout_list *layouts, + IN nfs41_client *client, IN const struct cb_layoutrecall_args *recall) { struct list_entry *entry; @@ -690,9 +696,9 @@ static enum pnfs_status file_layout_recall_fsid( dprintf(FLLVL, "--> file_layout_recall_fsid(%llu, %llu)\n", recall->recall.args.fsid.major, recall->recall.args.fsid.minor); - EnterCriticalSection(&layouts->lock); + EnterCriticalSection(&client->layouts->lock); - list_for_each(entry, &layouts->head) { + list_for_each(entry, &client->layouts->head) { layout = layout_entry(entry); /* no locks needed to read layout.meta_fh or superblock.fsid, * because they are only written once on creation */ @@ -701,7 +707,10 @@ static enum pnfs_status file_layout_recall_fsid( status = file_layout_recall(layout, recall); } - LeaveCriticalSection(&layouts->lock); + LeaveCriticalSection(&client->layouts->lock); + + /* bulk recalls require invalidation of cached device info */ + pnfs_file_device_list_invalidate(client->devices); dprintf(FLLVL, "<-- file_layout_recall_fsid() returning %s\n", pnfs_error_string(status)); @@ -709,7 +718,7 @@ static enum pnfs_status file_layout_recall_fsid( } static enum pnfs_status file_layout_recall_all( - IN struct pnfs_file_layout_list *layouts, + IN nfs41_client *client, IN const struct cb_layoutrecall_args *recall) { struct list_entry *entry; @@ -717,12 +726,15 @@ static enum pnfs_status file_layout_recall_all( dprintf(FLLVL, "--> file_layout_recall_all()\n"); - EnterCriticalSection(&layouts->lock); + EnterCriticalSection(&client->layouts->lock); - list_for_each(entry, &layouts->head) + list_for_each(entry, &client->layouts->head) status = file_layout_recall(layout_entry(entry), recall); - LeaveCriticalSection(&layouts->lock); + LeaveCriticalSection(&client->layouts->lock); + + /* bulk recalls require invalidation of cached device info */ + pnfs_file_device_list_invalidate(client->devices); dprintf(FLLVL, "<-- file_layout_recall_all() returning %s\n", pnfs_error_string(status)); @@ -730,7 +742,7 @@ static enum pnfs_status file_layout_recall_all( } enum pnfs_status pnfs_file_layout_recall( - IN struct pnfs_file_layout_list *layouts, + IN nfs41_client *client, IN const struct cb_layoutrecall_args *recall) { enum pnfs_status status = PNFS_SUCCESS; @@ -748,13 +760,13 @@ enum pnfs_status pnfs_file_layout_recall( switch (recall->recall.type) { case PNFS_RETURN_FILE: - status = file_layout_recall_file(layouts, recall); + status = file_layout_recall_file(client, recall); break; case PNFS_RETURN_FSID: - status = file_layout_recall_fsid(layouts, recall); + status = file_layout_recall_fsid(client, recall); break; case PNFS_RETURN_ALL: - status = file_layout_recall_all(layouts, recall); + status = file_layout_recall_all(client, recall); break; default: @@ -773,45 +785,45 @@ out: enum pnfs_status pnfs_layout_io_start( - IN pnfs_layout *layout) + IN pnfs_file_layout *layout) { enum pnfs_status status = PNFS_SUCCESS; - AcquireSRWLockExclusive(&layout->lock); + AcquireSRWLockExclusive(&layout->layout.lock); - if ((layout->status & PNFS_LAYOUT_RECALLED) != 0) { + if ((layout->layout.status & PNFS_LAYOUT_RECALLED) != 0) { /* don't start any more io if the layout has been recalled */ status = PNFSERR_LAYOUT_RECALLED; dprintf(FLLVL, "pnfs_layout_io_start() failed, layout was recalled\n"); } else { /* take a reference on the layout, so that it won't be recalled * until all io is finished */ - layout->io_count++; + layout->layout.io_count++; dprintf(FLLVL, "pnfs_layout_io_start(): count -> %u\n", - layout->io_count); + layout->layout.io_count); } - ReleaseSRWLockExclusive(&layout->lock); + ReleaseSRWLockExclusive(&layout->layout.lock); return status; } void pnfs_layout_io_finished( - IN pnfs_layout *layout) + IN pnfs_file_layout *layout) { - AcquireSRWLockExclusive(&layout->lock); + AcquireSRWLockExclusive(&layout->layout.lock); /* return the reference to signify that an io request is finished */ - layout->io_count--; + layout->layout.io_count--; dprintf(FLLVL, "pnfs_layout_io_finished() count -> %u\n", - layout->io_count); + layout->layout.io_count); - if (layout->io_count > 0) /* more io pending */ + if (layout->layout.io_count > 0) /* more io pending */ goto out_unlock; /* once all io is finished, check for layout recalls */ - if (layout->status & PNFS_LAYOUT_RECALLED) + if (layout->layout.status & PNFS_LAYOUT_RECALLED) layout_recall_return(layout); out_unlock: - ReleaseSRWLockExclusive(&layout->lock); + ReleaseSRWLockExclusive(&layout->layout.lock); }