pnfs: added status flags and ref count to struct pnfs_device

pnfs_device.status remembers whether a given device has been GRANTED/REVOKED

pnfs_device.layout_count tracks the number of layouts using the device, incremented by pnfs_file_device_get() and decremented by pnfs_file_device_put().  when pnfs_file_device_put() takes layout_count to 0, remove and free the device only if it's flagged as REVOKED

because pnfs_file_device_get() modifies pnfs_device.layout_count, we can no longer use a shared lock; changed pnfs_file_device.lock from SRWLOCK to CRITICAL_SECTION, and moved to pnfs_device.lock to document the fact that it's used for pnfs_device.status and pnfs_device.layout_count

Signed-off-by: Casey Bodley <cbodley@citi.umich.edu>

daemon/pnfs.h

@@ -99,6 +99,13 @@ enum pnfs_layout_status {
     PNFS_LAYOUT_NOT_RW      = 0x20,
 };
 
+enum pnfs_device_status {
+    /* GETDEVICEINFO was successful */
+    PNFS_DEVICE_GRANTED     = 0x1,
+    /* a bulk recall or lease expiration led to device invalidation */
+    PNFS_DEVICE_REVOKED     = 0x2,
+};
+
 enum pnfs_return_type {
     PNFS_RETURN_FILE        = 1,
     PNFS_RETURN_FSID        = 2,
@@ -117,6 +124,9 @@ enum pnfs_return_type {
 typedef struct __pnfs_device {
     unsigned char           deviceid[PNFS_DEVICEID_SIZE];
     enum pnfs_layout_type   type;
+    enum pnfs_device_status status;
+    uint32_t                layout_count; /* layouts using this device */
+    CRITICAL_SECTION        lock;
 } pnfs_device;
 
 typedef struct __pnfs_stripe_indices {
@@ -139,8 +149,8 @@ typedef struct __pnfs_file_device {
     pnfs_device             device;
     pnfs_stripe_indices     stripes;
     pnfs_data_server_list   servers;
-    struct list_entry       entry; /* position in nfs41_client.devices */
+    struct pnfs_file_device_list *devices; /* -> nfs41_client.devices */
-    SRWLOCK                 lock;
+    struct list_entry       entry; /* position in devices */
 } pnfs_file_device;
 
 
@@ -290,6 +300,9 @@ enum pnfs_status pnfs_file_device_get(
     IN unsigned char *deviceid,
     OUT pnfs_file_device **device_out);
 
+void pnfs_file_device_put(
+    IN pnfs_file_device *device);
+
 enum pnfs_status pnfs_data_server_client(
     IN struct __nfs41_root *root,
     IN pnfs_data_server *server,

daemon/pnfs_device.c

@@ -43,6 +43,7 @@ struct pnfs_file_device_list {
 
 static enum pnfs_status file_device_create(
     IN const unsigned char *deviceid,
+    IN struct pnfs_file_device_list *devices,
     OUT pnfs_file_device **device_out)
 {
     enum pnfs_status status = PNFS_SUCCESS;
@@ -55,7 +56,8 @@ static enum pnfs_status file_device_create(
     }
 
     memcpy(device->device.deviceid, deviceid, PNFS_DEVICEID_SIZE);
-    InitializeSRWLock(&device->lock);
+    device->devices = devices;
+    InitializeCriticalSection(&device->device.lock);
     *device_out = device;
 out:
     return status;
@@ -94,7 +96,7 @@ static enum pnfs_status file_device_find_or_create(
     if (entry == NULL) {
         /* create a new device */
         pnfs_file_device *device;
-        status = file_device_create(deviceid, &device);
+        status = file_device_create(deviceid, devices, &device);
         if (status == PNFS_SUCCESS) {
             /* add it to the list */
             list_add_tail(&devices->head, &device->entry);
@@ -172,39 +174,66 @@ enum pnfs_status pnfs_file_device_get(
     if (status)
         goto out;
 
-    AcquireSRWLockShared(&device->lock);
+    EnterCriticalSection(&device->device.lock);
-    status = device->device.type == 0 ? PNFS_PENDING : PNFS_SUCCESS;
-    ReleaseSRWLockShared(&device->lock);
 
-    if (status == PNFS_PENDING) {
+    /* don't give out a device that's been revoked */
-        AcquireSRWLockExclusive(&device->lock);
+    if (device->device.status & PNFS_DEVICE_REVOKED)
+        status = PNFSERR_NO_DEVICE;
+    else if (device->device.status & PNFS_DEVICE_GRANTED)
+        status = PNFS_SUCCESS;
+    else {
+        nfsstat = pnfs_rpc_getdeviceinfo(session, deviceid, device);
+        if (nfsstat == NFS4_OK) {
+            device->device.status = PNFS_DEVICE_GRANTED;
+            status = PNFS_SUCCESS;
 
-        status = device->device.type == 0 ? PNFS_PENDING : PNFS_SUCCESS;
+            dprintf(FDLVL, "Received device info:\n");
-        if (status == PNFS_PENDING) {
+            dprint_device(FDLVL, device);
-            nfsstat = pnfs_rpc_getdeviceinfo(session, deviceid, device);
+        } else {
-            if (nfsstat == NFS4_OK) {
+            status = PNFSERR_NO_DEVICE;
-                status = PNFS_SUCCESS;
 
-                dprintf(FDLVL, "Received device info:\n");
+            eprintf("pnfs_rpc_getdeviceinfo() failed with %s\n",
-                dprint_device(FDLVL, device);
+                nfs_error_string(nfsstat));
-            } else {
-                status = PNFSERR_NO_DEVICE;
-
-                eprintf("pnfs_rpc_getdeviceinfo() failed with %s\n",
-                    nfs_error_string(nfsstat));
-            }
         }
-
-        ReleaseSRWLockExclusive(&device->lock);
     }
 
-    *device_out = device;
+    if (status == PNFS_SUCCESS) {
+        device->device.layout_count++;
+        dprintf(FDLVL, "pnfs_file_device_get() -> %u\n",
+            device->device.layout_count);
+        *device_out = device;
+    }
+
+    LeaveCriticalSection(&device->device.lock);
 out:
     dprintf(FDLVL, "<-- pnfs_file_device_get() returning %s\n",
         pnfs_error_string(status));
     return status;
 }
 
+void pnfs_file_device_put(
+    IN pnfs_file_device *device)
+{
+    uint32_t count;
+    EnterCriticalSection(&device->device.lock);
+    count = --device->device.layout_count;
+    dprintf(FDLVL, "pnfs_file_device_put() -> %u\n", count);
+
+    /* if the device was revoked, remove/free the device on last reference */
+    if (count == 0 && device->device.status & PNFS_DEVICE_REVOKED) {
+        EnterCriticalSection(&device->devices->lock);
+        list_remove(&device->entry);
+        LeaveCriticalSection(&device->devices->lock);
+
+        LeaveCriticalSection(&device->device.lock);
+
+        file_device_free(device);
+        dprintf(FDLVL, "revoked file device freed after last reference\n");
+    } else {
+        LeaveCriticalSection(&device->device.lock);
+    }
+}
+
 static enum pnfs_status data_client_status(
     IN pnfs_data_server *server,
     OUT nfs41_client **client_out)

daemon/pnfs_layout.c

@@ -65,6 +65,7 @@ out:
 static void layout_free(
     IN pnfs_file_layout *layout)
 {
+    if (layout->device) pnfs_file_device_put(layout->device);
     free(layout->filehandles.arr);
     free(layout);
 }