ZFS hot spares

If you have more disks than you need to use in your ZFS pool, then by all means, give them to ZFS to use as “hot spares.”

Here is an experiment to show the process and results:

# zpool create testpool mirror c0t2d0 c0t3d0 mirror c0t4d0 c0t5d0
# zpool status
  pool: testpool
 state: ONLINE
 scan: none requested
config:

        NAME        STATE     READ WRITE CKSUM
        testpool    ONLINE       0     0     0
          mirror-0  ONLINE       0     0     0
            c0t2d0  ONLINE       0     0     0
            c0t3d0  ONLINE       0     0     0
          mirror-1  ONLINE       0     0     0
            c0t4d0  ONLINE       0     0     0
            c0t5d0  ONLINE       0     0     0

errors: No known data errors

# zpool add testpool spare c0t6d0 c0t7d0
# zpool status
  pool: testpool
 state: ONLINE
 scan: none requested
config:

        NAME        STATE     READ WRITE CKSUM
        testpool    ONLINE       0     0     0
          mirror-0  ONLINE       0     0     0
            c0t2d0  ONLINE       0     0     0
            c0t3d0  ONLINE       0     0     0
          mirror-1  ONLINE       0     0     0
            c0t4d0  ONLINE       0     0     0
            c0t5d0  ONLINE       0     0     0
        spares
          c0t6d0    AVAIL
          c0t7d0    AVAIL

errors: No known data errors

# zpool status
  pool: testpool
 state: DEGRADED
status: One or more devices could not be opened.  Sufficient replicas exist for
        the pool to continue functioning in a degraded state.
action: Attach the missing device and online it using 'zpool online'.
   see: http://www.sun.com/msg/ZFS-8000-2Q
 scan: none requested
config:

        NAME        STATE     READ WRITE CKSUM
        testpool    DEGRADED     0     0     0
          mirror-0  DEGRADED     0     0     0
            c0t2d0  UNAVAIL      0     0     0  cannot open
            c0t3d0  ONLINE       0     0     0
          mirror-1  ONLINE       0     0     0
            c0t4d0  ONLINE       0     0     0
            c0t5d0  ONLINE       0     0     0
        spares
          c0t6d0    AVAIL
          c0t7d0    AVAIL

errors: No known data errors

# zpool set autoreplace=on testpool

Oct 28 15:36:16 unknown EVENT-TIME: Fri Oct 28 15:36:16 CDT 2011
Oct 28 15:36:16 unknown PLATFORM: VirtualBox, CSN: 0, HOSTNAME: unknown
Oct 28 15:36:16 unknown SOURCE: zfs-diagnosis, REV: 1.0
Oct 28 15:36:16 unknown EVENT-ID: fb216b01-1dba-ebb6-8404-d7bcfd52bdbe
Oct 28 15:36:16 unknown DESC: A ZFS device failed.  Refer to http://sun.com/msg/ZFS-8000-D3 for more information.
Oct 28 15:36:16 unknown AUTO-RESPONSE: No automated response will occur.
Oct 28 15:36:16 unknown IMPACT: Fault tolerance of the pool may be compromised.
Oct 28 15:36:16 unknown REC-ACTION: Run 'zpool status -x' and replace the bad device.

# zpool status
  pool: testpool
 state: DEGRADED
status: One or more devices could not be opened.  Sufficient replicas exist for
        the pool to continue functioning in a degraded state.
action: Attach the missing device and online it using 'zpool online'.
   see: http://www.sun.com/msg/ZFS-8000-2Q
 scan: resilvered 52.5K in 0h0m with 0 errors on Fri Oct 28 15:36:17 2011
config:

        NAME          STATE     READ WRITE CKSUM
        testpool      DEGRADED     0     0     0
          mirror-0    DEGRADED     0     0     0
            spare-0   DEGRADED     0     0     0
              c0t2d0  UNAVAIL      0     0     0  cannot open
              c0t6d0  ONLINE       0     0     0
            c0t3d0    ONLINE       0     0     0
          mirror-1    ONLINE       0     0     0
            c0t4d0    ONLINE       0     0     0
            c0t5d0    ONLINE       0     0     0
        spares
          c0t6d0      INUSE     currently in use
          c0t7d0      AVAIL

errors: No known data errors

Oct 28 15:41:17 unknown EVENT-TIME: Fri Oct 28 15:36:16 CDT 2011
Oct 28 15:41:17 unknown PLATFORM: VirtualBox, CSN: 0, HOSTNAME: unknown
Oct 28 15:41:17 unknown SOURCE: zfs-diagnosis, REV: 1.0
Oct 28 15:41:17 unknown EVENT-ID: fb216b01-1dba-ebb6-8404-d7bcfd52bdbe
Oct 28 15:41:17 unknown DESC: All faults associated with an event id have been addressed.
Oct 28 15:41:17 unknown   Refer to http://sun.com/msg/FMD-8000-6U for more information.
Oct 28 15:41:17 unknown AUTO-RESPONSE: All system components offlined because of the original fault have been brought back online.
Oct 28 15:41:17 unknown IMPACT: Performance degradation of the system due to the original fault has been recovered.
Oct 28 15:41:17 unknown REC-ACTION: Use fmdump -v -u <EVENT-ID> to identify the repaired components.

# zpool status
  pool: testpool
 state: ONLINE
 scan: resilvered 52.5K in 0h0m with 0 errors on Fri Oct 28 15:36:17 2011
config:

        NAME          STATE     READ WRITE CKSUM
        testpool      ONLINE       0     0     0
          mirror-0    ONLINE       0     0     0
            spare-0   ONLINE       0     0     0
              c0t2d0  ONLINE       0     0     0
              c0t6d0  ONLINE       0     0     0
            c0t3d0    ONLINE       0     0     0
          mirror-1    ONLINE       0     0     0
            c0t4d0    ONLINE       0     0     0
            c0t5d0    ONLINE       0     0     0
        spares
          c0t6d0      INUSE     currently in use
          c0t7d0      AVAIL

errors: No known data errors

# zpool status
  pool: testpool
 state: ONLINE
 scan: resilvered 9.50K in 0h0m with 0 errors on Fri Oct 28 15:42:08 2011
config:

        NAME        STATE     READ WRITE CKSUM
        testpool    ONLINE       0     0     0
          mirror-0  ONLINE       0     0     0
            c0t2d0  ONLINE       0     0     0
            c0t3d0  ONLINE       0     0     0
          mirror-1  ONLINE       0     0     0
            c0t4d0  ONLINE       0     0     0
            c0t5d0  ONLINE       0     0     0
        spares
          c0t6d0    AVAIL
          c0t7d0    AVAIL

errors: No known data errors

As you can see if you add a spare AND you turn on autoreplace, then disks can come and go as they please and ZFS will deal with it by itself.

Obviously this simple disk pulling is a best case scenario. ZFS may or may not perform as admirably if the disk issue is more complex than a simple offline.

Here is the same example taken further:

Oct 28 15:46:29 unknown EVENT-TIME: Fri Oct 28 15:46:29 CDT 2011
Oct 28 15:46:29 unknown PLATFORM: VirtualBox, CSN: 0, HOSTNAME: unknown
Oct 28 15:46:29 unknown SOURCE: zfs-diagnosis, REV: 1.0
Oct 28 15:46:29 unknown EVENT-ID: e42609fe-da81-c68c-bea2-a232a2a5b6d0
Oct 28 15:46:29 unknown DESC: A ZFS device failed.  Refer to http://sun.com/msg/ZFS-8000-D3 for more information.
Oct 28 15:46:29 unknown AUTO-RESPONSE: No automated response will occur.
Oct 28 15:46:29 unknown IMPACT: Fault tolerance of the pool may be compromised.
Oct 28 15:46:29 unknown REC-ACTION: Run 'zpool status -x' and replace the bad device.
Oct 28 15:46:29 unknown fmd: [ID 377184 daemon.error] SUNW-MSG-ID: ZFS-8000-D3, TYPE: Fault, VER: 1, SEVERITY: Major

Oct 28 15:46:29 unknown EVENT-TIME: Fri Oct 28 15:46:29 CDT 2011
Oct 28 15:46:29 unknown PLATFORM: VirtualBox, CSN: 0, HOSTNAME: unknown
Oct 28 15:46:29 unknown SOURCE: zfs-diagnosis, REV: 1.0
Oct 28 15:46:29 unknown EVENT-ID: aaaac038-7699-eff1-aede-800ef60c7408
Oct 28 15:46:29 unknown DESC: A ZFS device failed.  Refer to http://sun.com/msg/ZFS-8000-D3 for more information.
Oct 28 15:46:29 unknown AUTO-RESPONSE: No automated response will occur.
Oct 28 15:46:29 unknown IMPACT: Fault tolerance of the pool may be compromised.
Oct 28 15:46:29 unknown REC-ACTION: Run 'zpool status -x' and replace the bad device.

# zpool status
  pool: testpool
 state: DEGRADED
status: One or more devices could not be opened.  Sufficient replicas exist for
        the pool to continue functioning in a degraded state.
action: Attach the missing device and online it using 'zpool online'.
   see: http://www.sun.com/msg/ZFS-8000-2Q
 scan: resilvered 71.5K in 0h0m with 0 errors on Fri Oct 28 15:46:31 2011
config:

        NAME          STATE     READ WRITE CKSUM
        testpool      DEGRADED     0     0     0
          mirror-0    DEGRADED     0     0     0
            spare-0   DEGRADED     0     0     0
              c0t2d0  UNAVAIL      0     0     0  cannot open
              c0t6d0  ONLINE       0     0     0
            c0t3d0    ONLINE       0     0     0
          mirror-1    DEGRADED     0     0     0
            spare-0   DEGRADED     0     0     0
              c0t4d0  UNAVAIL      0     0     0  cannot open
              c0t7d0  ONLINE       0     0     0
            c0t5d0    ONLINE       0     0     0
        spares
          c0t6d0      INUSE     currently in use
          c0t7d0      INUSE     currently in use

errors: No known data errors

# digest -a md5 -v /testpool/testfile
md5 (/testpool/testfile) = bb449a441fd64200213bf0217b13414b

Oct 28 15:52:46 unknown EVENT-TIME: Fri Oct 28 15:52:46 CDT 2011
Oct 28 15:52:46 unknown PLATFORM: VirtualBox, CSN: 0, HOSTNAME: unknown
Oct 28 15:52:46 unknown SOURCE: zfs-diagnosis, REV: 1.0
Oct 28 15:52:46 unknown EVENT-ID: 12273a2a-c3a5-6ff2-99fc-b68b9c06dcdf
Oct 28 15:52:46 unknown DESC: A ZFS device failed.  Refer to http://sun.com/msg/ZFS-8000-D3 for more information.
Oct 28 15:52:46 unknown AUTO-RESPONSE: No automated response will occur.
Oct 28 15:52:46 unknown IMPACT: Fault tolerance of the pool may be compromised.
Oct 28 15:52:46 unknown REC-ACTION: Run 'zpool status -x' and replace the bad device.
Oct 28 15:52:46 unknown fmd: [ID 377184 daemon.error] SUNW-MSG-ID: ZFS-8000-D3, TYPE: Fault, VER: 1, SEVERITY: Major

Oct 28 15:52:46 unknown EVENT-TIME: Fri Oct 28 15:52:46 CDT 2011
Oct 28 15:52:46 unknown PLATFORM: VirtualBox, CSN: 0, HOSTNAME: unknown
Oct 28 15:52:46 unknown SOURCE: zfs-diagnosis, REV: 1.0
Oct 28 15:52:46 unknown EVENT-ID: 9a80e839-c349-e246-e6e2-915bc0718bc1
Oct 28 15:52:46 unknown DESC: A ZFS device failed.  Refer to http://sun.com/msg/ZFS-8000-D3 for more information.
Oct 28 15:52:46 unknown AUTO-RESPONSE: No automated response will occur.
Oct 28 15:52:46 unknown IMPACT: Fault tolerance of the pool may be compromised.
Oct 28 15:52:46 unknown REC-ACTION: Run 'zpool status -x' and replace the bad device.

# zpool status
  pool: testpool
 state: DEGRADED
status: One or more devices could not be opened.  Sufficient replicas exist for
        the pool to continue functioning in a degraded state.
action: Attach the missing device and online it using 'zpool online'.
   see: http://www.sun.com/msg/ZFS-8000-2Q
 scan: resilvered 71.5K in 0h0m with 0 errors on Fri Oct 28 15:46:31 2011
config:

        NAME          STATE     READ WRITE CKSUM
        testpool      DEGRADED     0     0     0
          mirror-0    DEGRADED     0     0     0
            spare-0   DEGRADED     0     0     0
              c0t2d0  UNAVAIL      0     0     0  cannot open
              c0t6d0  ONLINE       0     0     0
            c0t3d0    UNAVAIL      0     0     0  cannot open
          mirror-1    DEGRADED     0     0     0
            spare-0   DEGRADED     0     0     0
              c0t4d0  UNAVAIL      0     0     0  cannot open
              c0t7d0  ONLINE       0     0     0
            c0t5d0    UNAVAIL      0     0     0  cannot open
        spares
          c0t6d0      INUSE     currently in use
          c0t7d0      INUSE     currently in use

errors: No known data errors

# digest -a md5 -v /testpool/testfile
md5 (/testpool/testfile) = bb449a441fd64200213bf0217b13414b

Now we have lost all four original drives, but the file system content is still in tact.

# zpool status
  pool: testpool
 state: ONLINE
 scan: resilvered 71.5K in 0h0m with 0 errors on Fri Oct 28 15:46:31 2011
config:

        NAME          STATE     READ WRITE CKSUM
        testpool      ONLINE       0     0     0
          mirror-0    ONLINE       0     0     0
            spare-0   ONLINE       0     0     0
              c0t2d0  ONLINE       0     0     0
              c0t6d0  ONLINE       0     0     0
            c0t3d0    ONLINE       0     0     0
          mirror-1    ONLINE       0     0     0
            spare-0   ONLINE       0     0     0
              c0t4d0  ONLINE       0     0     0
              c0t7d0  ONLINE       0     0     0
            c0t5d0    ONLINE       0     0     0
        spares
          c0t6d0      INUSE     currently in use
          c0t7d0      INUSE     currently in use

errors: No known data errors

# zpool status
  pool: testpool
 state: ONLINE
 scan: resilvered 354K in 0h0m with 0 errors on Fri Oct 28 15:57:19 2011
config:

        NAME        STATE     READ WRITE CKSUM
        testpool    ONLINE       0     0     0
          mirror-0  ONLINE       0     0     0
            c0t2d0  ONLINE       0     0     0
            c0t3d0  ONLINE       0     0     0
          mirror-1  ONLINE       0     0     0
            c0t4d0  ONLINE       0     0     0
            c0t5d0  ONLINE       0     0     0
        spares
          c0t6d0    AVAIL
          c0t7d0    AVAIL

errors: No known data errors

# digest -a md5 -v /testpool/testfile
md5 (/testpool/testfile) = bb449a441fd64200213bf0217b13414b

Put the disks back, and everything returns to normal all by itself, with no loss of file system integrity.

Obviously if we had not had time for the hot spare to silver before the next disk went in it would not have worked.

In this particular setup, it would probably actually make more sense to just mirror across three disks, so that you could lose two at once and still survive.

Also RAIDZ or RAIDZ2 could be used, allowing you to lose 1 or 2 disks respectively, and you can also use hot spares with RAIDZ.

# zpool destroy testpool
# zpool create testpool raidz2 c0t2d0 c0t3d0 c0t4d0 c0t5d0 c0t6d0
# zpool add testpool spare c0t7d0
# zpool status
  pool: testpool
 state: ONLINE
 scan: none requested
config:

        NAME        STATE     READ WRITE CKSUM
        testpool    ONLINE       0     0     0
          raidz2-0  ONLINE       0     0     0
            c0t2d0  ONLINE       0     0     0
            c0t3d0  ONLINE       0     0     0
            c0t4d0  ONLINE       0     0     0
            c0t5d0  ONLINE       0     0     0
            c0t6d0  ONLINE       0     0     0
        spares
          c0t7d0    AVAIL

errors: No known data errors
# zpool set autoreplace=on testpool
# cp /var/adm/messages /testpool/testfile
# digest -a md5 -v /testpool/testfile
md5 (/testpool/testfile) = 336cd7cbdf0926fbf04392a3ed9b512b

# Remove TWO disks at once
# RAIDZ2 keeps two parity disks, so it can lose two at once

# zpool status
  pool: testpool
 state: DEGRADED
status: One or more devices could not be opened.  Sufficient replicas exist for
        the pool to continue functioning in a degraded state.
action: Attach the missing device and online it using 'zpool online'.
   see: http://www.sun.com/msg/ZFS-8000-2Q
 scan: resilvered 166K in 0h0m with 0 errors on Fri Oct 28 16:07:58 2011
config:

        NAME          STATE     READ WRITE CKSUM
        testpool      DEGRADED     0     0     0
          raidz2-0    DEGRADED     0     0     0
            spare-0   DEGRADED     0     0     0
              c0t2d0  UNAVAIL      0     0     0  cannot open
              c0t7d0  ONLINE       0     0     0
            c0t3d0    UNAVAIL      0     0     0  cannot open
            c0t4d0    ONLINE       0     0     0
            c0t5d0    ONLINE       0     0     0
            c0t6d0    ONLINE       0     0     0
        spares
          c0t7d0      INUSE     currently in use

errors: No known data errors

# digest -a md5 -v /testpool/testfile
md5 (/testpool/testfile) = 336cd7cbdf0926fbf04392a3ed9b512b

# Data is still good and the spare has stepped in for one parity disk

# zpool status
  pool: testpool
 state: DEGRADED
status: One or more devices could not be opened.  Sufficient replicas exist for
        the pool to continue functioning in a degraded state.
action: Attach the missing device and online it using 'zpool online'.
   see: http://www.sun.com/msg/ZFS-8000-2Q
 scan: resilvered 166K in 0h0m with 0 errors on Fri Oct 28 16:07:58 2011
config:

        NAME          STATE     READ WRITE CKSUM
        testpool      DEGRADED     0     0     0
          raidz2-0    DEGRADED     0     0     0
            spare-0   DEGRADED     0     0     0
              c0t2d0  UNAVAIL      0     0     0  cannot open
              c0t7d0  ONLINE       0     0     0
            c0t3d0    UNAVAIL      0     0     0  cannot open
            c0t4d0    UNAVAIL      0     0     0  cannot open
            c0t5d0    ONLINE       0     0     0
            c0t6d0    ONLINE       0     0     0
        spares
          c0t7d0      INUSE     currently in use

errors: No known data errors

# Third disk gone, but still ok, as long as ZFS had time to rebuild the parity disk before it went away

# digest -a md5 -v /testpool/testfile
md5 (/testpool/testfile) = 336cd7cbdf0926fbf04392a3ed9b512b

# zpool status
  pool: testpool
 state: UNAVAIL
status: One or more devices could not be opened.  There are insufficient
        replicas for the pool to continue functioning.
action: Attach the missing device and online it using 'zpool online'.
   see: http://www.sun.com/msg/ZFS-8000-3C
 scan: none requested
config:

        NAME          STATE     READ WRITE CKSUM
        testpool      UNAVAIL      0     0     0  insufficient replicas
          raidz2-0    UNAVAIL      0     0     0  insufficient replicas
            spare-0   DEGRADED     0     0     0
              c0t2d0  UNAVAIL      0     0     0  cannot open
              c0t7d0  ONLINE       0     0     0
            c0t3d0    UNAVAIL      0     0     0  cannot open
            c0t4d0    UNAVAIL      0     0     0  cannot open
            c0t5d0    UNAVAIL      0     0     0  cannot open
            c0t6d0    ONLINE       0     0     0

# Ok, now we went too far. The volume is now offline due to "insufficient replicas"
# There is no no data mounted at /testpool

# But, in case you wonder, yes, if you just put the drives all back, ZFS will figure it out!

# zpool status
  pool: testpool
 state: ONLINE
 scan: resilvered 52.5K in 0h0m with 0 errors on Fri Oct 28 16:19:38 2011
config:

        NAME        STATE     READ WRITE CKSUM
        testpool    ONLINE       0     0     0
          raidz2-0  ONLINE       0     0     0
            c0t2d0  ONLINE       0     0     0
            c0t3d0  ONLINE       0     0     0
            c0t4d0  ONLINE       0     0     0
            c0t5d0  ONLINE       0     0     0
            c0t6d0  ONLINE       0     0     0
        spares
          c0t7d0    AVAIL

errors: No known data errors
# digest -a md5 -v /testpool/testfile
md5 (/testpool/testfile) = 336cd7cbdf0926fbf04392a3ed9b512b

# That won't help if two non spares plus all non spares truly go bad, but if you just make a mistake while swapping drives, you should still be able to recover.

And all of that was mostly just for my own entertainment, and also so I’ll have something to look at the next time I have a ZFS problem and forget how it works.

This entry was posted in Uncategorized. Bookmark the permalink.

Leave a Reply

Your email address will not be published. Required fields are marked *