3

I have an existing Fiber Channel Storage Array - SGI TP9300 (its an LSI product) it's all set up with LUNS and I have a LVM managing it with two Ubuntu 10.04 servers accessing the array. The Servers and the Array support multiple Fibre connections - there are two controllers in the Disk Array each with two connectors.

Is it possible to 'add multipath' to this set up without losing the existing LVM structures - and/or data - or do I have to back all the data and create the multipath devices and then add these to the LVM and then restore my data.

1

3 Answers 3

1

I installed multipath-tools etc and set the following values in /etc/lvm/lvm.conf

devices { ...

types = [ "device-mapper", 1]

# By default we accept every block device: #filter = [ "a/./" ] filter = [ "r|/dev/sd.|", "r|/dev/./by-path/.|", "r|/dev/./by-id/.|", "a/.*/" ]

...

}

and saw some good notes on: http://plone.uconn.edu/Members/jar02014/linux-lvm-and-multipath-configration

Here are the outputs of the various query tools

ase_prim is one of my volume groups:

dmsetup table | sort | grep ase_prim

ase_prim-data01: 0 20971520 linear 8:32 2097536
ase_prim-data02: 0 20971520 linear 8:32 23069056
ase_prim-data03: 0 20971520 linear 8:32 44040576
ase_prim-data04: 0 20971520 linear 8:32 65012096
ase_prim-data05: 0 20971520 linear 8:32 85983616
ase_prim-data06: 0 20971520 linear 8:32 106955136
ase_prim-idx01: 0 20971520 linear 8:32 127926656
ase_prim-idx02: 0 20971520 linear 8:32 148898176
ase_prim-idx03: 0 20971520 linear 8:32 169869696
ase_prim-idx04: 0 20971520 linear 8:32 190841216
ase_prim-idx05: 0 20971520 linear 8:32 211812736
ase_prim-idx06: 0 20971520 linear 8:32 232784256
ase_prim_log-log01: 0 4194304 linear 65:128 384
ase_prim_log-log02: 0 4194304 linear 65:128 4194688
ase_prim_log-log03: 0 4194304 linear 65:128 8388992
ase_prim_log-log04: 0 4194304 linear 65:128 12583296
ase_prim-master: 0 524288 linear 8:32 384
ase_prim-model: 0 524288 linear 8:32 1573248
ase_prim-spare: 0 16777216 linear 8:32 274727296
ase_prim-sybsystemdb: 0 524288 linear 8:32 1048960
ase_prim-sybsystemprocs: 0 524288 linear 8:32 524672
ase_prim-tmp01: 0 4194304 linear 8:32 253755776
ase_prim-tmp02: 0 4194304 linear 8:32 257950080
ase_prim-tmp03: 0 4194304 linear 8:32 262144384
ase_prim-tmp04: 0 4194304 linear 8:32 266338688
ase_prim-tmp05: 0 4194304 linear 8:32 270532992


dmsetup table | sort | grep multipath

3600a0b800018f18300002f514b9e8967: 0 62914560 multipath 0 0 1 1 round-robin 0 2 1 8:64 1000 8:96 1000
3600a0b800018f18300002f564b9e89c1: 0 230686720 multipath 0 0 1 1 round-robin 0 2 1 8:128 1000 8:144 1000
3600a0b800018f18300002f5b4b9e89f7: 0 83886080 multipath 0 0 1 1 round-robin 0 2 1 8:208 1000 8:192 1000
3600a0b800018f18300002f604b9e8a51: 0 41943040 multipath 0 0 1 1 round-robin 0 2 1 65:0 1000 65:16 1000
3600a0b800018f18300002f654b9e8aa3: 0 285700096 multipath 0 0 1 1 round-robin 0 2 1 65:80 1000 65:64 1000
3600a0b800018f18300002f724b9e8bb9: 0 41943040 multipath 0 0 1 1 round-robin 0 2 1 65:128 1000 65:144 1000
3600a0b800018f18300002f774b9e8bf1: 0 209715200 multipath 0 0 1 1 round-robin 0 2 1 65:192 1000 65:240 1000
3600a0b800018f18300002f7c4b9e8c33: 0 41943040 multipath 0 0 1 1 round-robin 0 2 1 65:224 1000 66:64 1000
3600a0b800018f18300002f814b9e8c6f: 0 261996544 multipath 0 0 1 1 round-robin 0 2 1 66:16 1000 66:96 1000
3600a0b800018f453000031af4b9e8afe: 0 293601280 multipath 0 0 1 1 round-robin 0 2 1 8:32 1000 8:48 1000
3600a0b800018f453000031b64b9e8b4e: 0 314572800 multipath 0 0 1 1 round-robin 0 2 1 8:80 1000 8:112 1000
3600a0b800018f453000031bc4b9e8b98: 0 104857600 multipath 0 0 1 1 round-robin 0 2 1 8:160 1000 8:176 1000
3600a0b800018f453000031c24b9e8bd4: 0 41943040 multipath 0 0 1 1 round-robin 0 2 1 8:224 1000 8:240 1000
3600a0b800018f453000031c84b9e8c20: 0 254095360 multipath 0 0 1 1 round-robin 0 2 1 65:32 1000 65:48 1000
3600a0b800018f453000031d84b9e8d56: 0 83886080 multipath 0 0 1 1 round-robin 0 2 1 65:96 1000 65:112 1000
3600a0b800018f453000031df4b9e8d8e: 0 62914560 multipath 0 0 1 1 round-robin 0 2 1 65:176 1000 65:160 1000
3600a0b800018f453000031e54b9e8dce: 0 230686720 multipath 0 0 1 1 round-robin 0 2 1 66:48 1000 65:208 1000
3600a0b800018f453000031eb4b9e8e08: 0 209715200 multipath 0 0 1 1 round-robin 0 2 1 66:0 1000 66:80 1000
3600a0b800018f453000031f14b9e8e66: 0 285700096 multipath 0 0 1 1 round-robin 0 2 1 66:32 1000 66:112 1000


dmsetup table | grep multipath | awk -F: '{print "/dev/mapper/"$1}' | xargs ls -l
brw-rw---- 1 root disk 251, 56 2012-05-18 19:08 /dev/mapper/3600a0b800018f18300002f514b9e8967
brw-rw---- 1 root disk 251, 40 2012-05-18 19:08 /dev/mapper/3600a0b800018f18300002f564b9e89c1
brw-rw---- 1 root disk 251, 39 2012-05-18 19:08 /dev/mapper/3600a0b800018f18300002f5b4b9e89f7
brw-rw---- 1 root disk 251, 36 2012-05-18 19:08 /dev/mapper/3600a0b800018f18300002f604b9e8a51
brw-rw---- 1 root disk 251, 44 2012-05-18 19:08 /dev/mapper/3600a0b800018f18300002f654b9e8aa3
brw-rw---- 1 root disk 251, 42 2012-05-18 19:08 /dev/mapper/3600a0b800018f18300002f724b9e8bb9
brw-rw---- 1 root disk 251, 33 2012-05-18 19:08 /dev/mapper/3600a0b800018f18300002f774b9e8bf1
brw-rw---- 1 root disk 251, 35 2012-05-18 19:08 /dev/mapper/3600a0b800018f18300002f7c4b9e8c33
brw-rw---- 1 root disk 251, 49 2012-05-18 19:08 /dev/mapper/3600a0b800018f18300002f814b9e8c6f
brw-rw---- 1 root disk 251, 45 2012-05-18 19:08 /dev/mapper/3600a0b800018f453000031af4b9e8afe
brw-rw---- 1 root disk 251, 52 2012-05-18 19:08 /dev/mapper/3600a0b800018f453000031b64b9e8b4e
brw-rw---- 1 root disk 251, 53 2012-05-18 19:08 /dev/mapper/3600a0b800018f453000031bc4b9e8b98
brw-rw---- 1 root disk 251, 50 2012-05-18 19:08 /dev/mapper/3600a0b800018f453000031c24b9e8bd4
brw-rw---- 1 root disk 251, 31 2012-05-18 19:08 /dev/mapper/3600a0b800018f453000031c84b9e8c20
brw-rw---- 1 root disk 251, 32 2012-05-18 19:08 /dev/mapper/3600a0b800018f453000031d84b9e8d56
brw-rw---- 1 root disk 251, 48 2012-05-18 19:08 /dev/mapper/3600a0b800018f453000031df4b9e8d8e
brw-rw---- 1 root disk 251, 47 2012-05-18 19:08 /dev/mapper/3600a0b800018f453000031e54b9e8dce
brw-rw---- 1 root disk 251, 46 2012-05-18 19:08 /dev/mapper/3600a0b800018f453000031eb4b9e8e08
brw-rw---- 1 root disk 251, 34 2012-05-18 19:08 /dev/mapper/3600a0b800018f453000031f14b9e8e66

multipath -l | grep : | sort -k3

_ 8:0:0:12 sdaa 65:160 [active][undef]
_ 9:0:0:12 sdab 65:176 [active][undef]
_ 8:0:0:13 sdac 65:192 [active][undef]
_ 8:0:0:14 sdad 65:208 [active][undef]
_ 8:0:0:15 sdae 65:224 [active][undef]
_ 9:0:0:13 sdaf 65:240 [active][undef]
_ 8:0:0:16 sdag 66:0 [active][undef]
_ 8:0:0:17 sdah 66:16 [active][undef]
_ 8:0:0:18 sdai 66:32 [active][undef]
_ 9:0:0:14 sdaj 66:48 [active][undef]
_ 9:0:0:15 sdak 66:64 [active][undef]
_ 9:0:0:16 sdal 66:80 [active][undef]
_ 9:0:0:17 sdam 66:96 [active][undef]
_ 9:0:0:18 sdan 66:112 [active][undef]
_ 8:0:0:0 sdc 8:32 [active][undef]
_ 9:0:0:0 sdd 8:48 [active][undef]
_ 8:0:0:1 sde 8:64 [active][undef]
_ 8:0:0:2 sdf 8:80 [active][undef]
_ 9:0:0:1 sdg 8:96 [active][undef]
_ 9:0:0:2 sdh 8:112 [active][undef]
_ 8:0:0:3 sdi 8:128 [active][undef]
_ 9:0:0:3 sdj 8:144 [active][undef]
_ 8:0:0:4 sdk 8:160 [active][undef]
_ 9:0:0:4 sdl 8:176 [active][undef]
_ 9:0:0:5 sdm 8:192 [active][undef]
_ 8:0:0:5 sdn 8:208 [active][undef]
_ 8:0:0:6 sdo 8:224 [active][undef]
_ 9:0:0:6 sdp 8:240 [active][undef]
_ 8:0:0:7 sdq 65:0 [active][undef]
_ 9:0:0:7 sdr 65:16 [active][undef]
_ 8:0:0:8 sds 65:32 [active][undef]
_ 9:0:0:8 sdt 65:48 [active][undef]
_ 9:0:0:9 sdu 65:64 [active][undef]
_ 8:0:0:9 sdv 65:80 [active][undef]
_ 8:0:0:10 sdw 65:96 [active][undef]
_ 9:0:0:10 sdx 65:112 [active][undef]
_ 8:0:0:11 sdy 65:128 [active][undef]
_ 9:0:0:11 sdz 65:144 [active][undef]

7
  • The [undef] status of the paths in the output of multipath -l is not quite right. I would rather see [active][ready] instead. Perhaps the pathchecker is not working correctly. Is lvm seeing the right physical volumes? May 18, 2012 at 11:15
  • Hi Dmitir - yes I saw that have fixed it, see above.
    – Kylea
    May 19, 2012 at 0:10
  • Not sure if you saw my additional question - I cannot seem to mount an non LVM managed multipath device as User read/write - root gets full rights - here are the attributes drwxr-xr-x - I have noticed in some posts the value wp=rw or [RW] when reviewing the output of multipath -ll: 3600a0b800018f453000031b64b9e8b4edm-52 SGI ,TP9300 [size=150G][features=0][hwhandler=0] _ round-robin 0 [prio=2][active] _ 8:0:0:2 sdf 8:80 [active][ready] _ 9:0:0:2 sdh 8:112 [active][ready]
    – Kylea
    May 19, 2012 at 0:13
  • Not sure if that issue is related to multipath. Multipath runs as root and does its wp test as root as well. Is there anything in the logs that might give a hint? May 19, 2012 at 5:30
  • I think I just figured out that I am an idiot - the attributes were set correctly for the owner and no Group 'Write' rights nor Guest 'Write' rights. I forgot that the device in question is actually owned by another user "sybase" as its the LUN with the Sybase ASE binaries. I was confused why I did have 'write' rights as my normal user. Just took a long time for the message to get through that I needed to explicitly allow group 'Write' rights and then add my self to the group. Apologies for that.
    – Kylea
    May 19, 2012 at 9:34
0

multipathing only changes the names of the block devices through device mapper interface. Your LVM structures probably exist within LVM formatted partitions residing on actual block devices.

If you install multipath tools and start multipathd daemon it will scan available block devices and create corresponding mappings via device mapper.

Then you will have to tell LVM tools to scan dm devices to see the new mappings.

As of multupath-tools FAQ, add the following line to /etc/lvm/lvm.conf:

 types = [ "device-mapper", 1 ]

If /etc/lvm/lvm.conf does not exist, you can create one based on your current/default config like so:

 lvm dumpconfig > /etc/lvm/lvm.conf

If the devices visible via device mapper come along with the original sd* block device nodes, filter the original block devices in lvm.conf.

If you have multiple volumes you may need to create additional character special device nodes for multipath:

#!/bin/bash -
x=0 
while (($x < 256))
do 
        y=$((x++))
        fname="/dev/sg"$y
        echo mknod $fname c 21 $y
        mknod $fname c 21 $y
done
1
  • I have another related question - how do I mark the devices as [rw] - I cannot mount the devices are user RW only root can write to the devices.
    – Kylea
    May 18, 2012 at 13:56
0

Output from multipath -ll

3600a0b800018f18300002f724b9e8bb9dm-42 SGI ,TP9300

[size=20G][features=0][hwhandler=0]
_ round-robin 0 [prio=2][active]
_ 8:0:0:11 sdy 65:128 [active][ready]
_ 9:0:0:11 sdz 65:144 [active][ready]

3600a0b800018f18300002f564b9e89c1dm-40 SGI ,TP9300

[size=110G][features=0][hwhandler=0]
_ round-robin 0 [prio=2][active]
_ 8:0:0:3 sdi 8:128 [active][ready]
_ 9:0:0:3 sdj 8:144 [active][ready]

3600a0b800018f453000031eb4b9e8e08dm-46 SGI ,TP9300

[size=100G][features=0][hwhandler=0]
_ round-robin 0 [prio=2][active]
_ 8:0:0:16 sdag 66:0 [active][ready]
_ 9:0:0:16 sdal 66:80 [active][ready]

3600a0b800018f18300002f5b4b9e89f7dm-39 SGI ,TP9300

[size=40G][features=0][hwhandler=0]
_ round-robin 0 [prio=2][active]
_ 8:0:0:5 sdn 8:208 [active][ready]
_ 9:0:0:5 sdm 8:192 [active][ready]

3600a0b800018f18300002f604b9e8a51dm-36 SGI ,TP9300

[size=20G][features=0][hwhandler=0]
_ round-robin 0 [prio=2][active]
_ 8:0:0:7 sdq 65:0 [active][ready]
_ 9:0:0:7 sdr 65:16 [active][ready]

3600a0b800018f453000031bc4b9e8b98dm-53 SGI ,TP9300

[size=50G][features=0][hwhandler=0]
_ round-robin 0 [prio=2][active]
_ 8:0:0:4 sdk 8:160 [active][ready]
_ 9:0:0:4 sdl 8:176 [active][ready]

3600a0b800018f18300002f774b9e8bf1dm-33 SGI ,TP9300

[size=100G][features=0][hwhandler=0]
_ round-robin 0 [prio=2][active]
_ 8:0:0:13 sdac 65:192 [active][ready]
_ 9:0:0:13 sdaf 65:240 [active][ready]

3600a0b800018f18300002f654b9e8aa3dm-44 SGI ,TP9300

[size=136G][features=0][hwhandler=0]
_ round-robin 0 [prio=2][active]
_ 8:0:0:9 sdv 65:80 [active][ready]
_ 9:0:0:9 sdu 65:64 [active][ready]

3600a0b800018f453000031df4b9e8d8edm-48 SGI ,TP9300

[size=30G][features=0][hwhandler=0]
_ round-robin 0 [prio=2][active]
_ 9:0:0:12 sdab 65:176 [active][ready]
_ 8:0:0:12 sdaa 65:160 [active][ready]

3600a0b800018f453000031e54b9e8dcedm-47 SGI ,TP9300

[size=110G][features=0][hwhandler=0]
_ round-robin 0 [prio=2][active]
_ 9:0:0:14 sdaj 66:48 [active][ready]
_ 8:0:0:14 sdad 65:208 [active][ready]

3600a0b800018f453000031c24b9e8bd4dm-50 SGI ,TP9300

[size=20G][features=0][hwhandler=0]
_ round-robin 0 [prio=2][active]
_ 8:0:0:6 sdo 8:224 [active][ready]
_ 9:0:0:6 sdp 8:240 [active][ready]

3600a0b800018f453000031d84b9e8d56dm-32 SGI ,TP9300

[size=40G][features=0][hwhandler=0]
_ round-robin 0 [prio=2][active]
_ 8:0:0:10 sdw 65:96 [active][ready]
_ 9:0:0:10 sdx 65:112 [active][ready]

3600a0b800018f453000031af4b9e8afedm-45 SGI ,TP9300

[size=140G][features=0][hwhandler=0]
_ round-robin 0 [prio=2][active]
_ 8:0:0:0 sdc 8:32 [active][ready]
_ 9:0:0:0 sdd 8:48 [active][ready]

3600a0b800018f18300002f814b9e8c6fdm-49 SGI ,TP9300

[size=125G][features=0][hwhandler=0]
_ round-robin 0 [prio=2][active]
_ 8:0:0:17 sdah 66:16 [active][ready]
_ 9:0:0:17 sdam 66:96 [active][ready]

3600a0b800018f453000031b64b9e8b4edm-52 SGI ,TP9300

[size=150G][features=0][hwhandler=0]
_ round-robin 0 [prio=2][active]
_ 8:0:0:2 sdf 8:80 [active][ready]
_ 9:0:0:2 sdh 8:112 [active][ready]

3600a0b800018f18300002f7c4b9e8c33dm-35 SGI ,TP9300

[size=20G][features=0][hwhandler=0]
_ round-robin 0 [prio=2][active]
_ 8:0:0:15 sdae 65:224 [active][ready]
_ 9:0:0:15 sdak 66:64 [active][ready]

3600a0b800018f18300002f514b9e8967dm-56 SGI ,TP9300

[size=30G][features=0][hwhandler=0]
_ round-robin 0 [prio=2][active]
_ 8:0:0:1 sde 8:64 [active][ready]
_ 9:0:0:1 sdg 8:96 [active][ready]

3600a0b800018f453000031f14b9e8e66dm-34 SGI ,TP9300

[size=136G][features=0][hwhandler=0]
_ round-robin 0 [prio=2][active]
_ 8:0:0:18 sdai 66:32 [active][ready]
_ 9:0:0:18 sdan 66:112 [active][ready]

3600a0b800018f453000031c84b9e8c20dm-31 SGI ,TP9300

[size=121G][features=0][hwhandler=0]
_ round-robin 0 [prio=2][active]
_ 8:0:0:8 sds 65:32 [active][ready]
_ 9:0:0:8 sdt 65:48 [active][ready]

You must log in to answer this question.

Not the answer you're looking for? Browse other questions tagged .