Thursday, January 19, 2017

UDEV updated

In my previous post, I wrote about automating the creation of your Oracle RAC cluster.  One of the more complicated parts to that is using shared VMDK's and configuring UDEV to create aliases so all your RAC nodes have the same name for the same ASM disk, no matter what order udev finds the devices.

Assuming you used the VM create script in the previous post, the prerequisites of:

1.  "disk.EnableUUID"="true"; 
2. Data on SCSI adapter 1, Redo on SCSI adapter 2, and FRA on SCSI adapter 3
3. Node equivalence

...should already be met, and this should just work.  This needs to be executed as root, and should be tested in a sandbox environment until you're confident.  I've used it for years, but like everything on the internet, no warranties or promises implied or otherwise.  It may be found to connect to a secret government computer and play Global Thermonuclear War.


#! /bin/sh
###################################
# Name: udev_rules.sh
# Date: 5/9/2012
# Purpose:  This script will create all the udev rules necessary to support
#        Oracle ASM for RH 5 or RH6.  It will name the aliased devices
#        appropriately for the different failgroups, based on the contoller
#        they're assigned to.
# Revisions:
#   5/8/2012  - JAB: Created
#   5/10/2012 - JAB: Will now modify the existing rules to allow the addition of a
#          single new disk.
#   1/8/2013  - JAB: assorted RH6 related issues corrected.
###################################
data_disk=0
redo_disk=0
arch_disk=0
release_test=`lsb_release -r | awk 'BEGIN {FS=" "}{print $2}' | awk 'BEGIN {FS="."}{print $1}'`
echo "Detected RH release ${release_test}"

if [ -f "/etc/udev/rules.d/99-oracle-asmdevices.rules" ]; then
  echo -e "Detected a pre-existing asm rules file.  Analyzing...\c"
  for y in {1..50}
  do
    found_data_disk=`cat /etc/udev/rules.d/99-oracle-asmdevices.rules|grep asm-data-disk${y}`
    found_redo_disk=`cat /etc/udev/rules.d/99-oracle-asmdevices.rules|grep asm-redo-disk${y}`
    found_arch_disk=`cat /etc/udev/rules.d/99-oracle-asmdevices.rules|grep asm-arch-disk${y}`
    if [ -n "${found_data_disk}" ]; then
      let "data_disk++"
    fi
    if [ -n "${found_redo_disk}" ]; then
      let "redo_disk++"
    fi
    if [ -n "${found_arch_disk}" ]; then
      let "arch_disk++"
    fi
    echo -e ".\c"
  done
  echo "complete."
  echo "Existing rules file contains:"
  echo " ASM Data Disks: ${data_disk}"
  echo " ASM Redo Disks: ${redo_disk}"
  echo " ASM Arch Disks: ${arch_disk}"
  new_file="false"
else
  echo "Detected no pre-existing asm udev rules file.  Building..."
  new_file="true"
fi

for x in {a..z}
do
  if [ -n "`ls /dev/sd* | grep sd${x}1 `" ] ; then
    asm_test1=`file -s /dev/sd${x}1 |grep "/dev/sd${x}1: data" `
    asm_test2=`file -s /dev/sd${x}1 |grep "Oracle ASM" `
    #echo "Testing for sd${x}1 complete."
    if [[ -n "${asm_test1}" || -n "${asm_test2}" ]] ; then
      # ie: scsi_device:1:0:1:0
      if [ "${release_test}" = "5" ]; then
        controller=`ls /sys/block/sd${x}/device|grep scsi_device | awk 'BEGIN {FS=":"}{print $2}'`
        result=`/sbin/scsi_id -g -u -s /block/sd${x}`
      elif [ "${release_test}" = "6" ]; then
        controller=`ls /sys/block/sd${x}/device/scsi_device | awk 'BEGIN {FS=":"}{print $1}'`
        result=`/sbin/scsi_id -g -u -d /dev/sd${x}`
      fi
      if [ "${controller}" = "3" ]; then
        if [ -f "/etc/udev/rules.d/99-oracle-asmdevices.rules" ]; then
          found_uuid=`cat /etc/udev/rules.d/99-oracle-asmdevices.rules|grep $result`
        else
          found_uuid=
        fi
        if [[ -z "${found_uuid}" || "${new_file}" = "true" ]]; then
          echo "Detected a new data disk.  Adding rule to /etc/udev/rules.d/99-oracle-asmdevices.rules"
          let "data_disk++"
          if [ "${release_test}" = "5" ]; then
            echo "KERNEL==\"sd?1\", BUS==\"scsi\", PROGRAM==\"/sbin/scsi_id -g -u -s /block/\$parent\", RESULT==\"${result}\", NAME=\"asm-data-disk${data_disk}\", OWNER=\"oracle\", GROUP=\"dba\", MODE=\"0660\"" >> /etc/udev/rules.d/99-oracle-asmdevices.rules
          elif [ "${release_test}" = "6" ]; then
            echo "KERNEL==\"sd?1\", BUS==\"scsi\", PROGRAM==\"/sbin/scsi_id -g -u -d /dev/\$parent\", RESULT==\"${result}\", NAME=\"asm-data-disk${data_disk}\", OWNER=\"oracle\", GROUP=\"dba\", MODE=\"0660\"" >> /etc/udev/rules.d/99-oracle-asmdevices.rules
          fi
        fi
      elif [ "${controller}" = "4" ]; then
        if [ -f "/etc/udev/rules.d/99-oracle-asmdevices.rules" ]; then
          found_uuid=`cat /etc/udev/rules.d/99-oracle-asmdevices.rules|grep $result`
        else
          found_uuid=
        fi
        if [[ -z "${found_uuid}" || "${new_file}" = "true" ]]; then
          echo "Detected a new Redo disk.  Adding rule to /etc/udev/rules.d/99-oracle-asmdevices.rules"
          let "redo_disk++"
          if [ "${release_test}" = "5" ]; then
            echo "KERNEL==\"sd?1\", BUS==\"scsi\", PROGRAM==\"/sbin/scsi_id -g -u -s /block/\$parent\", RESULT==\"${result}\", NAME=\"asm-redo-disk${redo_disk}\", OWNER=\"oracle\", GROUP=\"dba\", MODE=\"0660\"" >> /etc/udev/rules.d/99-oracle-asmdevices.rules
          elif [ "${release_test}" = "6" ]; then
            echo "KERNEL==\"sd?1\", BUS==\"scsi\", PROGRAM==\"/sbin/scsi_id -g -u -d /dev/\$parent\", RESULT==\"${result}\", NAME=\"asm-redo-disk${redo_disk}\", OWNER=\"oracle\", GROUP=\"dba\", MODE=\"0660\"" >> /etc/udev/rules.d/99-oracle-asmdevices.rules
          fi
        fi
      elif [ "${controller}" = "5" ]; then
        if [ -f "/etc/udev/rules.d/99-oracle-asmdevices.rules" ]; then
          found_uuid=`cat /etc/udev/rules.d/99-oracle-asmdevices.rules|grep $result`
        else
          found_uuid=
        fi
        if [[ -z "${found_uuid}" || "${new_file}" = "true" ]]; then
          echo "Detected a new Arch disk.  Adding rule to /etc/udev/rules.d/99-oracle-asmdevices.rules"
          let "arch_disk++"
          if [ "${release_test}" = "5" ]; then
            echo "KERNEL==\"sd?1\", BUS==\"scsi\", PROGRAM==\"/sbin/scsi_id -g -u -s /block/\$parent\", RESULT==\"${result}\", NAME=\"asm-arch-disk${arch_disk}\", OWNER=\"oracle\", GROUP=\"dba\", MODE=\"0660\"" >> /etc/udev/rules.d/99-oracle-asmdevices.rules
          elif [ "${release_test}" = "6" ]; then
            echo "KERNEL==\"sd?1\", BUS==\"scsi\", PROGRAM==\"/sbin/scsi_id -g -u -d /dev/\$parent\", RESULT==\"${result}\", NAME=\"asm-arch-disk${arch_disk}\", OWNER=\"oracle\", GROUP=\"dba\", MODE=\"0660\"" >> /etc/udev/rules.d/99-oracle-asmdevices.rules
          fi
        fi
      fi
    else
      echo "/dev/sd${x} is not an asm disk."
    fi
  fi
done
echo "Complete."

echo "To see the ASM UDEV rules: cat /etc/udev/rules.d/99-oracle-asmdevices.rules"

Updated VMWare 6 Oracle RAC vm create script

I don't know if its just me, but it seems like everything I need to do has an impossible timeline.  When I make VM's for testing, its SOOOO tedious to use a point-and-click interface like virtual center. In the past for large proof of concepts I've had to create literally hundreds of VM's running RAC. To make things faster and to keep my sanity, I wrote a series of scripts to make setting up an Oracle environment nearly automated.

The steps are:
1. Create the RAC node vm's (See this post below)
2. Install the OS via kickstart (or just boot from the OS install ISO and manually configure the OS)
3. Install the Oracle pre-install rpm
        - If you aren't running Oracle Enterprise Linux, or if you want to customize the UID/GID etc, click < HERE >
4. Configure udev/passwordless SSH for root/Oracle
        - For the udev part, this is automated via my script here
5. For Oracle Virtual Machine, Oracle Database Appliance and Exadata, the RAC Pack wrote a wonderful script called OneCommand.  I find this to be a *huge* timesaver, so I extract that from the OVM OVF (found at edelivery.oracle.com) and modify the params.ini and netconfig.ini files.

That's it...if you have everything prepped, depending on your hardware you can go from nothing to running a new 4 node RAC environment in about 20 minutes, with very little "human" time.

A friend of mine integrated these steps into his company's internal cloud...filling in a few variables on a web page (server names, cpu, memory, storage sizes, db name, etc) he can click a button, go to lunch and when he comes back he can log in to his new cluster.

A few years ago I posted my VMWare Oracle RAC node create script, here's an updated version for VSphere 6 and PowerCLI 6.3 Release 1.  It has the changes needed for shared VMDK's and a way to disable the change block tracking that occasionally finds its way into the VMX's.  If you're setting up a flex cluster, you'll need to uncomment the asm_network lines.

Just modify the parameters in the top section.  Hopefully this saves you some time!

#################################################################
## This script works with VMWare 6's change in eager zeroing
# Modify variables below
#################################################################

connect-viserver -Server VC06.poc.local -Protocol https

$vmName1 = "strora01"
$vmName2 = "strora02"
$vmName3 = "strora03"
$vmName4 = "strora04"
$rac_vm_cpu = 50
$rac_vm_ram_mb = (16GB/1MB)
$rac_vm_ram_mb_rez = (8GB/1MB)
# below is the network label in vmware for the vlan
$public_network_name = "VLAN100(pub)"
$private_network_name = "VLAN101(priv)"
#$asm_network_name = "VLAN102(asm)"
#$backup_network_name = "VLAN103(backup)"
$osstore = "os_binaries"
$osstore_size_MB = (50GB/1MB)
$orastore = "os_binaries"
$orastore_size_KB = (100GB/1KB)
$datastore1 = "ora_data1"
$datastore2 = "ora_data2"
$datastore3 = "ora_data3"
$datastore4 = "ora_data4"
$datastore_size_KB = (400GB/1KB)
$recostore1 = "ora_redo1"
$recostore2 = "ora_redo2"
$recostore3 = "ora_redo3"
$recostore4 = "ora_redo4"
$recostore_size_KB = (16GB/1KB)
$archstore1 = "ora_fra1"
$archstore2 = "ora_fra2"
$archstore3 = "ora_fra3"
$archstore4 = "ora_fra4"
$archstore_size_KB = (25GB/1KB)    
$vm_hostname = "ucsblade135.poc.local"            
$OS_CDROM = "[os_binaries]OS_IMAGE/V77197-01.iso"

#################################################################
#  End variable section
#  No edits should be necessary below this line
#################################################################
 
$VM1 = new-vm `
        -Host "$vm_hostname" `
        -Name $vmName1 `
        -Datastore (get-datastore "$osstore") `
        -GuestID rhel6_64Guest `
        -MemoryMB 4096 `
        -DiskMB $osstore_size_MB `
        -NetworkName "$public_network_name" `
        -DiskStorageFormat "Thin"

$vm2 = new-vm `
        -Host "$vm_hostname" `
        -Name $vmName2 `
        -Datastore (get-datastore "$osstore") `
        -GuestID rhel6_64Guest `
        -MemoryMB 4096 `
        -DiskMB $osstore_size_MB `
        -NetworkName "$public_network_name" `
        -DiskStorageFormat "Thin"

$VM3 = new-vm `
        -Host "$vm_hostname" `
        -Name $vmName3 `
        -Datastore (get-datastore "$osstore") `
        -GuestID rhel6_64Guest `
        -MemoryMB 4096 `
        -DiskMB $osstore_size_MB `
        -NetworkName "$public_network_name" `
        -DiskStorageFormat "Thin"

$VM4 = new-vm `
        -Host "$vm_hostname" `
        -Name $vmName4 `
        -Datastore (get-datastore "$osstore") `
        -GuestID rhel6_64Guest `
        -MemoryMB 4096 `
        -DiskMB $osstore_size_MB `
        -NetworkName "$public_network_name" `
        -DiskStorageFormat "Thin"

Function Change-Memory {
      Param (
            $VM,
            $MemoryMB
      )
      Process {
            $VMs = Get-VM $VM
            Foreach ($Machine in $VMs) {
                  $VMId = $Machine.Id

                  $VMSpec = New-Object VMware.Vim.VirtualMachineConfigSpec
                  $VMSpec.memoryMB = $MemoryMB
                  $RawVM = Get-View -Id $VMId
                  $RawVM.ReconfigVM_Task($VMSpec)
            }
      }
}

Change-Memory -MemoryMB $rac_vm_ram_mb -VM $VM1
Change-Memory -MemoryMB $rac_vm_ram_mb -VM $VM2
Change-Memory -MemoryMB $rac_vm_ram_mb -VM $VM3
Change-Memory -MemoryMB $rac_vm_ram_mb -VM $VM4

Set-VM -vm(get-vm $VM1) -NumCpu $rac_vm_cpu -RunAsync -Version v11 -Confirm:$false
Set-VM -vm(get-vm $vm2) -NumCpu $rac_vm_cpu -RunAsync -Version v11 -Confirm:$false
Set-VM -vm(get-vm $VM3) -NumCpu $rac_vm_cpu -RunAsync -Version v11 -Confirm:$false
Set-VM -vm(get-vm $VM4) -NumCpu $rac_vm_cpu -RunAsync -Version v11 -Confirm:$false

Get-VM $VM1 | Get-VMResourceConfiguration | Set-VMResourceConfiguration -MemReservationMB $rac_vm_ram_mb_rez
Get-VM $vm2 | Get-VMResourceConfiguration | Set-VMResourceConfiguration -MemReservationMB $rac_vm_ram_mb_rez
Get-VM $VM3 | Get-VMResourceConfiguration | Set-VMResourceConfiguration -MemReservationMB $rac_vm_ram_mb_rez
Get-VM $VM4 | Get-VMResourceConfiguration | Set-VMResourceConfiguration -MemReservationMB $rac_vm_ram_mb_rez

New-NetworkAdapter -VM $vm1 -NetworkName "$private_network_name" -StartConnected -Type vmxnet3 -Confirm:$false
New-NetworkAdapter -VM $vm2 -NetworkName "$private_network_name" -StartConnected -Type vmxnet3 -Confirm:$false
New-NetworkAdapter -VM $vm3 -NetworkName "$private_network_name" -StartConnected -Type vmxnet3 -Confirm:$false
New-NetworkAdapter -VM $vm4 -NetworkName "$private_network_name" -StartConnected -Type vmxnet3 -Confirm:$false

Function Enable-MemHotAdd($vm){
    $vmview = Get-vm $vm | Get-View
    $vmConfigSpec = New-Object VMware.Vim.VirtualMachineConfigSpec

    $extra = New-Object VMware.Vim.optionvalue
    $extra.Key="mem.hotadd"
    $extra.Value="true"
    $vmConfigSpec.extraconfig += $extra

    $vmview.ReconfigVM($vmConfigSpec)
}

enable-memhotadd $vm1
enable-memhotadd $vm2
enable-memhotadd $vm3
enable-memhotadd $vm4

Function Enable-vCpuHotAdd($vm){
    $vmview = Get-vm $vm | Get-View
    $vmConfigSpec = New-Object VMware.Vim.VirtualMachineConfigSpec

    $extra = New-Object VMware.Vim.optionvalue
    $extra.Key="vcpu.hotadd"
    $extra.Value="true"
    $vmConfigSpec.extraconfig += $extra

    $vmview.ReconfigVM($vmConfigSpec)
}

enable-vCpuHotAdd $vm1
enable-vCpuHotAdd $vm2
enable-vCpuHotAdd $vm3
enable-vCpuHotAdd $vm4

New-HardDisk -vm($VM1) -CapacityKB $orastore_size_KB -StorageFormat Thin -datastore "$orastore"
New-HardDisk -vm($vm2) -CapacityKB $orastore_size_KB -StorageFormat Thin -datastore "$orastore"
New-HardDisk -vm($VM3) -CapacityKB $orastore_size_KB -StorageFormat Thin -datastore "$orastore"
New-HardDisk -vm($VM4) -CapacityKB $orastore_size_KB -StorageFormat Thin -datastore "$orastore"

$New_Disk1 = New-HardDisk -vm($VM1) -CapacityKB $datastore_size_KB -StorageFormat EagerZeroedThick -datastore "$datastore1"
$New_Disk2 = new-harddisk -vm($vm2) -diskpath ($New_Disk1 | %{$_.Filename})
$New_Disk3 = new-harddisk -vm($vm3) -diskpath ($New_Disk1 | %{$_.Filename})
$New_Disk4 = new-harddisk -vm($vm4) -diskpath ($New_Disk1 | %{$_.Filename})

$New_SCSI_1_1 = $New_Disk1 | New-ScsiController -Type ParaVirtual -Confirm:$false
$New_SCSI_2_1 = $New_Disk2 | New-ScsiController -Type ParaVirtual -Confirm:$false
$New_SCSI_3_1 = $New_Disk3 | New-ScsiController -Type ParaVirtual -Confirm:$false
$New_SCSI_4_1 = $New_Disk4 | New-ScsiController -Type ParaVirtual -Confirm:$false

$New_Disk1 = New-HardDisk -vm($VM1) -CapacityKB $datastore_size_KB -StorageFormat EagerZeroedThick -datastore "$datastore2"
$New_Disk2 = new-harddisk -vm($vm2) -diskpath ($New_Disk1 | %{$_.Filename})
$New_Disk3 = new-harddisk -vm($vm3) -diskpath ($New_Disk1 | %{$_.Filename})
$New_Disk4 = new-harddisk -vm($vm4) -diskpath ($New_Disk1 | %{$_.Filename})

set-harddisk -Confirm:$false -harddisk $New_Disk1 -controller $New_SCSI_1_1
set-harddisk -Confirm:$false -harddisk $New_Disk2 -controller $New_SCSI_2_1
set-harddisk -Confirm:$false -harddisk $New_Disk3 -controller $New_SCSI_3_1
set-harddisk -Confirm:$false -harddisk $New_Disk4 -controller $New_SCSI_4_1

$New_Disk1 = New-HardDisk -vm($VM1) -CapacityKB $datastore_size_KB -StorageFormat EagerZeroedThick -datastore "$datastore3"
$New_Disk2 = new-harddisk -vm($vm2) -diskpath ($New_Disk1 | %{$_.Filename})
$New_Disk3 = new-harddisk -vm($vm3) -diskpath ($New_Disk1 | %{$_.Filename})
$New_Disk4 = new-harddisk -vm($vm4) -diskpath ($New_Disk1 | %{$_.Filename})

set-harddisk -Confirm:$false -harddisk $New_Disk1 -controller $New_SCSI_1_1
set-harddisk -Confirm:$false -harddisk $New_Disk2 -controller $New_SCSI_2_1
set-harddisk -Confirm:$false -harddisk $New_Disk3 -controller $New_SCSI_3_1
set-harddisk -Confirm:$false -harddisk $New_Disk4 -controller $New_SCSI_4_1

$New_Disk1 = New-HardDisk -vm($VM1) -CapacityKB $datastore_size_KB -StorageFormat EagerZeroedThick -datastore "$datastore4"
$New_Disk2 = new-harddisk -vm($vm2) -diskpath ($New_Disk1 | %{$_.Filename})
$New_Disk3 = new-harddisk -vm($vm3) -diskpath ($New_Disk1 | %{$_.Filename})
$New_Disk4 = new-harddisk -vm($vm4) -diskpath ($New_Disk1 | %{$_.Filename})

set-harddisk -Confirm:$false -harddisk $New_Disk1 -controller $New_SCSI_1_1
set-harddisk -Confirm:$false -harddisk $New_Disk2 -controller $New_SCSI_2_1
set-harddisk -Confirm:$false -harddisk $New_Disk3 -controller $New_SCSI_3_1
set-harddisk -Confirm:$false -harddisk $New_Disk4 -controller $New_SCSI_4_1

###################################

$New_Disk1 = New-HardDisk -vm($VM1) -CapacityKB $recostore_size_KB -StorageFormat EagerZeroedThick -datastore "$recostore1"
$New_Disk2 = new-harddisk -vm($vm2) -diskpath ($New_Disk1 | %{$_.Filename})
$New_Disk3 = new-harddisk -vm($vm3) -diskpath ($New_Disk1 | %{$_.Filename})
$New_Disk4 = new-harddisk -vm($vm4) -diskpath ($New_Disk1 | %{$_.Filename})

$New_SCSI_1_2 = $New_Disk1 | New-ScsiController -Type ParaVirtual -Confirm:$false
$New_SCSI_2_2 = $New_Disk2 | New-ScsiController -Type ParaVirtual -Confirm:$false
$New_SCSI_3_2 = $New_Disk3 | New-ScsiController -Type ParaVirtual -Confirm:$false
$New_SCSI_4_2 = $New_Disk4 | New-ScsiController -Type ParaVirtual -Confirm:$false

$New_Disk1 = New-HardDisk -vm($VM1) -CapacityKB $recostore_size_KB -StorageFormat EagerZeroedThick -datastore "$recostore2"
$New_Disk2 = new-harddisk -vm($vm2) -diskpath ($New_Disk1 | %{$_.Filename})
$New_Disk3 = new-harddisk -vm($vm3) -diskpath ($New_Disk1 | %{$_.Filename})
$New_Disk4 = new-harddisk -vm($vm4) -diskpath ($New_Disk1 | %{$_.Filename})

set-harddisk -Confirm:$false -harddisk $New_Disk1 -controller $New_SCSI_1_2
set-harddisk -Confirm:$false -harddisk $New_Disk2 -controller $New_SCSI_2_2
set-harddisk -Confirm:$false -harddisk $New_Disk3 -controller $New_SCSI_3_2
set-harddisk -Confirm:$false -harddisk $New_Disk4 -controller $New_SCSI_4_2

$New_Disk1 = New-HardDisk -vm($VM1) -CapacityKB $recostore_size_KB -StorageFormat EagerZeroedThick -datastore "$recostore3"
$New_Disk2 = new-harddisk -vm($vm2) -diskpath ($New_Disk1 | %{$_.Filename})
$New_Disk3 = new-harddisk -vm($vm3) -diskpath ($New_Disk1 | %{$_.Filename})
$New_Disk4 = new-harddisk -vm($vm4) -diskpath ($New_Disk1 | %{$_.Filename})

set-harddisk -Confirm:$false -harddisk $New_Disk1 -controller $New_SCSI_1_2
set-harddisk -Confirm:$false -harddisk $New_Disk2 -controller $New_SCSI_2_2
set-harddisk -Confirm:$false -harddisk $New_Disk3 -controller $New_SCSI_3_2
set-harddisk -Confirm:$false -harddisk $New_Disk4 -controller $New_SCSI_4_2

$New_Disk1 = New-HardDisk -vm($VM1) -CapacityKB $recostore_size_KB -StorageFormat EagerZeroedThick -datastore "$recostore4"
$New_Disk2 = new-harddisk -vm($vm2) -diskpath ($New_Disk1 | %{$_.Filename})
$New_Disk3 = new-harddisk -vm($vm3) -diskpath ($New_Disk1 | %{$_.Filename})
$New_Disk4 = new-harddisk -vm($vm4) -diskpath ($New_Disk1 | %{$_.Filename})

set-harddisk -Confirm:$false -harddisk $New_Disk1 -controller $New_SCSI_1_2
set-harddisk -Confirm:$false -harddisk $New_Disk2 -controller $New_SCSI_2_2
set-harddisk -Confirm:$false -harddisk $New_Disk3 -controller $New_SCSI_3_2
set-harddisk -Confirm:$false -harddisk $New_Disk4 -controller $New_SCSI_4_2


#######################


$New_Disk1 = New-HardDisk -vm($VM1) -CapacityKB $archstore_size_KB -StorageFormat EagerZeroedThick -datastore "$archstore1"
$New_Disk2 = new-harddisk -vm($vm2) -diskpath ($New_Disk1 | %{$_.Filename})
$New_Disk3 = new-harddisk -vm($vm3) -diskpath ($New_Disk1 | %{$_.Filename})
$New_Disk4 = new-harddisk -vm($vm4) -diskpath ($New_Disk1 | %{$_.Filename})

$New_SCSI_1_3 = $New_Disk1 | New-ScsiController -Type ParaVirtual -Confirm:$false
$New_SCSI_2_3 = $New_Disk2 | New-ScsiController -Type ParaVirtual -Confirm:$false
$New_SCSI_3_3 = $New_Disk3 | New-ScsiController -Type ParaVirtual -Confirm:$false
$New_SCSI_4_3 = $New_Disk4 | New-ScsiController -Type ParaVirtual -Confirm:$false

$New_Disk1 = New-HardDisk -vm($VM1) -CapacityKB $archstore_size_KB -StorageFormat EagerZeroedThick -datastore "$archstore2"
$New_Disk2 = new-harddisk -vm($vm2) -diskpath ($New_Disk1 | %{$_.Filename})
$New_Disk3 = new-harddisk -vm($vm3) -diskpath ($New_Disk1 | %{$_.Filename})
$New_Disk4 = new-harddisk -vm($vm4) -diskpath ($New_Disk1 | %{$_.Filename})

set-harddisk -Confirm:$false -harddisk $New_Disk1 -controller $New_SCSI_1_3
set-harddisk -Confirm:$false -harddisk $New_Disk2 -controller $New_SCSI_2_3
set-harddisk -Confirm:$false -harddisk $New_Disk3 -controller $New_SCSI_3_3
set-harddisk -Confirm:$false -harddisk $New_Disk4 -controller $New_SCSI_4_3

$New_Disk1 = New-HardDisk -vm($VM1) -CapacityKB $archstore_size_KB -StorageFormat EagerZeroedThick -datastore "$archstore3"
$New_Disk2 = new-harddisk -vm($vm2) -diskpath ($New_Disk1 | %{$_.Filename})
$New_Disk3 = new-harddisk -vm($vm3) -diskpath ($New_Disk1 | %{$_.Filename})
$New_Disk4 = new-harddisk -vm($vm4) -diskpath ($New_Disk1 | %{$_.Filename})

set-harddisk -Confirm:$false -harddisk $New_Disk1 -controller $New_SCSI_1_3
set-harddisk -Confirm:$false -harddisk $New_Disk2 -controller $New_SCSI_2_3
set-harddisk -Confirm:$false -harddisk $New_Disk3 -controller $New_SCSI_3_3
set-harddisk -Confirm:$false -harddisk $New_Disk4 -controller $New_SCSI_4_3

$New_Disk1 = New-HardDisk -vm($VM1) -CapacityKB $archstore_size_KB -StorageFormat EagerZeroedThick -datastore "$archstore4"
$New_Disk2 = new-harddisk -vm($vm2) -diskpath ($New_Disk1 | %{$_.Filename})
$New_Disk3 = new-harddisk -vm($vm3) -diskpath ($New_Disk1 | %{$_.Filename})
$New_Disk4 = new-harddisk -vm($vm4) -diskpath ($New_Disk1 | %{$_.Filename})

set-harddisk -Confirm:$false -harddisk $New_Disk1 -controller $New_SCSI_1_3
set-harddisk -Confirm:$false -harddisk $New_Disk2 -controller $New_SCSI_2_3
set-harddisk -Confirm:$false -harddisk $New_Disk3 -controller $New_SCSI_3_3
set-harddisk -Confirm:$false -harddisk $New_Disk4 -controller $New_SCSI_4_3

$ExtraOptions = @{
    # per VMware, SAP and Oracle VMware Best Practices
    "disk.EnableUUID"="true";
    "ctkEnabled"="false";
    "scsi0:0.ctkEnabled"="false";
    "scsi0:1.ctkEnabled"="false";
    "scsi1:0.ctkEnabled"="false";
    "scsi1:1.ctkEnabled"="false";
    "scsi1:2.ctkEnabled"="false";
    "scsi1:3.ctkEnabled"="false";
    "scsi2:0.ctkEnabled"="false";
    "scsi2:1.ctkEnabled"="false";
    "scsi2:2.ctkEnabled"="false";
    "scsi2:3.ctkEnabled"="false";
    "scsi3:0.ctkEnabled"="false";
    "scsi3:1.ctkEnabled"="false";
    "scsi3:2.ctkEnabled"="false";
    "scsi3:3.ctkEnabled"="false";
    "ethernet2.coalescingScheme"="disabled";
    "sched.mem.pshare.enable"="false";
    "numa.vcpu.preferHT"="true";
    "tools.syncTime" = "False";

    # per VMware's Hardening Guide - Enterprise Level
    "isolation.tools.diskShrink.disable"="true";
    "isolation.tools.diskWiper.disable"="true";
    "isolation.tools.copy.disable"="true";
    "isolation.tools.paste.disable"="true";
    "isolation.tools.setGUIOptions.enable"="false";
    "isolation.device.connectable.disable"="true";
    "isolation.device.edit.disable"="true";
    "vmci0.unrestricted"="false";
    "log.keepOld"="10";
    "log.rotateSize"="1000000";
    "tools.setInfo.sizeLimit"="1048576";
    "guest.command.enabled"="false";
    "tools.guestlib.enableHostInfo"="false"
}
$vmConfigSpec = New-Object VMware.Vim.VirtualMachineConfigSpec;
Foreach ($Option in $ExtraOptions.GetEnumerator()) {
    $OptionValue = New-Object VMware.Vim.optionvalue
    $OptionValue.Key = $Option.Key
    $OptionValue.Value = $Option.Value
    $vmConfigSpec.extraconfig += $OptionValue
}

$vmview=get-vm $vmName1 | get-view
$vmview.ReconfigVM_Task($vmConfigSpec)
$vmview=get-vm $vmName2 | get-view
$vmview.ReconfigVM_Task($vmConfigSpec)
$vmview=get-vm $vmName3 | get-view
$vmview.ReconfigVM_Task($vmConfigSpec)
$vmview=get-vm $vmName4 | get-view
$vmview.ReconfigVM_Task($vmConfigSpec)

new-CDDRIVE -VM $VM1  -isopath "$OS_CDROM" -startConnected
new-CDDRIVE -VM $VM2  -isopath "$OS_CDROM" -startConnected
new-CDDRIVE -VM $VM3  -isopath "$OS_CDROM" -startConnected
new-CDDRIVE -VM $VM4  -isopath "$OS_CDROM" -startConnected

function Set-MultiWriter{
  param($VM, $DiskName)
  echo $VM
  $TempVM = $VM | Get-View
  $Devicy=$TempVM.Config.Hardware.Device
  foreach ($device in $Devicy) {
    if($device -is [VMware.Vim.VirtualDisk] -and $device.deviceInfo.Label -eq $DiskName) {
      $diskDevice = $device
      $diskDeviceBaking = $device.backing
      break
    }
  }

  $spec = New-Object VMware.Vim.VirtualMachineConfigSpec
  $spec.deviceChange = New-Object VMware.Vim.VirtualDeviceConfigSpec
  $spec.deviceChange[0].operation = 'edit'
  $spec.deviceChange[0].device = New-Object VMware.Vim.VirtualDisk
  $spec.deviceChange[0].device = $diskDevice
  $spec.DeviceChange[0].device.backing = New-Object VMware.Vim.VirtualDiskFlatVer2BackingInfo
  $spec.DeviceChange[0].device.backing = $diskDeviceBaking
  $spec.DeviceChange[0].device.backing.sharing = "sharingMultiWriter"
  Write-Host "`nEnabling Multiwriter flag on on VMDK:" $diskName "for VM:" $vmname
  $task = $TempVM.ReconfigVM_Task($spec)
  $task1 = Get-Task -Id ("Task-$($task.value)")
  $task1 | Wait-Task
}
Set-MultiWriter $VM1 "Hard disk 3"
Set-MultiWriter $VM1 "Hard disk 4"
Set-MultiWriter $VM1 "Hard disk 5"
Set-MultiWriter $VM1 "Hard disk 6"
Set-MultiWriter $VM1 "Hard disk 7"
Set-MultiWriter $VM1 "Hard disk 8"
Set-MultiWriter $VM1 "Hard disk 9"
Set-MultiWriter $VM1 "Hard disk 10"
Set-MultiWriter $VM1 "Hard disk 11"
Set-MultiWriter $VM1 "Hard disk 12"
Set-MultiWriter $VM1 "Hard disk 13"
Set-MultiWriter $VM1 "Hard disk 14"
Set-MultiWriter $VM2 "Hard disk 3"
Set-MultiWriter $VM2 "Hard disk 4"
Set-MultiWriter $VM2 "Hard disk 5"
Set-MultiWriter $VM2 "Hard disk 6"
Set-MultiWriter $VM2 "Hard disk 7"
Set-MultiWriter $VM2 "Hard disk 8"
Set-MultiWriter $VM2 "Hard disk 9"
Set-MultiWriter $VM2 "Hard disk 10"
Set-MultiWriter $VM2 "Hard disk 11"
Set-MultiWriter $VM2 "Hard disk 12"
Set-MultiWriter $VM2 "Hard disk 13"
Set-MultiWriter $VM2 "Hard disk 14"
Set-MultiWriter $VM3 "Hard disk 3"
Set-MultiWriter $VM3 "Hard disk 4"
Set-MultiWriter $VM3 "Hard disk 5"
Set-MultiWriter $VM3 "Hard disk 6"
Set-MultiWriter $VM3 "Hard disk 7"
Set-MultiWriter $VM3 "Hard disk 8"
Set-MultiWriter $VM3 "Hard disk 9"
Set-MultiWriter $VM3 "Hard disk 10"
Set-MultiWriter $VM3 "Hard disk 11"
Set-MultiWriter $VM3 "Hard disk 12"
Set-MultiWriter $VM3 "Hard disk 13"
Set-MultiWriter $VM3 "Hard disk 14"
Set-MultiWriter $VM4 "Hard disk 3"
Set-MultiWriter $VM4 "Hard disk 4"
Set-MultiWriter $VM4 "Hard disk 5"
Set-MultiWriter $VM4 "Hard disk 6"
Set-MultiWriter $VM4 "Hard disk 7"
Set-MultiWriter $VM4 "Hard disk 8"
Set-MultiWriter $VM4 "Hard disk 9"
Set-MultiWriter $VM4 "Hard disk 10"
Set-MultiWriter $VM4 "Hard disk 11"
Set-MultiWriter $VM4 "Hard disk 12"
Set-MultiWriter $VM4 "Hard disk 13"
Set-MultiWriter $VM4 "Hard disk 14"