一、环境介绍

Redhat 5.8

Oracle 10g rac + ASM

目前数据库是一套Oracle 10g rac环境，使用的是ASM，ASM的存储空间来自多台存储设备，包括：XP12000、HDS等；由于XP12000存储已十分之老旧了，因此需要用新添加的HDS存储替换XP12000存储。

二、本地搭建测试环境（生产环境和测试环境中的ASM都是采用绑定裸设备的方式实现）

管理裸设备的启动和关闭（Redhat 4以前）

[root@rac2 rules.d]# cat /etc/sysconfig/rawdevices
# raw device bindings
# format:  <rawdev> <major> <minor>
#          <rawdev> <blockdev>
# example: /dev/raw/raw1 /dev/sda1
#          /dev/raw/raw2 8 5

/dev/raw/raw1 /dev/sdb
/dev/raw/raw2 /dev/sdc
/dev/raw/raw3 /dev/sdd
/dev/raw/raw4 /dev/sde
/dev/raw/raw5 /dev/sdf
/dev/raw/raw6 /dev/sdg
/dev/raw/raw7 /dev/sdh
/dev/raw/raw8 /dev/sdi
/dev/raw/raw9 /dev/sdj

/dev/raw/raw10 /dev/sdk
/dev/raw/raw11 /dev/sdl
/dev/raw/raw12 /dev/sdm
/dev/raw/raw13 /dev/sdn
/dev/raw/raw14 /dev/sdo
/dev/raw/raw15 /dev/sdp
/dev/raw/raw16 /dev/sdq
/dev/raw/raw17 /dev/sdr
/dev/raw/raw18 /dev/sds
/dev/raw/raw19 /dev/sdt
/dev/raw/raw20 /dev/sdu
/dev/raw/raw21 /dev/sdv
/dev/raw/raw22 /dev/sdw
[root@rac2 rules.d]#

在Redhat 5之后，原来的raw设备接口已经取消了，redhat 5中通过udev规则进行配置。

方法一、编辑/etc/udev/rules.d/60-raw.rules这个文件，添加如下内容：

[root@rac2 rules.d]# cat /etc/udev/rules.d/60-raw.rules 
# Enter raw device bindings here.
#
# An example would be:
#   ACTION=="add", KERNEL=="sda", RUN+="/bin/raw /dev/raw/raw1 %N"
# to bind /dev/raw/raw1 to /dev/sda, or
#   ACTION=="add", ENV{MAJOR}=="8", ENV{MINOR}=="1", RUN+="/bin/raw /dev/raw/raw2 %M %m"
# to bind /dev/raw/raw2 to the device with major 8, minor 1.

ACTION=="add", KERNEL=="sdb",RUN+="/bin/raw /dev/raw/raw1 %N"
ACTION=="add", KERNEL=="sdc",RUN+="/bin/raw /dev/raw/raw2 %N"
ACTION=="add", KERNEL=="sdd",RUN+="/bin/raw /dev/raw/raw3 %N"
ACTION=="add", KERNEL=="sde",RUN+="/bin/raw /dev/raw/raw4 %N"
ACTION=="add", KERNEL=="sdf",RUN+="/bin/raw /dev/raw/raw5 %N"
ACTION=="add", KERNEL=="sdg",RUN+="/bin/raw /dev/raw/raw6 %N"
ACTION=="add", KERNEL=="sdh",RUN+="/bin/raw /dev/raw/raw7 %N"
ACTION=="add", KERNEL=="sdi",RUN+="/bin/raw /dev/raw/raw8 %N"
ACTION=="add", KERNEL=="sdj",RUN+="/bin/raw /dev/raw/raw9 %N"

ACTION=="add", KERNEL=="sdk",RUN+="/bin/raw /dev/raw/raw10 %N"
ACTION=="add", KERNEL=="sdl",RUN+="/bin/raw /dev/raw/raw11 %N"
ACTION=="add", KERNEL=="sdm",RUN+="/bin/raw /dev/raw/raw12 %N"
ACTION=="add", KERNEL=="sdn",RUN+="/bin/raw /dev/raw/raw13 %N"
ACTION=="add", KERNEL=="sdo",RUN+="/bin/raw /dev/raw/raw14 %N"
ACTION=="add", KERNEL=="sdp",RUN+="/bin/raw /dev/raw/raw15 %N"
ACTION=="add", KERNEL=="sdq",RUN+="/bin/raw /dev/raw/raw16 %N"
ACTION=="add", KERNEL=="sdr",RUN+="/bin/raw /dev/raw/raw17 %N"
ACTION=="add", KERNEL=="sds",RUN+="/bin/raw /dev/raw/raw18 %N"
ACTION=="add", KERNEL=="sdt",RUN+="/bin/raw /dev/raw/raw19 %N"
ACTION=="add", KERNEL=="sdu",RUN+="/bin/raw /dev/raw/raw20 %N"
ACTION=="add", KERNEL=="sdv",RUN+="/bin/raw /dev/raw/raw21 %N"
ACTION=="add", KERNEL=="sdw",RUN+="/bin/raw /dev/raw/raw22 %N"
ACTION=="add",KERNEL=="raw[1-22]", OWNER="oracle", GROUP="oinstall", MODE="775"

[root@rac2 rules.d]#

启动raw设备

# start_udev 
Starting udev:                                             [  OK  ]

查看配置情况
[root@rac2 rules.d]# raw -qa
/dev/raw/raw1:    bound to major 8, minor 16
/dev/raw/raw2:    bound to major 8, minor 32
/dev/raw/raw3:    bound to major 8, minor 48
/dev/raw/raw4:    bound to major 8, minor 64
/dev/raw/raw5:    bound to major 8, minor 80
/dev/raw/raw6:    bound to major 8, minor 96
/dev/raw/raw7:    bound to major 8, minor 112
/dev/raw/raw8:    bound to major 8, minor 128
/dev/raw/raw9:    bound to major 8, minor 144
/dev/raw/raw10:    bound to major 8, minor 160
/dev/raw/raw11:    bound to major 8, minor 176
/dev/raw/raw12:    bound to major 8, minor 192
/dev/raw/raw13:    bound to major 8, minor 208
/dev/raw/raw14:    bound to major 8, minor 224
/dev/raw/raw15:    bound to major 8, minor 240
/dev/raw/raw16:    bound to major 65, minor 0
/dev/raw/raw17:    bound to major 65, minor 16
/dev/raw/raw18:    bound to major 65, minor 32
/dev/raw/raw19:    bound to major 65, minor 48
/dev/raw/raw20:    bound to major 65, minor 64
/dev/raw/raw21:    bound to major 65, minor 80
/dev/raw/raw22:    bound to major 65, minor 96
[root@rac2 rules.d]#

配置系统启动时加载裸设备：

[root@rac1 ~]# cat /etc/rc.d/rc.local
#!/bin/sh
#
# This script will be executed *after* all the other init scripts.
# You can put your own initialization stuff in here if you don't
# want to do the full Sys V style init stuff.

touch /var/lock/subsys/local

echo "chown oracle:oinstall /dev/raw/raw1" >> /etc/rc.d/rc.local
echo "chown oracle:oinstall /dev/raw/raw2" >> /etc/rc.d/rc.local
echo "chown oracle:oinstall /dev/raw/raw3" >> /etc/rc.d/rc.local
echo "chown oracle:oinstall /dev/raw/raw4" >> /etc/rc.d/rc.local
echo "chown oracle:oinstall /dev/raw/raw5" >> /etc/rc.d/rc.local
echo "chown oracle:oinstall /dev/raw/raw6" >> /etc/rc.d/rc.local
echo "chown oracle:oinstall /dev/raw/raw7" >> /etc/rc.d/rc.local
echo "chown oracle:oinstall /dev/raw/raw8" >> /etc/rc.d/rc.local
echo "chown oracle:oinstall /dev/raw/raw9" >> /etc/rc.d/rc.local

echo "chown oracle:oinstall /dev/raw/raw10" >> /etc/rc.d/rc.local
echo "chown oracle:oinstall /dev/raw/raw11" >> /etc/rc.d/rc.local
echo "chown oracle:oinstall /dev/raw/raw12" >> /etc/rc.d/rc.local
echo "chown oracle:oinstall /dev/raw/raw13" >> /etc/rc.d/rc.local
echo "chown oracle:oinstall /dev/raw/raw14" >> /etc/rc.d/rc.local
echo "chown oracle:oinstall /dev/raw/raw15" >> /etc/rc.d/rc.local
echo "chown oracle:oinstall /dev/raw/raw16" >> /etc/rc.d/rc.local
echo "chown oracle:oinstall /dev/raw/raw17" >> /etc/rc.d/rc.local
echo "chown oracle:oinstall /dev/raw/raw18" >> /etc/rc.d/rc.local
echo "chown oracle:oinstall /dev/raw/raw19" >> /etc/rc.d/rc.local
echo "chown oracle:oinstall /dev/raw/raw20" >> /etc/rc.d/rc.local
echo "chown oracle:oinstall /dev/raw/raw21" >> /etc/rc.d/rc.local
echo "chown oracle:oinstall /dev/raw/raw22" >> /etc/rc.d/rc.local

echo "chmod 775 /dev/raw/raw1" >> /etc/rc.d/rc.local
echo "chmod 775 /dev/raw/raw2" >> /etc/rc.d/rc.local
echo "chmod 775 /dev/raw/raw3" >> /etc/rc.d/rc.local
echo "chmod 775 /dev/raw/raw4" >> /etc/rc.d/rc.local
echo "chmod 775 /dev/raw/raw5" >> /etc/rc.d/rc.local
echo "chmod 775 /dev/raw/raw6" >> /etc/rc.d/rc.local
echo "chmod 775 /dev/raw/raw7" >> /etc/rc.d/rc.local
echo "chmod 775 /dev/raw/raw8" >> /etc/rc.d/rc.local
echo "chmod 775 /dev/raw/raw9" >> /etc/rc.d/rc.local
echo "chmod 775 /dev/raw/raw10" >> /etc/rc.d/rc.local
echo "chmod 775 /dev/raw/raw11" >> /etc/rc.d/rc.local
echo "chmod 775 /dev/raw/raw12" >> /etc/rc.d/rc.local
echo "chmod 775 /dev/raw/raw13" >> /etc/rc.d/rc.local
echo "chmod 775 /dev/raw/raw14" >> /etc/rc.d/rc.local
echo "chmod 775 /dev/raw/raw15" >> /etc/rc.d/rc.local
echo "chmod 775 /dev/raw/raw16" >> /etc/rc.d/rc.local
echo "chmod 775 /dev/raw/raw17" >> /etc/rc.d/rc.local
echo "chmod 775 /dev/raw/raw18" >> /etc/rc.d/rc.local
echo "chmod 775 /dev/raw/raw19" >> /etc/rc.d/rc.local
echo "chmod 775 /dev/raw/raw20" >> /etc/rc.d/rc.local
echo "chmod 775 /dev/raw/raw21" >> /etc/rc.d/rc.local
echo "chmod 775 /dev/raw/raw22" >> /etc/rc.d/rc.local

方法二：/etc/sysconfig/rawdevices

[root@rac2 ~]# cat /etc/sysconfig/rawdevices    （Redhat4前的版本）
# raw device bindings
# format:  <rawdev> <major> <minor>
#          <rawdev> <blockdev>
# example: /dev/raw/raw1 /dev/sda1
#          /dev/raw/raw2 8 5

/dev/raw/raw1 /dev/sdb
/dev/raw/raw2 /dev/sdc
/dev/raw/raw3 /dev/sdd
/dev/raw/raw4 /dev/sde
/dev/raw/raw5 /dev/sdf
/dev/raw/raw6 /dev/sdg
/dev/raw/raw7 /dev/sdh
/dev/raw/raw8 /dev/sdi
/dev/raw/raw9 /dev/sdj

/dev/raw/raw10 /dev/sdk
/dev/raw/raw11 /dev/sdl
/dev/raw/raw12 /dev/sdm
/dev/raw/raw13 /dev/sdn
/dev/raw/raw14 /dev/sdo
/dev/raw/raw15 /dev/sdp
/dev/raw/raw16 /dev/sdq
/dev/raw/raw17 /dev/sdr
/dev/raw/raw18 /dev/sds
/dev/raw/raw19 /dev/sdt
/dev/raw/raw20 /dev/sdu
/dev/raw/raw21 /dev/sdv
/dev/raw/raw22 /dev/sdw

[root@rac2 ~]#

配置裸设备权限，编辑/etc/udev/rules.d/60-raw.rules文件，添加如下内容：

[root@rac2 ~]# cat /etc/udev/rules.d/60-raw.rules 
# Enter raw device bindings here.
#
# An example would be:
#   ACTION=="add", KERNEL=="sda", RUN+="/bin/raw /dev/raw/raw1 %N"
# to bind /dev/raw/raw1 to /dev/sda, or
#   ACTION=="add", ENV{MAJOR}=="8", ENV{MINOR}=="1", RUN+="/bin/raw /dev/raw/raw2 %M %m"
# to bind /dev/raw/raw2 to the device with major 8, minor 1.

ACTION=="add", KERNEL=="sdb",RUN+="/bin/raw /dev/raw/raw1 %N"
ACTION=="add", KERNEL=="sdc",RUN+="/bin/raw /dev/raw/raw2 %N"
ACTION=="add", KERNEL=="sdd",RUN+="/bin/raw /dev/raw/raw3 %N"
ACTION=="add", KERNEL=="sde",RUN+="/bin/raw /dev/raw/raw4 %N"
ACTION=="add", KERNEL=="sdf",RUN+="/bin/raw /dev/raw/raw5 %N"
ACTION=="add", KERNEL=="sdg",RUN+="/bin/raw /dev/raw/raw6 %N"
ACTION=="add", KERNEL=="sdh",RUN+="/bin/raw /dev/raw/raw7 %N"
ACTION=="add", KERNEL=="sdi",RUN+="/bin/raw /dev/raw/raw8 %N"
ACTION=="add", KERNEL=="sdj",RUN+="/bin/raw /dev/raw/raw9 %N"
ACTION=="add", KERNEL=="sdk",RUN+="/bin/raw /dev/raw/raw10 %N"
ACTION=="add", KERNEL=="sdl",RUN+="/bin/raw /dev/raw/raw11 %N"
ACTION=="add", KERNEL=="sdm",RUN+="/bin/raw /dev/raw/raw12 %N"
ACTION=="add", KERNEL=="sdn",RUN+="/bin/raw /dev/raw/raw13 %N"
ACTION=="add", KERNEL=="sdo",RUN+="/bin/raw /dev/raw/raw14 %N"
ACTION=="add", KERNEL=="sdp",RUN+="/bin/raw /dev/raw/raw15 %N"
ACTION=="add", KERNEL=="sdq",RUN+="/bin/raw /dev/raw/raw16 %N"
ACTION=="add", KERNEL=="sdr",RUN+="/bin/raw /dev/raw/raw17 %N"
ACTION=="add", KERNEL=="sds",RUN+="/bin/raw /dev/raw/raw18 %N"
ACTION=="add", KERNEL=="sdt",RUN+="/bin/raw /dev/raw/raw19 %N"
ACTION=="add", KERNEL=="sdu",RUN+="/bin/raw /dev/raw/raw20 %N"
ACTION=="add", KERNEL=="sdv",RUN+="/bin/raw /dev/raw/raw21 %N"
ACTION=="add", KERNEL=="sdw",RUN+="/bin/raw /dev/raw/raw22 %N"
ACTION=="add", KERNEL=="raw[1-2]", OWNER="root", GROUP="oinstall", MODE="775"
ACTION=="add", KERNEL=="raw3", OWNER="oracle", GROUP="oinstall", MODE="775"
ACTION=="add", KERNEL=="raw4", OWNER="oracle", GROUP="oinstall", MODE="775"
ACTION=="add", KERNEL=="raw5", OWNER="oracle", GROUP="oinstall", MODE="775"
ACTION=="add", KERNEL=="raw6", OWNER="oracle", GROUP="oinstall", MODE="775"
ACTION=="add", KERNEL=="raw7", OWNER="oracle", GROUP="oinstall", MODE="775"
ACTION=="add", KERNEL=="raw8", OWNER="oracle", GROUP="oinstall", MODE="775"
ACTION=="add", KERNEL=="raw9", OWNER="oracle", GROUP="oinstall", MODE="775"
ACTION=="add", KERNEL=="raw10", OWNER="oracle", GROUP="oinstall", MODE="775"
ACTION=="add", KERNEL=="raw11", OWNER="oracle", GROUP="oinstall", MODE="775"
ACTION=="add", KERNEL=="raw12", OWNER="oracle", GROUP="oinstall", MODE="775"
ACTION=="add", KERNEL=="raw13", OWNER="oracle", GROUP="oinstall", MODE="775"
ACTION=="add", KERNEL=="raw14", OWNER="oracle", GROUP="oinstall", MODE="775"
ACTION=="add", KERNEL=="raw15", OWNER="oracle", GROUP="oinstall", MODE="775"
ACTION=="add", KERNEL=="raw16", OWNER="oracle", GROUP="oinstall", MODE="775"
ACTION=="add", KERNEL=="raw17", OWNER="oracle", GROUP="oinstall", MODE="775"
ACTION=="add", KERNEL=="raw18", OWNER="oracle", GROUP="oinstall", MODE="775"
ACTION=="add", KERNEL=="raw19", OWNER="oracle", GROUP="oinstall", MODE="775"
ACTION=="add", KERNEL=="raw20", OWNER="oracle", GROUP="oinstall", MODE="775"
ACTION=="add", KERNEL=="raw21", OWNER="oracle", GROUP="oinstall", MODE="775"
ACTION=="add", KERNEL=="raw22", OWNER="oracle", GROUP="oinstall", MODE="775"

[root@rac2 ~]#

设置裸设备开机自启动：

[root@rac2 ~]# cat /etc/rc.d/rc.local
#!/bin/sh
#
# This script will be executed *after* all the other init scripts.
# You can put your own initialization stuff in here if you don't
# want to do the full Sys V style init stuff.

touch /var/lock/subsys/local

/bin/raw /dev/raw/raw1 /dev/mapper/sdb
/bin/raw /dev/raw/raw2 /dev/mapper/sdc
/bin/raw /dev/raw/raw3 /dev/mapper/sdd
/bin/raw /dev/raw/raw4 /dev/mapper/sde
/bin/raw /dev/raw/raw5 /dev/mapper/sdf
/bin/raw /dev/raw/raw6 /dev/mapper/sdg
/bin/raw /dev/raw/raw7 /dev/mapper/sdh
/bin/raw /dev/raw/raw8 /dev/mapper/sdi
/bin/raw /dev/raw/raw9 /dev/mapper/sdj
/bin/raw /dev/raw/raw10 /dev/mapper/sdk
/bin/raw /dev/raw/raw11 /dev/mapper/sdl
/bin/raw /dev/raw/raw12 /dev/mapper/sdm
/bin/raw /dev/raw/raw13 /dev/mapper/sdn
/bin/raw /dev/raw/raw14 /dev/mapper/sdo
/bin/raw /dev/raw/raw15 /dev/mapper/sdp
/bin/raw /dev/raw/raw16 /dev/mapper/sdq
/bin/raw /dev/raw/raw17 /dev/mapper/sdr
/bin/raw /dev/raw/raw18 /dev/mapper/sds
/bin/raw /dev/raw/raw19 /dev/mapper/sdt
/bin/raw /dev/raw/raw20 /dev/mapper/sdu
/bin/raw /dev/raw/raw21 /dev/mapper/sdv
/bin/raw /dev/raw/raw22 /dev/mapper/sdw

[root@rac2 ~]#

查看裸设备权限：

[root@rac2 ~]#  ls -lrt /dev/raw/raw*
crwxrwxr-x 1 oracle oinstall 162,  6 Sep 26 11:19 /dev/raw/raw6
crwxrwxr-x 1 oracle oinstall 162, 11 Sep 26 11:19 /dev/raw/raw11
crwxrwxr-x 1 oracle oinstall 162, 10 Sep 26 11:19 /dev/raw/raw10
crwxrwxr-x 1 oracle oinstall 162, 13 Sep 26 11:19 /dev/raw/raw13
crwxrwxr-x 1 oracle oinstall 162, 12 Sep 26 11:19 /dev/raw/raw12
crwxrwxr-x 1 oracle oinstall 162, 16 Sep 26 11:19 /dev/raw/raw16
crwxrwxr-x 1 oracle oinstall 162, 15 Sep 26 11:19 /dev/raw/raw15
crwxrwxr-x 1 oracle oinstall 162, 14 Sep 26 11:19 /dev/raw/raw14
crwxrwxr-x 1 oracle oinstall 162, 21 Sep 26 11:19 /dev/raw/raw21
crwxrwxr-x 1 oracle oinstall 162, 20 Sep 26 11:19 /dev/raw/raw20
crwxrwxr-x 1 oracle oinstall 162, 19 Sep 26 11:19 /dev/raw/raw19
crwxrwxr-x 1 oracle oinstall 162, 17 Sep 26 11:19 /dev/raw/raw17
crwxrwxr-x 1 oracle oinstall 162, 18 Sep 26 11:19 /dev/raw/raw18
crwxrwxr-x 1 oracle oinstall 162, 22 Sep 26 11:19 /dev/raw/raw22
crwxrwxr-x 1 root   oinstall 162,  2 Sep 26 11:26 /dev/raw/raw2
crwxrwxr-x 1 root   oinstall 162,  1 Sep 26 11:26 /dev/raw/raw1
crwxrwxr-x 1 oracle oinstall 162,  9 Sep 26 11:34 /dev/raw/raw9
crwxrwxr-x 1 oracle oinstall 162,  8 Sep 26 11:34 /dev/raw/raw8
crwxrwxr-x 1 oracle oinstall 162,  7 Sep 26 11:34 /dev/raw/raw7
crwxrwxr-x 1 oracle oinstall 162,  4 Sep 26 11:34 /dev/raw/raw4
crwxrwxr-x 1 oracle oinstall 162,  3 Sep 26 11:34 /dev/raw/raw3
crwxrwxr-x 1 oracle oinstall 162,  5 Sep 26 11:34 /dev/raw/raw5
[root@rac2 ~]#

[root@rac2 ~]# raw -qa
/dev/raw/raw1:    bound to major 8, minor 16
/dev/raw/raw2:    bound to major 8, minor 32
/dev/raw/raw3:    bound to major 8, minor 48
/dev/raw/raw4:    bound to major 8, minor 64
/dev/raw/raw5:    bound to major 8, minor 80
/dev/raw/raw6:    bound to major 8, minor 96
/dev/raw/raw7:    bound to major 8, minor 112
/dev/raw/raw8:    bound to major 8, minor 128
/dev/raw/raw9:    bound to major 8, minor 144
/dev/raw/raw10:    bound to major 8, minor 160
/dev/raw/raw11:    bound to major 8, minor 176
/dev/raw/raw12:    bound to major 8, minor 192
/dev/raw/raw13:    bound to major 8, minor 208
/dev/raw/raw14:    bound to major 8, minor 224
/dev/raw/raw15:    bound to major 8, minor 240
/dev/raw/raw16:    bound to major 65, minor 0
/dev/raw/raw17:    bound to major 65, minor 16
/dev/raw/raw18:    bound to major 65, minor 32
/dev/raw/raw19:    bound to major 65, minor 48
/dev/raw/raw20:    bound to major 65, minor 64
/dev/raw/raw21:    bound to major 65, minor 80
/dev/raw/raw22:    bound to major 65, minor 96
[root@rac2 ~]#

三、安装Oracle 10g cluster + database （略）

四、查看ASM磁盘信息

[oracle@rac2 ~]$ export ORACLE_SID=+ASM2
[oracle@rac2 ~]$ sqlplus / as sysdba

SQL*Plus: Release 10.2.0.5.0 - Production on Wed Sep 26 12:32:27 2018

Copyright (c) 1982, 2010, Oracle.  All Rights Reserved.

Connected to:
Oracle Database 10g Enterprise Edition Release 10.2.0.5.0 - 64bit Production
With the Partitioning, Real Application Clusters, OLAP, Data Mining
and Real Application Testing options

SQL> set linesize 120
SQL> col path format a28
SQL> desc v$asm_disk
 Name                                   Null?    Type
 ----------------------------------------------------------------- -------- --------------------------------------------
 GROUP_NUMBER                                    NUMBER
 DISK_NUMBER                                    NUMBER
 COMPOUND_INDEX                                 NUMBER
 INCARNATION                                    NUMBER
 MOUNT_STATUS                                    VARCHAR2(7)
 HEADER_STATUS                                    VARCHAR2(12)
 MODE_STATUS                                    VARCHAR2(7)
 STATE                                        VARCHAR2(8)
 REDUNDANCY                                    VARCHAR2(7)
 LIBRARY                                    VARCHAR2(64)
 TOTAL_MB                                    NUMBER
 FREE_MB                                    NUMBER
 NAME                                        VARCHAR2(30)
 FAILGROUP                                    VARCHAR2(30)
 LABEL                                        VARCHAR2(31)
 PATH                                        VARCHAR2(256)
 UDID                                        VARCHAR2(64)
 PRODUCT                                    VARCHAR2(32)
 CREATE_DATE                                    DATE
 MOUNT_DATE                                    DATE
 REPAIR_TIMER                                    NUMBER
 READS                                        NUMBER
 WRITES                                     NUMBER
 READ_ERRS                                    NUMBER
 WRITE_ERRS                                    NUMBER
 READ_TIME                                    NUMBER
 WRITE_TIME                                    NUMBER
 BYTES_READ                                    NUMBER
 BYTES_WRITTEN                                    NUMBER

SQL> select GROUP_NUMBER,DISK_NUMBER,MOUNT_STATUS,HEADER_STATUS,MODE_STATUS,STATE,PATH,TOTAL_MB from v$asm_disk;

GROUP_NUMBER DISK_NUMBER MOUNT_S HEADER_STATU MODE_ST STATE    PATH                  TOTAL_MB
------------ ----------- ------- ------------ ------- -------- ---------------------------- ----------
       0           0 CLOSED  CANDIDATE    ONLINE  NORMAL   /dev/raw/raw22              3072
       0           1 CLOSED  CANDIDATE    ONLINE  NORMAL   /dev/raw/raw18              3072
       0          21 CLOSED  FOREIGN      ONLINE  NORMAL   /dev/raw/raw1               512
       0          20 CLOSED  FOREIGN      ONLINE  NORMAL   /dev/raw/raw2               512
       0           2 CLOSED  CANDIDATE    ONLINE  NORMAL   /dev/raw/raw17               512
       0           5 CLOSED  FOREIGN      ONLINE  NORMAL   /dev/raw/raw4               512
       0           6 CLOSED  CANDIDATE    ONLINE  NORMAL   /dev/raw/raw19              3072
       0           7 CLOSED  CANDIDATE    ONLINE  NORMAL   /dev/raw/raw20              3072
       0           8 CLOSED  CANDIDATE    ONLINE  NORMAL   /dev/raw/raw21              3072
       0           9 CLOSED  CANDIDATE    ONLINE  NORMAL   /dev/raw/raw14               512
       0          10 CLOSED  CANDIDATE    ONLINE  NORMAL   /dev/raw/raw15               512

GROUP_NUMBER DISK_NUMBER MOUNT_S HEADER_STATU MODE_ST STATE    PATH                  TOTAL_MB
------------ ----------- ------- ------------ ------- -------- ---------------------------- ----------
       0          11 CLOSED  CANDIDATE    ONLINE  NORMAL   /dev/raw/raw16               512
       0          13 CLOSED  CANDIDATE    ONLINE  NORMAL   /dev/raw/raw12               512
       0          14 CLOSED  CANDIDATE    ONLINE  NORMAL   /dev/raw/raw13               512
       0          15 CLOSED  CANDIDATE    ONLINE  NORMAL   /dev/raw/raw11               512
       0          16 CLOSED  CANDIDATE    ONLINE  NORMAL   /dev/raw/raw10               512
       0          17 CLOSED  CANDIDATE    ONLINE  NORMAL   /dev/raw/raw6               512
       0          18 CLOSED  FOREIGN      ONLINE  NORMAL   /dev/raw/raw5               512
       0          19 CLOSED  FOREIGN      ONLINE  NORMAL   /dev/raw/raw3               512
       1           1 CACHED  MEMBER       ONLINE  NORMAL   /dev/raw/raw8              3072
       1           0 CACHED  MEMBER       ONLINE  NORMAL   /dev/raw/raw7              3072
       1           2 CACHED  MEMBER       ONLINE  NORMAL   /dev/raw/raw9              3072

22 rows selected.

SQL>

五、查看votedisk磁盘信息：

[oracle@rac2 ~]$ crsctl query css votedisk
 0.     0    /dev/raw/raw3                            ----------- 3、4、5为votedisk文件
 1.     0    /dev/raw/raw4
 2.     0    /dev/raw/raw5

located 3 votedisk(s).
[oracle@rac2 ~]$

六、查看ocssd进程号定位的votedisk文件

[oracle@rac2 fd]$ ps -e | grep -i ocssd
 6650 ?        00:00:09 ocssd.bin
[oracle@rac2 fd]$ 
[oracle@rac2 fd]$ cd /proc/6650/fd
[oracle@rac2 fd]$ ll
total 0
lr-x------ 1 oracle oinstall 64 Sep 26 14:23 0 -> /dev/null
lrwx------ 1 oracle oinstall 64 Sep 26 14:23 1 -> /u01/app/oracle/product/10.2.0/crs/log/rac2/cssd/cssdOUT.log
lrwx------ 1 oracle oinstall 64 Sep 26 14:23 10 -> socket:[19873]
lrwx------ 1 oracle oinstall 64 Sep 26 14:23 11 -> /dev/raw/raw3    ----- 3、4、5为votedisk文件
lrwx------ 1 oracle oinstall 64 Sep 26 14:23 12 -> /dev/raw/raw3
lrwx------ 1 oracle oinstall 64 Sep 26 14:23 13 -> /dev/raw/raw4
lrwx------ 1 oracle oinstall 64 Sep 26 14:23 14 -> /dev/raw/raw4
lrwx------ 1 oracle oinstall 64 Sep 26 14:23 15 -> /dev/raw/raw5
lrwx------ 1 oracle oinstall 64 Sep 26 14:23 16 -> /dev/raw/raw5
l-wx------ 1 oracle oinstall 64 Sep 26 14:23 17 -> /u01/app/oracle/product/10.2.0/crs/log/rac2/alertrac2.log
lr-x------ 1 oracle oinstall 64 Sep 26 14:23 18 -> /u01/app/oracle/product/10.2.0/crs/has/mesg/clsdus.msb
lrwx------ 1 oracle oinstall 64 Sep 26 14:23 19 -> socket:[20162]
lrwx------ 1 oracle oinstall 64 Sep 26 14:23 2 -> /u01/app/oracle/product/10.2.0/crs/log/rac2/cssd/cssdOUT.log
lrwx------ 1 oracle oinstall 64 Sep 26 14:23 20 -> socket:[20165]
lrwx------ 1 oracle oinstall 64 Sep 26 14:23 21 -> socket:[21303]
lrwx------ 1 oracle oinstall 64 Sep 26 14:23 22 -> socket:[20171]
lrwx------ 1 oracle oinstall 64 Sep 26 14:23 23 -> socket:[20172]
lrwx------ 1 oracle oinstall 64 Sep 26 14:23 24 -> socket:[20174]
lrwx------ 1 oracle oinstall 64 Sep 26 14:23 25 -> socket:[20176]
lrwx------ 1 oracle oinstall 64 Sep 26 14:23 26 -> socket:[20178]
lrwx------ 1 oracle oinstall 64 Sep 26 14:23 27 -> socket:[20179]
lrwx------ 1 oracle oinstall 64 Sep 26 14:23 28 -> socket:[21304]
lrwx------ 1 oracle oinstall 64 Sep 26 14:23 29 -> socket:[21305]
lrwx------ 1 oracle oinstall 64 Sep 26 14:23 3 -> /u01/app/oracle/product/10.2.0/crs/log/rac2/cssd/cssdOUT.log
lrwx------ 1 oracle oinstall 64 Sep 26 14:23 30 -> socket:[22196]
lrwx------ 1 oracle oinstall 64 Sep 26 14:23 31 -> socket:[24417]
lrwx------ 1 oracle oinstall 64 Sep 26 14:23 32 -> socket:[21707]
lrwx------ 1 oracle oinstall 64 Sep 26 14:23 33 -> socket:[21826]
lrwx------ 1 oracle oinstall 64 Sep 26 14:23 34 -> socket:[21844]
lrwx------ 1 oracle oinstall 64 Sep 26 14:23 35 -> socket:[22103]
lrwx------ 1 oracle oinstall 64 Sep 26 14:23 36 -> socket:[21896]
lrwx------ 1 oracle oinstall 64 Sep 26 14:23 37 -> socket:[22074]
lrwx------ 1 oracle oinstall 64 Sep 26 14:23 38 -> socket:[22333]
lrwx------ 1 oracle oinstall 64 Sep 26 14:23 39 -> socket:[22687]
lr-x------ 1 oracle oinstall 64 Sep 26 14:23 4 -> /u01/app/oracle/product/10.2.0/crs/srvm/mesg/procus.msb
lrwx------ 1 oracle oinstall 64 Sep 26 14:23 40 -> socket:[22864]
lrwx------ 1 oracle oinstall 64 Sep 26 14:23 41 -> socket:[22886]
lrwx------ 1 oracle oinstall 64 Sep 26 14:23 42 -> socket:[22927]
lrwx------ 1 oracle oinstall 64 Sep 26 14:23 43 -> socket:[23003]
lrwx------ 1 oracle oinstall 64 Sep 26 14:23 44 -> socket:[23074]
lrwx------ 1 oracle oinstall 64 Sep 26 14:23 45 -> socket:[23029]
lrwx------ 1 oracle oinstall 64 Sep 26 14:23 46 -> socket:[29402]
lrwx------ 1 oracle oinstall 64 Sep 26 14:23 47 -> socket:[23138]
lrwx------ 1 oracle oinstall 64 Sep 26 14:23 48 -> socket:[28492]
lrwx------ 1 oracle oinstall 64 Sep 26 14:23 49 -> socket:[28543]
lr-x------ 1 oracle oinstall 64 Sep 26 14:23 5 -> /dev/raw/raw1
lrwx------ 1 oracle oinstall 64 Sep 26 14:23 50 -> socket:[29400]
lr-x------ 1 oracle oinstall 64 Sep 26 14:23 6 -> /dev/raw/raw2
lr-x------ 1 oracle oinstall 64 Sep 26 14:23 7 -> /u01/app/oracle/product/10.2.0/crs/css/mesg/clssus.msb
l-wx------ 1 oracle oinstall 64 Sep 26 14:23 8 -> /u01/app/oracle/product/10.2.0/crs/log/rac2/cssd/ocssd.log
lrwx------ 1 oracle oinstall 64 Sep 26 14:23 9 -> socket:[19872]
[oracle@rac2 fd]$

七、备份votedisk文件

[root@rac2 ~]# dd if=/dev/raw/raw3 of=/u01/vote1.exp
1048576+0 records in
1048576+0 records out
536870912 bytes (537 MB) copied, 423.355 seconds, 1.3 MB/s
[root@rac2 ~]# 
[root@rac2 ~]# dd if=/dev/raw/raw4 of=/u01/vote2.exp
1048576+0 records in
1048576+0 records out
536870912 bytes (537 MB) copied, 431.888 seconds, 1.2 MB/s
[root@rac2 ~]# 
[root@rac2 ~]# dd if=/dev/raw/raw5 of=/u01/vote3.exp
1048576+0 records in
1048576+0 records out
536870912 bytes (537 MB) copied, 414.658 seconds, 1.3 MB/s
[root@rac2 ~]#

[oracle@rac2 u01]$ ls
app  initrac1.ora  patch2.log  vote1.exp  vote2.exp  vote3.exp
[oracle@rac2 u01]$

八、每个节点关闭crs，离线替换votedisk磁盘

[root@rac2 ~]# cd /u01/app/oracle/product/10.2.0/crs/bin/
[root@rac2 bin]# ./crsctl stop crs
Stopping resources. This could take several minutes.
Successfully stopped CRS resources.
Stopping CSSD.
Shutting down CSS daemon.
Shutdown request successfully issued.
[root@rac2 bin]#

九、删除原来的votedisk磁盘并添加新的votedisk磁盘：

[root@rac2 bin]# ./crsctl delete css votedisk /dev/raw/raw3 -force
successful deletion of votedisk /dev/raw/raw3.
[root@rac2 bin]# 
[root@rac2 bin]# ./crsctl add css votedisk /dev/raw/raw13 -force
Now formatting voting disk: /dev/raw/raw13
successful addition of votedisk /dev/raw/raw13.
[root@rac2 bin]# 
[root@rac2 bin]# ./crsctl delete css votedisk /dev/raw/raw4 -force
successful deletion of votedisk /dev/raw/raw4.
[root@rac2 bin]# 
[root@rac2 bin]# ./crsctl add css votedisk /dev/raw/raw14 -force
Now formatting voting disk: /dev/raw/raw14
successful addition of votedisk /dev/raw/raw14.
[root@rac2 bin]# 
[root@rac2 bin]# ./crsctl delete css votedisk /dev/raw/raw5 -force
successful deletion of votedisk /dev/raw/raw5.
[root@rac2 bin]# 
[root@rac2 bin]# ./crsctl add css votedisk /dev/raw/raw15 -force
Now formatting voting disk: /dev/raw/raw15
successful addition of votedisk /dev/raw/raw15.
[root@rac2 bin]#

十、查看votedisk的位置及名称并启动crs，检查cluster的运行情况：

[root@rac2 bin]# ./crsctl query css votedisk
 0.     0    /dev/raw/raw13
 1.     0    /dev/raw/raw14
 2.     0    /dev/raw/raw15

located 3 votedisk(s).
[root@rac2 bin]#

[root@rac2 bin]# ./crsctl start crs
Attempting to start CRS stack 
The CRS stack will be started shortly
[root@rac2 bin]#

[root@rac2 bin]# ./crsctl check crs
CSS appears healthy
CRS appears healthy
EVM appears healthy
[root@rac2 bin]#

[oracle@rac1 ~]$ crs_stat -t -v
Name           Type           R/RA   F/FT   Target    State     Host        
----------------------------------------------------------------------
ora.rac.db     application    0/1    0/1    ONLINE    ONLINE    rac1        
ora....c1.inst application    0/5    0/0    ONLINE    ONLINE    rac1        
ora....c2.inst application    0/5    0/0    ONLINE    ONLINE    rac2        
ora....SM1.asm application    0/5    0/0    ONLINE    ONLINE    rac1        
ora....C1.lsnr application    0/5    0/0    ONLINE    ONLINE    rac1        
ora.rac1.gsd   application    0/5    0/0    ONLINE    ONLINE    rac1        
ora.rac1.ons   application    0/3    0/0    ONLINE    ONLINE    rac1        
ora.rac1.vip   application    0/0    0/0    ONLINE    ONLINE    rac1        
ora....SM2.asm application    0/5    0/0    ONLINE    ONLINE    rac2        
ora....C2.lsnr application    0/5    0/0    ONLINE    ONLINE    rac2        
ora.rac2.gsd   application    0/5    0/0    ONLINE    ONLINE    rac2        
ora.rac2.ons   application    0/3    0/0    ONLINE    ONLINE    rac2        
ora.rac2.vip   application    0/0    0/0    ONLINE    ONLINE    rac2        
[oracle@rac1 ~]$

[oracle@rac1 ~]$ crsctl query css votedisk
 0.     0    /dev/raw/raw13            ----votedisk文件已由3、4、5替换为13、14、15
 1.     0    /dev/raw/raw14
 2.     0    /dev/raw/raw15

located 3 votedisk(s).
[oracle@rac1 ~]$

十一、再次查看ocssd进程号定位的votedisk文件

[oracle@rac2 u01]$ ps -e | grep -i ocssd
 3704 ?        00:00:00 ocssd.bin
[oracle@rac2 u01]$ 
[oracle@rac2 u01]$ cd /proc/3704/fd
[oracle@rac2 fd]$ 
[oracle@rac2 fd]$ ll
total 0
lr-x------ 1 oracle oinstall 64 Sep 26 15:16 0 -> /dev/null
lrwx------ 1 oracle oinstall 64 Sep 26 15:16 1 -> /u01/app/oracle/product/10.2.0/crs/log/rac2/cssd/cssdOUT.log
lrwx------ 1 oracle oinstall 64 Sep 26 15:16 10 -> socket:[89476]
lrwx------ 1 oracle oinstall 64 Sep 26 15:16 11 -> /dev/raw/raw13    ----votedisk文件已由3、4、5替换为13、14、15
lrwx------ 1 oracle oinstall 64 Sep 26 15:16 12 -> /dev/raw/raw13
lrwx------ 1 oracle oinstall 64 Sep 26 15:16 13 -> /dev/raw/raw14
lrwx------ 1 oracle oinstall 64 Sep 26 15:16 14 -> /dev/raw/raw14
lrwx------ 1 oracle oinstall 64 Sep 26 15:16 15 -> /dev/raw/raw15
lrwx------ 1 oracle oinstall 64 Sep 26 15:16 16 -> /dev/raw/raw15
l-wx------ 1 oracle oinstall 64 Sep 26 15:16 17 -> /u01/app/oracle/product/10.2.0/crs/log/rac2/alertrac2.log
lr-x------ 1 oracle oinstall 64 Sep 26 15:16 18 -> /u01/app/oracle/product/10.2.0/crs/has/mesg/clsdus.msb
lrwx------ 1 oracle oinstall 64 Sep 26 15:16 19 -> socket:[89616]
lrwx------ 1 oracle oinstall 64 Sep 26 15:16 2 -> /u01/app/oracle/product/10.2.0/crs/log/rac2/cssd/cssdOUT.log
lrwx------ 1 oracle oinstall 64 Sep 26 15:16 20 -> socket:[89619]
lrwx------ 1 oracle oinstall 64 Sep 26 15:16 21 -> socket:[89656]
lrwx------ 1 oracle oinstall 64 Sep 26 15:16 22 -> socket:[89627]
lrwx------ 1 oracle oinstall 64 Sep 26 15:16 23 -> socket:[89630]
lrwx------ 1 oracle oinstall 64 Sep 26 15:16 24 -> socket:[89632]
lrwx------ 1 oracle oinstall 64 Sep 26 15:16 25 -> socket:[89634]
lrwx------ 1 oracle oinstall 64 Sep 26 15:16 26 -> socket:[89636]
lrwx------ 1 oracle oinstall 64 Sep 26 15:16 27 -> socket:[89637]
lrwx------ 1 oracle oinstall 64 Sep 26 15:16 28 -> socket:[89658]
lrwx------ 1 oracle oinstall 64 Sep 26 15:16 29 -> socket:[89659]
lrwx------ 1 oracle oinstall 64 Sep 26 15:16 3 -> /u01/app/oracle/product/10.2.0/crs/log/rac2/cssd/cssdOUT.log
lrwx------ 1 oracle oinstall 64 Sep 26 15:16 30 -> socket:[89660]
lrwx------ 1 oracle oinstall 64 Sep 26 15:16 31 -> socket:[89661]
lrwx------ 1 oracle oinstall 64 Sep 26 15:16 33 -> socket:[90493]
lrwx------ 1 oracle oinstall 64 Sep 26 15:16 34 -> socket:[90115]
lrwx------ 1 oracle oinstall 64 Sep 26 15:16 35 -> socket:[90234]
lrwx------ 1 oracle oinstall 64 Sep 26 15:16 36 -> socket:[90253]
lrwx------ 1 oracle oinstall 64 Sep 26 15:16 37 -> socket:[90592]
lrwx------ 1 oracle oinstall 64 Sep 26 15:16 38 -> socket:[90304]
lrwx------ 1 oracle oinstall 64 Sep 26 15:16 39 -> socket:[90729]
lr-x------ 1 oracle oinstall 64 Sep 26 15:16 4 -> /u01/app/oracle/product/10.2.0/crs/srvm/mesg/procus.msb
lrwx------ 1 oracle oinstall 64 Sep 26 15:16 40 -> socket:[90919]
lrwx------ 1 oracle oinstall 64 Sep 26 15:16 41 -> socket:[90996]
lrwx------ 1 oracle oinstall 64 Sep 26 15:16 42 -> socket:[91174]
lrwx------ 1 oracle oinstall 64 Sep 26 15:16 43 -> socket:[91194]
lrwx------ 1 oracle oinstall 64 Sep 26 15:16 44 -> socket:[91235]
lrwx------ 1 oracle oinstall 64 Sep 26 15:16 45 -> socket:[91310]
lrwx------ 1 oracle oinstall 64 Sep 26 15:16 47 -> socket:[91337]
lrwx------ 1 oracle oinstall 64 Sep 26 15:16 48 -> socket:[91356]
lrwx------ 1 oracle oinstall 64 Sep 26 15:16 49 -> socket:[91442]
lr-x------ 1 oracle oinstall 64 Sep 26 15:16 5 -> /dev/raw/raw1
lrwx------ 1 oracle oinstall 64 Sep 26 15:16 50 -> socket:[91454]
lrwx------ 1 oracle oinstall 64 Sep 26 15:16 51 -> socket:[91466]
lrwx------ 1 oracle oinstall 64 Sep 26 15:16 52 -> socket:[91478]
lr-x------ 1 oracle oinstall 64 Sep 26 15:16 6 -> /dev/raw/raw2
lr-x------ 1 oracle oinstall 64 Sep 26 15:16 7 -> /u01/app/oracle/product/10.2.0/crs/css/mesg/clssus.msb
l-wx------ 1 oracle oinstall 64 Sep 26 15:16 8 -> /u01/app/oracle/product/10.2.0/crs/log/rac2/cssd/ocssd.log
lrwx------ 1 oracle oinstall 64 Sep 26 15:16 9 -> socket:[89475]
[oracle@rac2 fd]$

十二、查看ocr的磁盘信息：

[oracle@rac2 fd]$ ocrcheck
Status of Oracle Cluster Registry is as follows :
     Version                  :          2
     Total space (kbytes)     :     524040
     Used space (kbytes)      :       3860
     Available space (kbytes) :     520180
     ID                       :  687142446
     Device/File Name         : /dev/raw/raw1
                                    Device/File integrity check succeeded
     Device/File Name         : /dev/raw/raw2
                                    Device/File integrity check succeeded

Cluster registry integrity check succeeded

[oracle@rac2 fd]$

[oracle@rac2 fd]$ ocrconfig -showbackup

rac2     2018/08/05 19:14:21     /u01/app/oracle/product/10.2.0/crs/cdata/crs

rac2     2018/08/05 15:14:21     /u01/app/oracle/product/10.2.0/crs/cdata/crs

rac1     2018/07/13 16:39:13     /u01/app/oracle/product/10.2.0/crs/cdata/crs

rac2     2018/08/05 15:14:21     /u01/app/oracle/product/10.2.0/crs/cdata/crs

rac2     2018/08/05 15:14:21     /u01/app/oracle/product/10.2.0/crs/cdata/crs
[oracle@rac2 fd]$

十三、关闭crs手动备份ocr信息：

[root@rac2 bin]# ./crsctl stop crs
Stopping resources. This could take several minutes.
Successfully stopped CRS resources.
Stopping CSSD.
Shutting down CSS daemon.
Shutdown request successfully issued.
[root@rac2 bin]#

[root@rac2 bin]# ./ocrconfig -export /u01/app/ocr_bak.exp
[root@rac2 bin]#

root@rac2 ~]# cd /u01/app
[root@rac2 app]# ls
ocr_bak.exp  oracle
[root@rac2 app]#

十四、各节点启动crs并查看ocssd进程定位到的ocr文件：

[root@rac2 bin]# ./crsctl start crs
Attempting to start CRS stack 
The CRS stack will be started shortly
[root@rac2 bin]#

[oracle@rac2 fd]$ ps -e | grep -i ocssd
12577 ?        00:00:00 ocssd.bin
[oracle@rac2 fd]$ 
[oracle@rac2 fd]$ cd /proc/12577/fd
[oracle@rac2 fd]$ 
[oracle@rac2 fd]$ ll
total 0
lr-x------ 1 oracle oinstall 64 Sep 26 15:30 0 -> /dev/null
lrwx------ 1 oracle oinstall 64 Sep 26 15:30 1 -> /u01/app/oracle/product/10.2.0/crs/log/rac2/cssd/cssdOUT.log
lrwx------ 1 oracle oinstall 64 Sep 26 15:30 10 -> socket:[101050]
lrwx------ 1 oracle oinstall 64 Sep 26 15:30 11 -> /dev/raw/raw13
lrwx------ 1 oracle oinstall 64 Sep 26 15:30 12 -> /dev/raw/raw13
lrwx------ 1 oracle oinstall 64 Sep 26 15:30 13 -> /dev/raw/raw14
lrwx------ 1 oracle oinstall 64 Sep 26 15:30 14 -> /dev/raw/raw14
lrwx------ 1 oracle oinstall 64 Sep 26 15:30 15 -> /dev/raw/raw15
lrwx------ 1 oracle oinstall 64 Sep 26 15:30 16 -> /dev/raw/raw15
l-wx------ 1 oracle oinstall 64 Sep 26 15:30 17 -> /u01/app/oracle/product/10.2.0/crs/log/rac2/alertrac2.log
lr-x------ 1 oracle oinstall 64 Sep 26 15:30 18 -> /u01/app/oracle/product/10.2.0/crs/has/mesg/clsdus.msb
lrwx------ 1 oracle oinstall 64 Sep 26 15:30 19 -> socket:[101194]
lrwx------ 1 oracle oinstall 64 Sep 26 15:30 2 -> /u01/app/oracle/product/10.2.0/crs/log/rac2/cssd/cssdOUT.log
lrwx------ 1 oracle oinstall 64 Sep 26 15:30 20 -> socket:[101197]
lrwx------ 1 oracle oinstall 64 Sep 26 15:30 21 -> socket:[101225]
lrwx------ 1 oracle oinstall 64 Sep 26 15:30 22 -> socket:[101199]
lrwx------ 1 oracle oinstall 64 Sep 26 15:30 23 -> socket:[101204]
lrwx------ 1 oracle oinstall 64 Sep 26 15:30 24 -> socket:[101206]
lrwx------ 1 oracle oinstall 64 Sep 26 15:30 25 -> socket:[101208]
lrwx------ 1 oracle oinstall 64 Sep 26 15:30 26 -> socket:[101210]
lrwx------ 1 oracle oinstall 64 Sep 26 15:30 27 -> socket:[101211]
lrwx------ 1 oracle oinstall 64 Sep 26 15:30 28 -> socket:[101226]
lrwx------ 1 oracle oinstall 64 Sep 26 15:30 29 -> socket:[101227]
lrwx------ 1 oracle oinstall 64 Sep 26 15:30 3 -> /u01/app/oracle/product/10.2.0/crs/log/rac2/cssd/cssdOUT.log
lrwx------ 1 oracle oinstall 64 Sep 26 15:30 31 -> socket:[102108]
lrwx------ 1 oracle oinstall 64 Sep 26 15:30 32 -> socket:[101626]
lrwx------ 1 oracle oinstall 64 Sep 26 15:30 33 -> socket:[101745]
lrwx------ 1 oracle oinstall 64 Sep 26 15:30 34 -> socket:[101764]
lrwx------ 1 oracle oinstall 64 Sep 26 15:30 35 -> socket:[102257]
lrwx------ 1 oracle oinstall 64 Sep 26 15:30 36 -> socket:[101815]
lrwx------ 1 oracle oinstall 64 Sep 26 15:30 37 -> socket:[102195]
lrwx------ 1 oracle oinstall 64 Sep 26 15:30 38 -> socket:[101944]
lrwx------ 1 oracle oinstall 64 Sep 26 15:30 39 -> socket:[101951]
lr-x------ 1 oracle oinstall 64 Sep 26 15:30 4 -> /u01/app/oracle/product/10.2.0/crs/srvm/mesg/procus.msb
lrwx------ 1 oracle oinstall 64 Sep 26 15:30 40 -> socket:[101991]
lrwx------ 1 oracle oinstall 64 Sep 26 15:30 41 -> socket:[102510]
lrwx------ 1 oracle oinstall 64 Sep 26 15:30 42 -> socket:[102688]
lrwx------ 1 oracle oinstall 64 Sep 26 15:30 43 -> socket:[102708]
lrwx------ 1 oracle oinstall 64 Sep 26 15:30 44 -> socket:[102749]
lrwx------ 1 oracle oinstall 64 Sep 26 15:30 45 -> socket:[102825]
lrwx------ 1 oracle oinstall 64 Sep 26 15:30 47 -> socket:[102880]
lrwx------ 1 oracle oinstall 64 Sep 26 15:30 48 -> socket:[102948]
lrwx------ 1 oracle oinstall 64 Sep 26 15:30 49 -> socket:[102974]
lr-x------ 1 oracle oinstall 64 Sep 26 15:30 5 -> /dev/raw/raw1         ----raw1为ocr文件
lrwx------ 1 oracle oinstall 64 Sep 26 15:30 50 -> socket:[102986]
lrwx------ 1 oracle oinstall 64 Sep 26 15:30 51 -> socket:[102998]
lrwx------ 1 oracle oinstall 64 Sep 26 15:30 52 -> socket:[103010]
lr-x------ 1 oracle oinstall 64 Sep 26 15:30 6 -> /dev/raw/raw2        ----raw2为raw1的镜像文件
lr-x------ 1 oracle oinstall 64 Sep 26 15:30 7 -> /u01/app/oracle/product/10.2.0/crs/css/mesg/clssus.msb
l-wx------ 1 oracle oinstall 64 Sep 26 15:30 8 -> /u01/app/oracle/product/10.2.0/crs/log/rac2/cssd/ocssd.log
lrwx------ 1 oracle oinstall 64 Sep 26 15:30 9 -> socket:[101049]
[oracle@rac2 fd]$

十五、现在来给ocr添加一个镜像：（注意：如果ocr只有一个，没有镜像的话，是不能在线更改ocr位置的，所以必须给ocr添加一个mirror；如果已经有mirror了也可继续添加mirror，相当于修改了mirror）

首先，替换ocr的镜像文件：

[root@rac2 bin]# ./ocrconfig -replace ocrmirror /dev/raw/raw12
[root@rac2 bin]#

[oracle@rac2 fd]$ ocrcheck
Status of Oracle Cluster Registry is as follows :
     Version                  :          2
     Total space (kbytes)     :     524040
     Used space (kbytes)      :       3860
     Available space (kbytes) :     520180
     ID                       :  687142446
     Device/File Name         : /dev/raw/raw1
                                    Device/File integrity check succeeded
     Device/File Name         : /dev/raw/raw12                               -----这里ocr的镜像文件已经变为raw12了
                                    Device/File integrity check succeeded

     Cluster registry integrity check succeeded

[oracle@rac2 fd]$ 

接下来就是真正替换primary ocr文件了（修改OCR位置）：

[root@rac2 bin]# ./ocrconfig -replace ocr /dev/raw/raw11
[root@rac2 bin]#

[oracle@rac2 fd]$ ocrcheck
Status of Oracle Cluster Registry is as follows :
     Version                  :          2
     Total space (kbytes)     :     524040
     Used space (kbytes)      :       3860
     Available space (kbytes) :     520180
     ID                       :  687142446
     Device/File Name         : /dev/raw/raw11                    -------primary ocr文件已经替换了
                                    Device/File integrity check succeeded
     Device/File Name         : /dev/raw/raw12
                                    Device/File integrity check succeeded

Cluster registry integrity check succeeded

[oracle@rac2 fd]$

十六、每个节点重启crs并检查crs状态：

[root@rac2 bin]# ./crsctl check crs
CSS appears healthy
CRS appears healthy
EVM appears healthy
[root@rac2 bin]#

[oracle@rac1 ~]$ crs_stat -t -v
Name           Type           R/RA   F/FT   Target    State     Host        
----------------------------------------------------------------------
ora.rac.db     application    0/1    0/1    ONLINE    ONLINE    rac1        
ora....c1.inst application    0/5    0/0    ONLINE    ONLINE    rac1        
ora....c2.inst application    0/5    0/0    ONLINE    ONLINE    rac2        
ora....SM1.asm application    0/5    0/0    ONLINE    ONLINE    rac1        
ora....C1.lsnr application    0/5    0/0    ONLINE    ONLINE    rac1        
ora.rac1.gsd   application    0/5    0/0    ONLINE    ONLINE    rac1        
ora.rac1.ons   application    0/3    0/0    ONLINE    ONLINE    rac1        
ora.rac1.vip   application    0/0    0/0    ONLINE    ONLINE    rac1        
ora....SM2.asm application    0/5    0/0    ONLINE    ONLINE    rac2        
ora....C2.lsnr application    0/5    0/0    ONLINE    ONLINE    rac2        
ora.rac2.gsd   application    0/5    0/0    ONLINE    ONLINE    rac2        
ora.rac2.ons   application    0/3    0/0    ONLINE    ONLINE    rac2        
ora.rac2.vip   application    0/0    0/0    ONLINE    ONLINE    rac2        
[oracle@rac1 ~]$

十七、查看ocssd进程定位到的ocr文件：

[oracle@rac2 fd]$ ps -e | grep -i ocssd
22719 ?        00:00:00 ocssd.bin
[oracle@rac2 fd]$ 
[oracle@rac2 fd]$ cd /proc/22719/fd
[oracle@rac2 fd]$ 
[oracle@rac2 fd]$ ll
total 0
lr-x------ 1 oracle oinstall 64 Sep 26 15:51 0 -> /dev/null
lrwx------ 1 oracle oinstall 64 Sep 26 15:51 1 -> /u01/app/oracle/product/10.2.0/crs/log/rac2/cssd/cssdOUT.log
lrwx------ 1 oracle oinstall 64 Sep 26 15:51 10 -> socket:[111712]
lrwx------ 1 oracle oinstall 64 Sep 26 15:51 11 -> /dev/raw/raw13
lrwx------ 1 oracle oinstall 64 Sep 26 15:51 12 -> /dev/raw/raw13
lrwx------ 1 oracle oinstall 64 Sep 26 15:51 13 -> /dev/raw/raw14
lrwx------ 1 oracle oinstall 64 Sep 26 15:51 14 -> /dev/raw/raw14
lrwx------ 1 oracle oinstall 64 Sep 26 15:51 15 -> /dev/raw/raw15
lrwx------ 1 oracle oinstall 64 Sep 26 15:51 16 -> /dev/raw/raw15
l-wx------ 1 oracle oinstall 64 Sep 26 15:51 17 -> /u01/app/oracle/product/10.2.0/crs/log/rac2/alertrac2.log
lr-x------ 1 oracle oinstall 64 Sep 26 15:51 18 -> /u01/app/oracle/product/10.2.0/crs/has/mesg/clsdus.msb
lrwx------ 1 oracle oinstall 64 Sep 26 15:51 19 -> socket:[111840]
lrwx------ 1 oracle oinstall 64 Sep 26 15:51 2 -> /u01/app/oracle/product/10.2.0/crs/log/rac2/cssd/cssdOUT.log
lrwx------ 1 oracle oinstall 64 Sep 26 15:51 20 -> socket:[111843]
lrwx------ 1 oracle oinstall 64 Sep 26 15:51 21 -> socket:[111875]
lrwx------ 1 oracle oinstall 64 Sep 26 15:51 22 -> socket:[111845]
lrwx------ 1 oracle oinstall 64 Sep 26 15:51 23 -> socket:[111855]
lrwx------ 1 oracle oinstall 64 Sep 26 15:51 24 -> socket:[111857]
lrwx------ 1 oracle oinstall 64 Sep 26 15:51 25 -> socket:[111859]
lrwx------ 1 oracle oinstall 64 Sep 26 15:51 26 -> socket:[111862]
lrwx------ 1 oracle oinstall 64 Sep 26 15:51 27 -> socket:[111863]
lrwx------ 1 oracle oinstall 64 Sep 26 15:51 28 -> socket:[111876]
lrwx------ 1 oracle oinstall 64 Sep 26 15:51 29 -> socket:[111877]
lrwx------ 1 oracle oinstall 64 Sep 26 15:51 3 -> /u01/app/oracle/product/10.2.0/crs/log/rac2/cssd/cssdOUT.log
lrwx------ 1 oracle oinstall 64 Sep 26 15:51 30 -> socket:[112049]
lrwx------ 1 oracle oinstall 64 Sep 26 15:51 31 -> socket:[112050]
lrwx------ 1 oracle oinstall 64 Sep 26 15:51 33 -> socket:[112895]
lrwx------ 1 oracle oinstall 64 Sep 26 15:51 34 -> socket:[112269]
lrwx------ 1 oracle oinstall 64 Sep 26 15:51 35 -> socket:[112388]
lrwx------ 1 oracle oinstall 64 Sep 26 15:51 36 -> socket:[112407]
lrwx------ 1 oracle oinstall 64 Sep 26 15:51 37 -> socket:[112676]
lrwx------ 1 oracle oinstall 64 Sep 26 15:51 38 -> socket:[112458]
lrwx------ 1 oracle oinstall 64 Sep 26 15:51 39 -> socket:[112750]
lr-x------ 1 oracle oinstall 64 Sep 26 15:51 4 -> /u01/app/oracle/product/10.2.0/crs/srvm/mesg/procus.msb
lrwx------ 1 oracle oinstall 64 Sep 26 15:51 40 -> socket:[112597]
lrwx------ 1 oracle oinstall 64 Sep 26 15:51 41 -> socket:[113146]
lrwx------ 1 oracle oinstall 64 Sep 26 15:51 42 -> socket:[113322]
lrwx------ 1 oracle oinstall 64 Sep 26 15:51 43 -> socket:[113344]
lrwx------ 1 oracle oinstall 64 Sep 26 15:51 44 -> socket:[113385]
lrwx------ 1 oracle oinstall 64 Sep 26 15:51 45 -> socket:[113461]
lrwx------ 1 oracle oinstall 64 Sep 26 15:51 46 -> socket:[113532]
lrwx------ 1 oracle oinstall 64 Sep 26 15:51 47 -> socket:[113488]
lrwx------ 1 oracle oinstall 64 Sep 26 15:51 49 -> socket:[113595]
lr-x------ 1 oracle oinstall 64 Sep 26 15:51 5 -> /dev/raw/raw11     ----OCR文件已经替换完成
lrwx------ 1 oracle oinstall 64 Sep 26 15:51 50 -> socket:[113607]
lrwx------ 1 oracle oinstall 64 Sep 26 15:51 51 -> socket:[113619]
lrwx------ 1 oracle oinstall 64 Sep 26 15:51 52 -> socket:[113631]
lr-x------ 1 oracle oinstall 64 Sep 26 15:51 6 -> /dev/raw/raw12
lr-x------ 1 oracle oinstall 64 Sep 26 15:51 7 -> /u01/app/oracle/product/10.2.0/crs/css/mesg/clssus.msb
l-wx------ 1 oracle oinstall 64 Sep 26 15:51 8 -> /u01/app/oracle/product/10.2.0/crs/log/rac2/cssd/ocssd.log
lrwx------ 1 oracle oinstall 64 Sep 26 15:51 9 -> socket:[111711]
[oracle@rac2 fd]$

十八、从ASM中删除raw1、raw2、raw3、raw4、raw5（相当于删除老存储映射过来的磁盘），重启操作系统

[root@rac2 ~]# cat /etc/udev/rules.d/60-raw.rules 
# Enter raw device bindings here.
#
# An example would be:
#   ACTION=="add", KERNEL=="sda", RUN+="/bin/raw /dev/raw/raw1 %N"
# to bind /dev/raw/raw1 to /dev/sda, or
#   ACTION=="add", ENV{MAJOR}=="8", ENV{MINOR}=="1", RUN+="/bin/raw /dev/raw/raw2 %M %m"
# to bind /dev/raw/raw2 to the device with major 8, minor 1.

#ACTION=="add", KERNEL=="sdb",RUN+="/bin/raw /dev/raw/raw1 %N"   ----删除掉raw[1-5]块磁盘
#ACTION=="add", KERNEL=="sdc",RUN+="/bin/raw /dev/raw/raw2 %N"
#ACTION=="add", KERNEL=="sdd",RUN+="/bin/raw /dev/raw/raw3 %N"
#ACTION=="add", KERNEL=="sde",RUN+="/bin/raw /dev/raw/raw4 %N"
#ACTION=="add", KERNEL=="sdf",RUN+="/bin/raw /dev/raw/raw5 %N"
ACTION=="add", KERNEL=="sdg",RUN+="/bin/raw /dev/raw/raw6 %N"
ACTION=="add", KERNEL=="sdh",RUN+="/bin/raw /dev/raw/raw7 %N"
ACTION=="add", KERNEL=="sdi",RUN+="/bin/raw /dev/raw/raw8 %N"
ACTION=="add", KERNEL=="sdj",RUN+="/bin/raw /dev/raw/raw9 %N"
ACTION=="add", KERNEL=="sdk",RUN+="/bin/raw /dev/raw/raw10 %N"
ACTION=="add", KERNEL=="sdl",RUN+="/bin/raw /dev/raw/raw11 %N"
ACTION=="add", KERNEL=="sdm",RUN+="/bin/raw /dev/raw/raw12 %N"
ACTION=="add", KERNEL=="sdn",RUN+="/bin/raw /dev/raw/raw13 %N"
ACTION=="add", KERNEL=="sdo",RUN+="/bin/raw /dev/raw/raw14 %N"
ACTION=="add", KERNEL=="sdp",RUN+="/bin/raw /dev/raw/raw15 %N"
ACTION=="add", KERNEL=="sdq",RUN+="/bin/raw /dev/raw/raw16 %N"
ACTION=="add", KERNEL=="sdr",RUN+="/bin/raw /dev/raw/raw17 %N"
ACTION=="add", KERNEL=="sds",RUN+="/bin/raw /dev/raw/raw18 %N"
ACTION=="add", KERNEL=="sdt",RUN+="/bin/raw /dev/raw/raw19 %N"
ACTION=="add", KERNEL=="sdu",RUN+="/bin/raw /dev/raw/raw20 %N"
ACTION=="add", KERNEL=="sdv",RUN+="/bin/raw /dev/raw/raw21 %N"
ACTION=="add", KERNEL=="sdw",RUN+="/bin/raw /dev/raw/raw22 %N"
#ACTION=="add", KERNEL=="raw[1-2]", OWNER="root", GROUP="oinstall", MODE="775"    ----删除掉raw[1-5]块磁盘
#ACTION=="add", KERNEL=="raw3", OWNER="oracle", GROUP="oinstall", MODE="775"
#ACTION=="add", KERNEL=="raw4", OWNER="oracle", GROUP="oinstall", MODE="775"
#ACTION=="add", KERNEL=="raw5", OWNER="oracle", GROUP="oinstall", MODE="775"
ACTION=="add", KERNEL=="raw6", OWNER="oracle", GROUP="oinstall", MODE="775"
ACTION=="add", KERNEL=="raw7", OWNER="oracle", GROUP="oinstall", MODE="775"
ACTION=="add", KERNEL=="raw8", OWNER="oracle", GROUP="oinstall", MODE="775"
ACTION=="add", KERNEL=="raw9", OWNER="oracle", GROUP="oinstall", MODE="775"
ACTION=="add", KERNEL=="raw10", OWNER="oracle", GROUP="oinstall", MODE="775"
ACTION=="add", KERNEL=="raw11", OWNER="oracle", GROUP="oinstall", MODE="775"
ACTION=="add", KERNEL=="raw12", OWNER="oracle", GROUP="oinstall", MODE="775"
ACTION=="add", KERNEL=="raw13", OWNER="oracle", GROUP="oinstall", MODE="775"
ACTION=="add", KERNEL=="raw14", OWNER="oracle", GROUP="oinstall", MODE="775"
ACTION=="add", KERNEL=="raw15", OWNER="oracle", GROUP="oinstall", MODE="775"
ACTION=="add", KERNEL=="raw16", OWNER="oracle", GROUP="oinstall", MODE="775"
ACTION=="add", KERNEL=="raw17", OWNER="oracle", GROUP="oinstall", MODE="775"
ACTION=="add", KERNEL=="raw18", OWNER="oracle", GROUP="oinstall", MODE="775"
ACTION=="add", KERNEL=="raw19", OWNER="oracle", GROUP="oinstall", MODE="775"
ACTION=="add", KERNEL=="raw20", OWNER="oracle", GROUP="oinstall", MODE="775"
ACTION=="add", KERNEL=="raw21", OWNER="oracle", GROUP="oinstall", MODE="775"
ACTION=="add", KERNEL=="raw22", OWNER="oracle", GROUP="oinstall", MODE="775"

[root@rac2 ~]#

[root@rac2 ~]# cat /etc/rc.d/rc.local
#!/bin/sh
#
# This script will be executed *after* all the other init scripts.
# You can put your own initialization stuff in here if you don't
# want to do the full Sys V style init stuff.

touch /var/lock/subsys/local

#/bin/raw /dev/raw/raw1 /dev/mapper/sdb    ----删除掉raw[1-5]块磁盘
#/bin/raw /dev/raw/raw2 /dev/mapper/sdc
#/bin/raw /dev/raw/raw3 /dev/mapper/sdd
#/bin/raw /dev/raw/raw4 /dev/mapper/sde
#/bin/raw /dev/raw/raw5 /dev/mapper/sdf
/bin/raw /dev/raw/raw6 /dev/mapper/sdg
/bin/raw /dev/raw/raw7 /dev/mapper/sdh
/bin/raw /dev/raw/raw8 /dev/mapper/sdi
/bin/raw /dev/raw/raw9 /dev/mapper/sdj
/bin/raw /dev/raw/raw10 /dev/mapper/sdk
/bin/raw /dev/raw/raw11 /dev/mapper/sdl
/bin/raw /dev/raw/raw12 /dev/mapper/sdm
/bin/raw /dev/raw/raw13 /dev/mapper/sdn
/bin/raw /dev/raw/raw14 /dev/mapper/sdo
/bin/raw /dev/raw/raw15 /dev/mapper/sdp
/bin/raw /dev/raw/raw16 /dev/mapper/sdq
/bin/raw /dev/raw/raw17 /dev/mapper/sdr
/bin/raw /dev/raw/raw18 /dev/mapper/sds
/bin/raw /dev/raw/raw19 /dev/mapper/sdt
/bin/raw /dev/raw/raw20 /dev/mapper/sdu
/bin/raw /dev/raw/raw21 /dev/mapper/sdv
/bin/raw /dev/raw/raw22 /dev/mapper/sdw

[root@rac2 ~]#

[root@rac1 ~]# ls -lrt /dev/raw/raw*         ----可以看到raw[1-5]块磁盘已删除
crwxrwxr-x 1 oracle oinstall 162, 20 Sep 26 16:03 /dev/raw/raw20
crwxrwxr-x 1 oracle oinstall 162,  6 Sep 26 16:03 /dev/raw/raw6
crwxrwxr-x 1 oracle oinstall 162, 21 Sep 26 16:03 /dev/raw/raw21
crwxrwxr-x 1 oracle oinstall 162, 10 Sep 26 16:03 /dev/raw/raw10
crwxrwxr-x 1 oracle oinstall 162, 16 Sep 26 16:03 /dev/raw/raw16
crwxrwxr-x 1 oracle oinstall 162, 18 Sep 26 16:03 /dev/raw/raw18
crwxrwxr-x 1 oracle oinstall 162, 22 Sep 26 16:03 /dev/raw/raw22
crwxrwxr-x 1 oracle oinstall 162, 19 Sep 26 16:03 /dev/raw/raw19
crwxrwxr-x 1 oracle oinstall 162, 17 Sep 26 16:03 /dev/raw/raw17
crwxrwxr-x 1 oracle oinstall 162, 12 Sep 26 16:04 /dev/raw/raw12
crwxrwxr-x 1 oracle oinstall 162, 11 Sep 26 16:04 /dev/raw/raw11
crwxrwxr-x 1 oracle oinstall 162, 15 Sep 26 16:06 /dev/raw/raw15
crwxrwxr-x 1 oracle oinstall 162, 14 Sep 26 16:06 /dev/raw/raw14
crwxrwxr-x 1 oracle oinstall 162, 13 Sep 26 16:06 /dev/raw/raw13
crwxrwxr-x 1 oracle oinstall 162,  9 Sep 26 16:06 /dev/raw/raw9
crwxrwxr-x 1 oracle oinstall 162,  8 Sep 26 16:06 /dev/raw/raw8
crwxrwxr-x 1 oracle oinstall 162,  7 Sep 26 16:06 /dev/raw/raw7
[root@rac1 ~]#

重启操作系统

# reboot

十九、重启操作系统后，检查crs状态并查看ASM磁盘信息：

[oracle@rac1 ~]$ crs_stat -t -v
Name           Type           R/RA   F/FT   Target    State     Host        
----------------------------------------------------------------------
ora.rac.db     application    0/1    0/1    ONLINE    ONLINE    rac1        
ora....c1.inst application    0/5    0/0    ONLINE    ONLINE    rac1        
ora....c2.inst application    0/5    0/0    ONLINE    ONLINE    rac2        
ora....SM1.asm application    0/5    0/0    ONLINE    ONLINE    rac1        
ora....C1.lsnr application    0/5    0/0    ONLINE    ONLINE    rac1        
ora.rac1.gsd   application    0/5    0/0    ONLINE    ONLINE    rac1        
ora.rac1.ons   application    0/3    0/0    ONLINE    ONLINE    rac1        
ora.rac1.vip   application    0/0    0/0    ONLINE    ONLINE    rac1        
ora....SM2.asm application    0/5    0/0    ONLINE    ONLINE    rac2        
ora....C2.lsnr application    0/5    0/0    ONLINE    ONLINE    rac2        
ora.rac2.gsd   application    0/5    0/0    ONLINE    ONLINE    rac2        
ora.rac2.ons   application    0/3    0/0    ONLINE    ONLINE    rac2        
ora.rac2.vip   application    0/0    0/0    ONLINE    ONLINE    rac2        
[oracle@rac1 ~]$

[root@rac2 ~]# cd /u01/app/oracle/product/10.2.0/crs/bin/
[root@rac2 bin]# ./crsctl check crs
CSS appears healthy
CRS appears healthy
EVM appears healthy
[root@rac2 bin]#

[root@rac2 ~]# su - oracle
[oracle@rac2 ~]$ export ORACLE_SID=+ASM2
[oracle@rac2 ~]$ sqlplus / as sysdba

SQL*Plus: Release 10.2.0.5.0 - Production on Wed Sep 26 16:12:19 2018

Copyright (c) 1982, 2010, Oracle.  All Rights Reserved.

Connected to:
Oracle Database 10g Enterprise Edition Release 10.2.0.5.0 - 64bit Production
With the Partitioning, Real Application Clusters, OLAP, Data Mining
and Real Application Testing options

SQL> desc v$asm_disk
 Name                       Null?    Type
 ----------------------------------------- -------- ----------------------------
 GROUP_NUMBER                        NUMBER
 DISK_NUMBER                        NUMBER
 COMPOUND_INDEX                     NUMBER
 INCARNATION                        NUMBER
 MOUNT_STATUS                        VARCHAR2(7)
 HEADER_STATUS                        VARCHAR2(12)
 MODE_STATUS                        VARCHAR2(7)
 STATE                            VARCHAR2(8)
 REDUNDANCY                        VARCHAR2(7)
 LIBRARY                        VARCHAR2(64)
 TOTAL_MB                        NUMBER
 FREE_MB                        NUMBER
 NAME                            VARCHAR2(30)
 FAILGROUP                        VARCHAR2(30)
 LABEL                            VARCHAR2(31)
 PATH                            VARCHAR2(256)
 UDID                            VARCHAR2(64)
 PRODUCT                        VARCHAR2(32)
 CREATE_DATE                        DATE
 MOUNT_DATE                        DATE
 REPAIR_TIMER                        NUMBER
 READS                            NUMBER
 WRITES                         NUMBER
 READ_ERRS                        NUMBER
 WRITE_ERRS                        NUMBER
 READ_TIME                        NUMBER
 WRITE_TIME                        NUMBER
 BYTES_READ                        NUMBER
 BYTES_WRITTEN                        NUMBER

SQL> set linesize 200
SQL> col path format a20
SQL> col name format a20
SQL> select PATH,GROUP_NUMBER,DISK_NUMBER,MOUNT_STATUS,HEADER_STATUS,MODE_STATUS,STATE,name,TOTAL_MB,FREE_MB from v$asm_disk;

PATH             GROUP_NUMBER DISK_NUMBER MOUNT_S HEADER_STATU MODE_ST STATE    NAME           TOTAL_MB    FREE_MB
-------------------- ------------ ----------- ------- ------------ ------- -------- -------------------- ---------- ----------
/dev/raw/raw17            0        0 CLOSED  CANDIDATE    ONLINE  NORMAL                512         0
/dev/raw/raw13            0        1 CLOSED  FOREIGN       ONLINE  NORMAL                512         0
/dev/raw/raw6            0       16 CLOSED  CANDIDATE    ONLINE  NORMAL                512         0
/dev/raw/raw12            0       12 CLOSED  FOREIGN       ONLINE  NORMAL                512         0
/dev/raw/raw22            0        2 CLOSED  CANDIDATE    ONLINE  NORMAL                   3072         0
/dev/raw/raw20            0        3 CLOSED  CANDIDATE    ONLINE  NORMAL                   3072         0
/dev/raw/raw19            0        4 CLOSED  CANDIDATE    ONLINE  NORMAL                   3072         0
/dev/raw/raw18            0        5 CLOSED  CANDIDATE    ONLINE  NORMAL                   3072         0
/dev/raw/raw21            0        6 CLOSED  CANDIDATE    ONLINE  NORMAL                   3072         0
/dev/raw/raw16            0        7 CLOSED  CANDIDATE    ONLINE  NORMAL                512         0
/dev/raw/raw15            0        8 CLOSED  FOREIGN       ONLINE  NORMAL                512         0

PATH             GROUP_NUMBER DISK_NUMBER MOUNT_S HEADER_STATU MODE_ST STATE    NAME           TOTAL_MB    FREE_MB
-------------------- ------------ ----------- ------- ------------ ------- -------- -------------------- ---------- ----------
/dev/raw/raw10            0        9 CLOSED  CANDIDATE    ONLINE  NORMAL                512         0
/dev/raw/raw14            0       10 CLOSED  FOREIGN       ONLINE  NORMAL                512         0
/dev/raw/raw11            0       11 CLOSED  FOREIGN       ONLINE  NORMAL                512         0
/dev/raw/raw9            1        2 CACHED  MEMBER       ONLINE  NORMAL   DATA_0002               3072      1367
/dev/raw/raw8            1        1 CACHED  MEMBER       ONLINE  NORMAL   DATA_0001               3072      1370
/dev/raw/raw7            1        0 CACHED  MEMBER       ONLINE  NORMAL   DATA_0000               3072      1363

17 rows selected.

SQL>

可以看到ASM磁盘里面已经没有raw[1-5]块磁盘了。

二十、替换ASM中的DATA数据磁盘：

删除计划剔除的磁盘：DATA_0002（/dev/raw/raw9）

SQL> alter diskgroup data drop disk 'DATA_0002' rebalance power 7 wait;

Diskgroup altered.

SQL>

等待同步完成：

[oracle@rac2 ~]$ export ORACLE_SID=+ASM2
[oracle@rac2 ~]$ sqlplus / as sysdba

SQL*Plus: Release 10.2.0.5.0 - Production on Wed Sep 26 16:22:55 2018

Copyright (c) 1982, 2010, Oracle.  All Rights Reserved.

Connected to:
Oracle Database 10g Enterprise Edition Release 10.2.0.5.0 - 64bit Production
With the Partitioning, Real Application Clusters, OLAP, Data Mining
and Real Application Testing options

SQL> select * from v$asm_operation;

GROUP_NUMBER OPERA STAT      POWER     ACTUAL       SOFAR   EST_WORK   EST_RATE EST_MINUTES
------------ ----- ---- ---------- ---------- ---------- ---------- ---------- -----------
       1 REBAL RUN         7        7         418       2271      1712         1

SQL>

SQL> select disk_number, state, name, total_mb, free_mb from v$asm_disk where group_number = 1;

DISK_NUMBER STATE    NAME                  TOTAL_MB      FREE_MB
----------- -------- ------------------------------ ---------- ----------
      2 DROPPING DATA_0002                  3072         2316
      1 NORMAL   DATA_0001                  3072          952
      0 NORMAL   DATA_0000                  3072          947

SQL>

SQL> col path format a20
SQL> select path,group_number,disk_number,mount_status,header_status,mode_status,state from v$asm_disk;

PATH             GROUP_NUMBER DISK_NUMBER MOUNT_S HEADER_STATU MODE_ST STATE
-------------------- ------------ ----------- ------- ------------ ------- --------
/dev/raw/raw17            0        0 CLOSED  CANDIDATE    ONLINE  NORMAL
/dev/raw/raw13            0        1 CLOSED  FOREIGN       ONLINE  NORMAL
/dev/raw/raw6            0       16 CLOSED  CANDIDATE    ONLINE  NORMAL
/dev/raw/raw22            0        2 CLOSED  CANDIDATE    ONLINE  NORMAL
/dev/raw/raw20            0        3 CLOSED  CANDIDATE    ONLINE  NORMAL
/dev/raw/raw19            0        4 CLOSED  CANDIDATE    ONLINE  NORMAL
/dev/raw/raw18            0        5 CLOSED  CANDIDATE    ONLINE  NORMAL
/dev/raw/raw21            0        6 CLOSED  CANDIDATE    ONLINE  NORMAL
/dev/raw/raw16            0        7 CLOSED  CANDIDATE    ONLINE  NORMAL
/dev/raw/raw15            0        8 CLOSED  FOREIGN       ONLINE  NORMAL
/dev/raw/raw10            0        9 CLOSED  CANDIDATE    ONLINE  NORMAL

PATH             GROUP_NUMBER DISK_NUMBER MOUNT_S HEADER_STATU MODE_ST STATE
-------------------- ------------ ----------- ------- ------------ ------- --------
/dev/raw/raw14            0       10 CLOSED  FOREIGN       ONLINE  NORMAL
/dev/raw/raw11            0       11 CLOSED  FOREIGN       ONLINE  NORMAL
/dev/raw/raw12            0       12 CLOSED  FOREIGN       ONLINE  NORMAL
/dev/raw/raw9            0       13 CLOSED  FORMER       ONLINE  NORMAL    ----已删除
/dev/raw/raw8            1        1 CACHED  MEMBER       ONLINE  NORMAL
/dev/raw/raw7            1        0 CACHED  MEMBER       ONLINE  NORMAL

17 rows selected.

SQL>

查看告警日志，里面也记录了删除磁盘组中的磁盘的操作：

[root@rac2 ~]# cd /u01/app/oracle/admin/rac/bdump/
[root@rac2 bdump]# tail -n 10 alert_rac2.log 
Wed Sep 26 16:04:37 CST 2018
Database Characterset is ZHS16GBK
Opening with internal Resource Manager plan 
replication_dependency_tracking turned off (no async multimaster replication found)
Starting background process QMNC
QMNC started with pid=28, OS id=7265
Wed Sep 26 16:04:40 CST 2018
Completed: ALTER DATABASE OPEN
Wed Sep 26 16:24:10 CST 2018
SUCCESS: disk DATA_0002 (2.3915955210) dropped from diskgroup DATA
[root@rac2 bdump]#

将规划好的新存储的磁盘加入到ASM DATA磁盘组中：

SQL> alter diskgroup DATA add disk '/dev/raw/raw19' rebalance power 7;

Diskgroup altered.

SQL>

等待同步完成：

SQL> select * from v$asm_operation;

GROUP_NUMBER OPERA STAT      POWER     ACTUAL       SOFAR   EST_WORK   EST_RATE EST_MINUTES
------------ ----- ---- ---------- ---------- ---------- ---------- ---------- -----------
       1 REBAL RUN         7        7          14       1762       588         2

SQL>

SQL> select path,group_number,disk_number,mount_status,header_status,mode_status,state from v$asm_disk;

PATH             GROUP_NUMBER DISK_NUMBER MOUNT_S HEADER_STATU MODE_ST STATE
-------------------- ------------ ----------- ------- ------------ ------- --------
/dev/raw/raw17            0        0 CLOSED  CANDIDATE    ONLINE  NORMAL
/dev/raw/raw13            0        1 CLOSED  FOREIGN       ONLINE  NORMAL
/dev/raw/raw6            0       16 CLOSED  CANDIDATE    ONLINE  NORMAL
/dev/raw/raw9            0       13 CLOSED  FORMER       ONLINE  NORMAL    ---raw9已经从ASM的DATA磁盘中干净的删除
/dev/raw/raw22            0        2 CLOSED  CANDIDATE    ONLINE  NORMAL
/dev/raw/raw20            0        3 CLOSED  CANDIDATE    ONLINE  NORMAL
/dev/raw/raw18            0        5 CLOSED  CANDIDATE    ONLINE  NORMAL
/dev/raw/raw21            0        6 CLOSED  CANDIDATE    ONLINE  NORMAL
/dev/raw/raw16            0        7 CLOSED  CANDIDATE    ONLINE  NORMAL
/dev/raw/raw15            0        8 CLOSED  FOREIGN       ONLINE  NORMAL
/dev/raw/raw10            0        9 CLOSED  CANDIDATE    ONLINE  NORMAL

PATH             GROUP_NUMBER DISK_NUMBER MOUNT_S HEADER_STATU MODE_ST STATE
-------------------- ------------ ----------- ------- ------------ ------- --------
/dev/raw/raw14            0       10 CLOSED  FOREIGN       ONLINE  NORMAL
/dev/raw/raw11            0       11 CLOSED  FOREIGN       ONLINE  NORMAL
/dev/raw/raw12            0       12 CLOSED  FOREIGN       ONLINE  NORMAL
/dev/raw/raw19            1        2 CACHED  MEMBER       ONLINE  NORMAL     ----raw19已经变为DATA磁盘组里的成员盘了
/dev/raw/raw8            1        1 CACHED  MEMBER       ONLINE  NORMAL
/dev/raw/raw7            1        0 CACHED  MEMBER       ONLINE  NORMAL

17 rows selected.

SQL>

同样告警日志里面也记录这样一条信息：

[root@rac2 bdump]# tail -n 10 alert_rac2.log 
Opening with internal Resource Manager plan 
replication_dependency_tracking turned off (no async multimaster replication found)
Starting background process QMNC
QMNC started with pid=28, OS id=7265
Wed Sep 26 16:04:40 CST 2018
Completed: ALTER DATABASE OPEN
Wed Sep 26 16:24:10 CST 2018
SUCCESS: disk DATA_0002 (2.3915955210) dropped from diskgroup DATA
Wed Sep 26 16:30:40 CST 2018
SUCCESS: disk DATA_0002 (2.3915955213) added to diskgroup DATA
[root@rac2 bdump]#

二十一、现在替换第二块数据磁盘：

将规划好的磁盘（/dev/raw/raw18）添加到DATA磁盘组中，替换掉旧（/dev/raw/raw8）磁盘：

SQL> alter diskgroup DATA add disk '/dev/raw/raw18' rebalance power 7;

Diskgroup altered.

SQL>

等待同步完成：

SQL> select * from v$asm_operation;

GROUP_NUMBER OPERA STAT      POWER     ACTUAL       SOFAR   EST_WORK   EST_RATE EST_MINUTES
------------ ----- ---- ---------- ---------- ---------- ---------- ---------- -----------
       1 REBAL RUN         7        7           7       1615       840         1

SQL> select path,group_number,disk_number,mount_status,header_status,mode_status,state from v$asm_disk;

PATH             GROUP_NUMBER DISK_NUMBER MOUNT_S HEADER_STATU MODE_ST STATE
-------------------- ------------ ----------- ------- ------------ ------- --------
/dev/raw/raw17            0        0 CLOSED  CANDIDATE    ONLINE  NORMAL
/dev/raw/raw13            0        1 CLOSED  FOREIGN       ONLINE  NORMAL
/dev/raw/raw6            0       16 CLOSED  CANDIDATE    ONLINE  NORMAL
/dev/raw/raw9            0       13 CLOSED  FORMER       ONLINE  NORMAL
/dev/raw/raw12            0       12 CLOSED  FOREIGN       ONLINE  NORMAL
/dev/raw/raw22            0        2 CLOSED  CANDIDATE    ONLINE  NORMAL
/dev/raw/raw20            0        3 CLOSED  CANDIDATE    ONLINE  NORMAL
/dev/raw/raw21            0        6 CLOSED  CANDIDATE    ONLINE  NORMAL
/dev/raw/raw16            0        7 CLOSED  CANDIDATE    ONLINE  NORMAL
/dev/raw/raw15            0        8 CLOSED  FOREIGN       ONLINE  NORMAL
/dev/raw/raw10            0        9 CLOSED  CANDIDATE    ONLINE  NORMAL

PATH             GROUP_NUMBER DISK_NUMBER MOUNT_S HEADER_STATU MODE_ST STATE
-------------------- ------------ ----------- ------- ------------ ------- --------
/dev/raw/raw14            0       10 CLOSED  FOREIGN       ONLINE  NORMAL
/dev/raw/raw11            0       11 CLOSED  FOREIGN       ONLINE  NORMAL
/dev/raw/raw19            1        2 CACHED  MEMBER       ONLINE  NORMAL
/dev/raw/raw18            1        3 CACHED  MEMBER       ONLINE  NORMAL
/dev/raw/raw8            1        1 CACHED  MEMBER       ONLINE  NORMAL
/dev/raw/raw7            1        0 CACHED  MEMBER       ONLINE  NORMAL

17 rows selected.

SQL>

现在来剔除/dev/raw/raw8磁盘：

SQL>  alter diskgroup data drop disk 'DATA_0001' rebalance power 7 wait;

Diskgroup altered.

SQL>

等待同步完成：

SQL> select * from v$asm_operation;

GROUP_NUMBER OPERA STAT      POWER     ACTUAL       SOFAR   EST_WORK   EST_RATE EST_MINUTES
------------ ----- ---- ---------- ---------- ---------- ---------- ---------- -----------
       1 REBAL RUN         7        7          32       2105       798         2

SQL> SQL> select path,group_number,disk_number,mount_status,header_status,mode_status,state from v$asm_disk;

PATH             GROUP_NUMBER DISK_NUMBER MOUNT_S HEADER_STATU MODE_ST STATE
-------------------- ------------ ----------- ------- ------------ ------- --------
/dev/raw/raw17            0        0 CLOSED  CANDIDATE    ONLINE  NORMAL
/dev/raw/raw13            0        1 CLOSED  FOREIGN       ONLINE  NORMAL
/dev/raw/raw6            0       16 CLOSED  CANDIDATE    ONLINE  NORMAL
/dev/raw/raw9            0       13 CLOSED  FORMER       ONLINE  NORMAL
/dev/raw/raw12            0       12 CLOSED  FOREIGN       ONLINE  NORMAL
/dev/raw/raw22            0        2 CLOSED  CANDIDATE    ONLINE  NORMAL
/dev/raw/raw20            0        3 CLOSED  CANDIDATE    ONLINE  NORMAL
/dev/raw/raw21            0        6 CLOSED  CANDIDATE    ONLINE  NORMAL
/dev/raw/raw16            0        7 CLOSED  CANDIDATE    ONLINE  NORMAL
/dev/raw/raw15            0        8 CLOSED  FOREIGN       ONLINE  NORMAL
/dev/raw/raw10            0        9 CLOSED  CANDIDATE    ONLINE  NORMAL

PATH             GROUP_NUMBER DISK_NUMBER MOUNT_S HEADER_STATU MODE_ST STATE
-------------------- ------------ ----------- ------- ------------ ------- --------
/dev/raw/raw14            0       10 CLOSED  FOREIGN       ONLINE  NORMAL
/dev/raw/raw11            0       11 CLOSED  FOREIGN       ONLINE  NORMAL
/dev/raw/raw19            1        2 CACHED  MEMBER       ONLINE  NORMAL
/dev/raw/raw18            1        3 CACHED  MEMBER       ONLINE  NORMAL
/dev/raw/raw8            1        1 CACHED  MEMBER       ONLINE  DROPPING
/dev/raw/raw7            1        0 CACHED  MEMBER       ONLINE  NORMAL

17 rows selected.

SQL> select path,group_number,disk_number,mount_status,header_status,mode_status,state from v$asm_disk;

PATH             GROUP_NUMBER DISK_NUMBER MOUNT_S HEADER_STATU MODE_ST STATE
-------------------- ------------ ----------- ------- ------------ ------- --------
/dev/raw/raw17            0        0 CLOSED  CANDIDATE    ONLINE  NORMAL
/dev/raw/raw13            0        1 CLOSED  FOREIGN       ONLINE  NORMAL
/dev/raw/raw6            0       16 CLOSED  CANDIDATE    ONLINE  NORMAL
/dev/raw/raw9            0       13 CLOSED  FORMER       ONLINE  NORMAL
/dev/raw/raw22            0        2 CLOSED  CANDIDATE    ONLINE  NORMAL
/dev/raw/raw20            0        3 CLOSED  CANDIDATE    ONLINE  NORMAL
/dev/raw/raw8            0        4 CLOSED  FORMER       ONLINE  NORMAL
/dev/raw/raw21            0        6 CLOSED  CANDIDATE    ONLINE  NORMAL
/dev/raw/raw16            0        7 CLOSED  CANDIDATE    ONLINE  NORMAL
/dev/raw/raw15            0        8 CLOSED  FOREIGN       ONLINE  NORMAL
/dev/raw/raw10            0        9 CLOSED  CANDIDATE    ONLINE  NORMAL

PATH             GROUP_NUMBER DISK_NUMBER MOUNT_S HEADER_STATU MODE_ST STATE
-------------------- ------------ ----------- ------- ------------ ------- --------
/dev/raw/raw14            0       10 CLOSED  FOREIGN       ONLINE  NORMAL
/dev/raw/raw11            0       11 CLOSED  FOREIGN       ONLINE  NORMAL
/dev/raw/raw12            0       12 CLOSED  FOREIGN       ONLINE  NORMAL
/dev/raw/raw19            1        2 CACHED  MEMBER       ONLINE  NORMAL
/dev/raw/raw18            1        3 CACHED  MEMBER       ONLINE  NORMAL
/dev/raw/raw7            1        0 CACHED  MEMBER       ONLINE  NORMAL

17 rows selected.

SQL>

[root@rac2 bdump]# tail -f alert_rac2.log 
Starting background process QMNC
QMNC started with pid=28, OS id=7265
Wed Sep 26 16:04:40 CST 2018
Completed: ALTER DATABASE OPEN
Wed Sep 26 16:24:10 CST 2018
SUCCESS: disk DATA_0002 (2.3915955210) dropped from diskgroup DATA
Wed Sep 26 16:30:40 CST 2018
SUCCESS: disk DATA_0002 (2.3915955213) added to diskgroup DATA
Wed Sep 26 16:38:14 CST 2018
SUCCESS: disk DATA_0003 (3.3915955214) added to diskgroup DATA
Wed Sep 26 16:41:53 CST 2018
SUCCESS: disk DATA_0001 (1.3915955211) dropped from diskgroup DATA

以此类推，直至完全替换完毕即可。

Oracle 10g rac + ASM数据迁移（替换ASM中的磁盘，用新存储替换旧存储）相关推荐

1. oracle 10g RAC app VIP ,ONS,GSD,ASM,OCR,VOTE名称及代表的涵义。

   To verify the existence of node applications, virtual IP (VIP) Oracle Notification Services (ONS) Gl ...

2. Redhat 5.4 Oracle 10g RAC Openfiler+Multipath + RAW+ ASM 安装文档

   在Redhat 平台上RAC 装过多次. 本来也没打算整理这篇文章. 但是在搭建这个环境的过程中有了一下新的体会. 而且RAC 架构也不一样. 搭建之后对RAC 也有更深的理解. 实验平台配置信息: ...

3. 基于Linux (RHEL 5.5) 安装Oracle 10g RAC

   本文所描述的是在Red Hat 5.5下使用vmware server 来安装Oracle 10g RAC(OCFS + ASM),本文假定你的RHEL5已经安装完毕,下面列出的是对Linux的具体配 ...

4. 基于Linux RHEL 5 5 安装Oracle 10g RAC

   分享一下我老师大神的人工智能教程!零基础,通俗易懂!http://blog.csdn.net/jiangjunshow 也欢迎大家转载本篇文章.分享知识,造福人民,实现我们中华民族伟大复兴! 本文所描 ...

5. GoldenGate配置实例：RHEL 4.7下的Oracle 10g RAC到单实例的单向同步(一)

   环境 源端是一个RAC rac1 rac2: RHEL 4u7 + ORACLE 10.2.0.4 + ASM rac1 外网IP 192.168.47.241 虚拟IP 192.168.47.243 ...

6. Vmware+RHEL5+Oracle 10G RAC安装实验--详细过程（一）

   实验环境: 普通PC一台(3GB内存,500GB硬盘) Vmware Server 1.0.6版 RedHat Linux Enterprise 5.1 Oracle  10g for linux(1 ...

7. oracle 云服务器cpu升级,Oracle 10g RAC 升级（CPU Update）之--升级CRS

   Oracle 10g RAC 升级(CPU Update)之--升级CRS 发布时间:2020-07-16 09:13:19 来源:51CTO 阅读:593 作者:客居天涯 Oracle 10g RA ...

8. oracle rac添加监听,【学习笔记】Oracle 10G RAC增加节点时手动注册监听服务的案例步骤...

   [学习笔记]Oracle 10G RAC增加节点时手动注册监听服务的案例步骤 时间:2016-10-22 22:53   来源:Oracle研究中心   作者:HTZ   点击: 次 天萃荷净 Ora ...

9. Oracle 10g RAC 升级（CPU Update）之--升级CRS

   Oracle 10g RAC 升级(CPU Update)之--升级CRS 系统环境: 操作系统:RedHat EL5 Cluster: Oracle CRS 10.2.0.1.0 Oracle:   ...

最新文章

1. python基础教程第三版电子版百度云-《python基础教程第三版》高清版PDF免费下载...
2. 网络yum源 自定义yum仓库 zip备份 编译安装
3. （2）双机调试+符号文件
4. RabbitMQ安装方法 安装完成已验证方法步骤可行性
5. 【数据结构与算法】之深入解析“二叉搜索树中的插入操作”的求解思路与算法示例
6. JAXB –新手的观点，第2部分
7. Spring4：没有默认构造函数的基于CGLIB的代理类
8. json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0)
9. oracle插入性能优化,Oracle- insert性能优化
10. 阿里云何川：开放兼容的云，计算巢帮助合作伙伴云化升级
11. 使用Advanced Installer将.exe程序重新封装为.msi程序
12. linux dns区域传送,Linux DNS服务器主从复制及区域传送
13. Springboot Bean Validation校验框架
14. Android 获取手机通讯录信息 — 头像、姓名和A-Z的快速查询
15. 如何处理Oracle中TEMP表空间满的问题（转载）
16. C#中手动引用COM组建的例子
17. 硅谷产品联盟合伙人：每一个伟大产品的背后
18. 三维von Mises-Fisher分布的均值方差
19. 老板，上班迷路了，晚点到~
20. 计算机考试机试题目word文档,计算机考试 word

热门文章

1. Bootstrap静止在页面底部的导航栏
2. python 数据分析 实际案例-Python数据分析案例实战
3. Java后端学习路线分享
4. 企业域名邮箱是什么？企业邮箱地址在哪里找？
5. ubuntu装R和rstudio
6. 字符串,对象,list判断是否为空
7. npm下载Vue-cli问题
8. Docker的搭建一键重启
9. 小程序流量主 公众号流量主
10. jquery 中substring，substr，split的用法