228 lines
5.8 KiB
Markdown
228 lines
5.8 KiB
Markdown
### 模块
|
||
OSD(rados): 核心,用于集群中数据与对象的存储,处理对象的复制,恢复,再均衡等
|
||
MON: 监控集群状态(奇数个), 维护集群clustermap,保证数据一致性
|
||
RGW(radosgw): 对象存储需要
|
||
MDS: 文件系统存储需要
|
||
|
||
MGR: 监控
|
||
|
||
### 配置文件
|
||
/etc/ceph.conf ## mon及密码等, 客户端连接密码可以不配置
|
||
|
||
/etc/ceph/ceph.client.admin.keyring ## 客户端密码文件
|
||
|
||
* 时间需要同步
|
||
|
||
### 安装ceph-deploy
|
||
```shell
|
||
# 需要在生成免密登录服务器上执行
|
||
yum -y install ceph-deploy
|
||
# 帮助
|
||
ceph-deploy mon --help
|
||
```
|
||
### 创建节点生成配置文件
|
||
```shell
|
||
ceph-deploy new node1 node2 node3
|
||
```
|
||
### 安装ceph
|
||
```shell
|
||
yum -y install ceph-mon ceph-osd ceph-mds ceph-radosgw
|
||
# ceph-deploy也可以安装
|
||
ceph-deploy install node1 node2 node3
|
||
```
|
||
|
||
### 初始化节点（配置文件传输到节点，启动mon）
|
||
```shell
|
||
ceph-deploy mon create-initial
|
||
# 或以下
|
||
ceph-deploy mon add node1
|
||
ceph-deploy mon add node2
|
||
ceph-deploy mon add node3
|
||
```
|
||
### 查状态
|
||
```shell
|
||
ceph -s
|
||
```
|
||
### 创建osd
|
||
```shell
|
||
# 需要缓存盘用于缓存写入数据
|
||
# 若全部使用，可以使用ceph-deploy做，若有不同分区，需自行分区
|
||
parted /dev/vdb mklabel gpt
|
||
parted /dev/vdb mkpart primary 1 50%
|
||
parted /dev/vdb mkpart 50% 100%
|
||
# 修改所属用户与组，使ceph有权限写入
|
||
chown ceph:ceph /dev/vdb1
|
||
chown ceph:ceph /dev/vdb2
|
||
# 以上操作只能保证当前有效，重启会恢复。以下操作可以永久生效
|
||
cat >> /etc/udev/rules.d/70-vdb.rules <<EOF
|
||
ENV{DEVNAME}=="dev/vdb1",OWNER="ceph",GROUP="ceph"
|
||
ENV{DEVNAME}=="dev/vdb2",OWNER="ceph",GROUP="ceph"
|
||
EOF
|
||
# 格式化数据盘
|
||
ceph-deploy disk zap node1:/dev/vdc node1:/dev/vdd
|
||
ceph-deploy disk zap node2:/dev/vdc node2:/dev/vdd
|
||
ceph-deploy disk zap node3:/dev/vdc node3:/dev/vdd
|
||
|
||
# 创建osd vdc，缓存为/dev/vdb1, osd vdd,缓存/dev/vdb2,并启动服务
|
||
ceph-deploy osd create node1:vdc:/dev/vdb1 node1:vdd:/dev/vdb2
|
||
ceph-deploy osd create node2:vdc:/dev/vdb1 node2:vdd:/dev/vdb2
|
||
ceph-deploy osd create node3:vdc:/dev/vdb1 node3:vdd:/dev/vdb2
|
||
|
||
# 查看状态
|
||
ceph -s
|
||
# 查看osd信息
|
||
ceph osd tree
|
||
```
|
||
## 存储池
|
||
```shell
|
||
# 默认已经创建0 rdb
|
||
ceph osd lspools
|
||
```
|
||
## 镜像
|
||
```shell
|
||
# layering: 开启cow功能，写时复制(可以做增量快照)
|
||
rdb create <image-name> --image-feature layering --size 10g
|
||
rdb create <pool-name>/<image-name> --image-feature layering --size 10g
|
||
|
||
# 查看
|
||
rbd list
|
||
rbd info <image-name>
|
||
# 扩缩
|
||
# --allow-shrink: 缩减需要添加
|
||
rbd resize --rize 7G <image-name> --allow-shrink
|
||
```
|
||
|
||
## 客户端访问
|
||
```shell
|
||
# 客户端安装
|
||
yum install -y ceph-common
|
||
# 配置文件及密码
|
||
# 从集群中复制相应文件
|
||
/etc/ceph/ceph.conf
|
||
/etc/ceph/ceph.client.admin.keyring
|
||
|
||
# 添加集群
|
||
rbd map <image-name>
|
||
# 查看本地磁盘
|
||
# 本地多了rbd*,可以正常格式化挂载操作
|
||
lsblk
|
||
|
||
# 查看image使用情况
|
||
rbd showmapped
|
||
# 查看image
|
||
rbd image list
|
||
```
|
||
## 快照
|
||
ceph多副本,保证osd异常时,数据不丢失
|
||
快照防止误操作
|
||
```shell
|
||
# 查看image快照
|
||
rbd snap ls <image-name>
|
||
# 创建快照
|
||
rdb snap creat image --snap <snap-image>
|
||
# 操作数据,如删除
|
||
# 卸载挂载目录(挂载地方)
|
||
umount /mnt
|
||
# 还原数据(恢复快照)
|
||
rbd snap rollback <image> --snap <snap-name>
|
||
# 重新挂载目录即可恢复数据
|
||
```
|
||
## 克隆
|
||
```shell
|
||
# 使用快照克隆比较快
|
||
# protect: 保护snap,防止被删
|
||
rbd snap protect <image-name> --snap <snap-name>
|
||
# 如下命令会删除失败
|
||
rbd snap rm <imag-name> --snap <snap-name>
|
||
# 克隆image
|
||
rbd clone <image-name> --snap <snap-name> image-clone --image-feature layering
|
||
# 查看父子image
|
||
rbd info <image-name-clone>
|
||
# 去掉父子关系, 使复制后的image独立工作,需要时间
|
||
rbd flatten <image-name-clone>
|
||
# 快照删除
|
||
rbd snap unprotect <image-name> --snap <snap-name>
|
||
rbd snap rm <imag-name> --snap <snap-name>
|
||
```
|
||
## 本地去掉盘
|
||
```shell
|
||
umount /mnt
|
||
rbd unmap <image-name>
|
||
```
|
||
## 虚拟机中使用ceph
|
||
```shell
|
||
# ceph需要认证
|
||
# 安装虚拟机的服务器中,需要配置如下文件
|
||
cat > secret.xml <<EOF
|
||
<secret ephemeral='no' private='no'>
|
||
<usage type='ceph'>
|
||
<name>client-admin secret</name>
|
||
</usage>
|
||
</secret>
|
||
EOF
|
||
# 创建
|
||
virsh secret-define secret.xml
|
||
virsh secret-list
|
||
# 绑定密码
|
||
# secret-uuid为以上命令查询所得
|
||
# /etc/ceph/ceph.client.admin.keyring中key后的密码
|
||
virsh secret-set-value --secret <secret-uuid> --base64 <client.admin.key>
|
||
# 修改虚拟机xml文件
|
||
# 虚拟机配置文件/etc/libvirt/qemu/xxx.xml
|
||
# 虚拟机数据文件/var/lib/libvirt/images/xxx.qcow2
|
||
# xxx为虚拟机名字
|
||
virsh list --all
|
||
virsh edit xxx
|
||
# devices下添加, 注意修改uuid/pool-name/image-name/vdb
|
||
<disk type='network' device='disk'>
|
||
<driver name='qemu' type='raw'/>
|
||
<auth username='admin'>
|
||
<secret type='ceph' uuid='<uuid>'>
|
||
</auth>
|
||
<source protocol='rbd' name='<pool-name>/<image-name>'> <host name='<mon-ip>' port='6789'> </source>
|
||
<target dev='<vdb>' bus='virtio'/>
|
||
</disk>
|
||
```
|
||
|
||
## 文件存储
|
||
```shell
|
||
# mds服务器
|
||
yum install -y ceph-mds
|
||
# admin
|
||
# 创建mds服务
|
||
ceph-deploy mds create node4
|
||
# 同步配置文件及key
|
||
ceph-deploy admin node4
|
||
|
||
# 创建存储池
|
||
# 创建存储池,用于存储block(块),128PG
|
||
# pg为2的次方，推荐64或128
|
||
ceph osd pool create <data-name> 128
|
||
# 创建存储池,用于存放inode.128PG
|
||
ceph osd poll create <inode-name> 128
|
||
# 查看mds状态
|
||
ceph mds stat
|
||
|
||
# 创建文件系统
|
||
ceph fs new <fs-name> <inode-name> <data-name>
|
||
# 查看文件系统
|
||
ceph ls fs
|
||
# 挂载
|
||
mount -t ceph <mon-ip>:6789:/ /mnt -o name=admin,secret=<client.admin.key>
|
||
```
|
||
|
||
## 对象存储
|
||
```shell
|
||
# admin
|
||
ceph-deploy install --rgw node5
|
||
# 以下命令一样
|
||
yum install -y ceph-radosgw
|
||
ceph-deploy rgw create node5
|
||
# 修改服务端口,若需要
|
||
cat >> /etc/ceph.conf <<EOF
|
||
[client.rgw.node5]
|
||
host=node5
|
||
rgw_frontends='civetweb.port=8000'
|
||
EOF
|
||
```
|
||
|