更新 '存储/ceph/ceph-deploy.md'
This commit is contained in:
parent
150b88013a
commit
e7be90fdcb
1 changed files with 0 additions and 0 deletions
228
存储/ceph/base.md
228
存储/ceph/base.md
|
|
@ -1,228 +0,0 @@
|
|||
### 模块
|
||||
OSD(rados): 核心,用于集群中数据与对象的存储,处理对象的复制,恢复,再均衡等
|
||||
MON: 监控集群状态(奇数个), 维护集群clustermap,保证数据一致性
|
||||
RGW(radosgw): 对象存储需要
|
||||
MDS: 文件系统存储需要
|
||||
|
||||
MGR: 监控
|
||||
|
||||
### 配置文件
|
||||
/etc/ceph.conf ## mon及密码等, 客户端连接密码可以不配置
|
||||
|
||||
/etc/ceph/ceph.client.admin.keyring ## 客户端密码文件
|
||||
|
||||
* 时间需要同步
|
||||
|
||||
### 安装ceph-deploy
|
||||
```shell
|
||||
# 需要在生成免密登录服务器上执行
|
||||
yum -y install ceph-deploy
|
||||
# 帮助
|
||||
ceph-deploy mon --help
|
||||
```
|
||||
### 创建节点生成配置文件
|
||||
```shell
|
||||
ceph-deploy new node1 node2 node3
|
||||
```
|
||||
### 安装ceph
|
||||
```shell
|
||||
yum -y install ceph-mon ceph-osd ceph-mds ceph-radosgw
|
||||
# ceph-deploy也可以安装
|
||||
ceph-deploy install node1 node2 node3
|
||||
```
|
||||
|
||||
### 初始化节点（配置文件传输到节点，启动mon）
|
||||
```shell
|
||||
ceph-deploy mon create-initial
|
||||
# 或以下
|
||||
ceph-deploy mon add node1
|
||||
ceph-deploy mon add node2
|
||||
ceph-deploy mon add node3
|
||||
```
|
||||
### 查状态
|
||||
```shell
|
||||
ceph -s
|
||||
```
|
||||
### 创建osd
|
||||
```shell
|
||||
# 需要缓存盘用于缓存写入数据
|
||||
# 若全部使用，可以使用ceph-deploy做，若有不同分区，需自行分区
|
||||
parted /dev/vdb mklabel gpt
|
||||
parted /dev/vdb mkpart primary 1 50%
|
||||
parted /dev/vdb mkpart 50% 100%
|
||||
# 修改所属用户与组，使ceph有权限写入
|
||||
chown ceph:ceph /dev/vdb1
|
||||
chown ceph:ceph /dev/vdb2
|
||||
# 以上操作只能保证当前有效，重启会恢复。以下操作可以永久生效
|
||||
cat >> /etc/udev/rules.d/70-vdb.rules <<EOF
|
||||
ENV{DEVNAME}=="dev/vdb1",OWNER="ceph",GROUP="ceph"
|
||||
ENV{DEVNAME}=="dev/vdb2",OWNER="ceph",GROUP="ceph"
|
||||
EOF
|
||||
# 格式化数据盘
|
||||
ceph-deploy disk zap node1:/dev/vdc node1:/dev/vdd
|
||||
ceph-deploy disk zap node2:/dev/vdc node2:/dev/vdd
|
||||
ceph-deploy disk zap node3:/dev/vdc node3:/dev/vdd
|
||||
|
||||
# 创建osd vdc，缓存为/dev/vdb1, osd vdd,缓存/dev/vdb2,并启动服务
|
||||
ceph-deploy osd create node1:vdc:/dev/vdb1 node1:vdd:/dev/vdb2
|
||||
ceph-deploy osd create node2:vdc:/dev/vdb1 node2:vdd:/dev/vdb2
|
||||
ceph-deploy osd create node3:vdc:/dev/vdb1 node3:vdd:/dev/vdb2
|
||||
|
||||
# 查看状态
|
||||
ceph -s
|
||||
# 查看osd信息
|
||||
ceph osd tree
|
||||
```
|
||||
## 存储池
|
||||
```shell
|
||||
# 默认已经创建0 rdb
|
||||
ceph osd lspools
|
||||
```
|
||||
## 镜像
|
||||
```shell
|
||||
# layering: 开启cow功能，写时复制(可以做增量快照)
|
||||
rdb create <image-name> --image-feature layering --size 10g
|
||||
rdb create <pool-name>/<image-name> --image-feature layering --size 10g
|
||||
|
||||
# 查看
|
||||
rbd list
|
||||
rbd info <image-name>
|
||||
# 扩缩
|
||||
# --allow-shrink: 缩减需要添加
|
||||
rbd resize --rize 7G <image-name> --allow-shrink
|
||||
```
|
||||
|
||||
## 客户端访问
|
||||
```shell
|
||||
# 客户端安装
|
||||
yum install -y ceph-common
|
||||
# 配置文件及密码
|
||||
# 从集群中复制相应文件
|
||||
/etc/ceph/ceph.conf
|
||||
/etc/ceph/ceph.client.admin.keyring
|
||||
|
||||
# 添加集群
|
||||
rbd map <image-name>
|
||||
# 查看本地磁盘
|
||||
# 本地多了rbd*,可以正常格式化挂载操作
|
||||
lsblk
|
||||
|
||||
# 查看image使用情况
|
||||
rbd showmapped
|
||||
# 查看image
|
||||
rbd image list
|
||||
```
|
||||
## 快照
|
||||
ceph多副本,保证osd异常时,数据不丢失
|
||||
快照防止误操作
|
||||
```shell
|
||||
# 查看image快照
|
||||
rbd snap ls <image-name>
|
||||
# 创建快照
|
||||
rdb snap creat image --snap <snap-image>
|
||||
# 操作数据,如删除
|
||||
# 卸载挂载目录(挂载地方)
|
||||
umount /mnt
|
||||
# 还原数据(恢复快照)
|
||||
rbd snap rollback <image> --snap <snap-name>
|
||||
# 重新挂载目录即可恢复数据
|
||||
```
|
||||
## 克隆
|
||||
```shell
|
||||
# 使用快照克隆比较快
|
||||
# protect: 保护snap,防止被删
|
||||
rbd snap protect <image-name> --snap <snap-name>
|
||||
# 如下命令会删除失败
|
||||
rbd snap rm <imag-name> --snap <snap-name>
|
||||
# 克隆image
|
||||
rbd clone <image-name> --snap <snap-name> image-clone --image-feature layering
|
||||
# 查看父子image
|
||||
rbd info <image-name-clone>
|
||||
# 去掉父子关系, 使复制后的image独立工作,需要时间
|
||||
rbd flatten <image-name-clone>
|
||||
# 快照删除
|
||||
rbd snap unprotect <image-name> --snap <snap-name>
|
||||
rbd snap rm <imag-name> --snap <snap-name>
|
||||
```
|
||||
## 本地去掉盘
|
||||
```shell
|
||||
umount /mnt
|
||||
rbd unmap <image-name>
|
||||
```
|
||||
## 虚拟机中使用ceph
|
||||
```shell
|
||||
# ceph需要认证
|
||||
# 安装虚拟机的服务器中,需要配置如下文件
|
||||
cat > secret.xml <<EOF
|
||||
<secret ephemeral='no' private='no'>
|
||||
<usage type='ceph'>
|
||||
<name>client-admin secret</name>
|
||||
</usage>
|
||||
</secret>
|
||||
EOF
|
||||
# 创建
|
||||
virsh secret-define secret.xml
|
||||
virsh secret-list
|
||||
# 绑定密码
|
||||
# secret-uuid为以上命令查询所得
|
||||
# /etc/ceph/ceph.client.admin.keyring中key后的密码
|
||||
virsh secret-set-value --secret <secret-uuid> --base64 <client.admin.key>
|
||||
# 修改虚拟机xml文件
|
||||
# 虚拟机配置文件/etc/libvirt/qemu/xxx.xml
|
||||
# 虚拟机数据文件/var/lib/libvirt/images/xxx.qcow2
|
||||
# xxx为虚拟机名字
|
||||
virsh list --all
|
||||
virsh edit xxx
|
||||
# devices下添加, 注意修改uuid/pool-name/image-name/vdb
|
||||
<disk type='network' device='disk'>
|
||||
<driver name='qemu' type='raw'/>
|
||||
<auth username='admin'>
|
||||
<secret type='ceph' uuid='<uuid>'>
|
||||
</auth>
|
||||
<source protocol='rbd' name='<pool-name>/<image-name>'> <host name='<mon-ip>' port='6789'> </source>
|
||||
<target dev='<vdb>' bus='virtio'/>
|
||||
</disk>
|
||||
```
|
||||
|
||||
## 文件存储
|
||||
```shell
|
||||
# mds服务器
|
||||
yum install -y ceph-mds
|
||||
# admin
|
||||
# 创建mds服务
|
||||
ceph-deploy mds create node4
|
||||
# 同步配置文件及key
|
||||
ceph-deploy admin node4
|
||||
|
||||
# 创建存储池
|
||||
# 创建存储池,用于存储block(块),128PG
|
||||
# pg为2的次方，推荐64或128
|
||||
ceph osd pool create <data-name> 128
|
||||
# 创建存储池,用于存放inode.128PG
|
||||
ceph osd poll create <inode-name> 128
|
||||
# 查看mds状态
|
||||
ceph mds stat
|
||||
|
||||
# 创建文件系统
|
||||
ceph fs new <fs-name> <inode-name> <data-name>
|
||||
# 查看文件系统
|
||||
ceph ls fs
|
||||
# 挂载
|
||||
mount -t ceph <mon-ip>:6789:/ /mnt -o name=admin,secret=<client.admin.key>
|
||||
```
|
||||
|
||||
## 对象存储
|
||||
```shell
|
||||
# admin
|
||||
ceph-deploy install --rgw node5
|
||||
# 以下命令一样
|
||||
yum install -y ceph-radosgw
|
||||
ceph-deploy rgw create node5
|
||||
# 修改服务端口,若需要
|
||||
cat >> /etc/ceph.conf <<EOF
|
||||
[client.rgw.node5]
|
||||
host=node5
|
||||
rgw_frontends='civetweb.port=8000'
|
||||
EOF
|
||||
```
|
||||
|
||||
Loading…
Add table
Add a link
Reference in a new issue