Ceph cache 开启验证生效

qq78707

• 　　Nova 配置
  disk_cachemodes = "network=writeback” (启用)  
  改为 disk_cachemodes = “network=none"（关闭）
  
  
• 　　Ceph 配置
  
  
  
  • 　　开启 ceph rbd cache
    client]
    rbd_cache = true
    rbd_cache_writethrough_until_flush = true
    admin_socket = /var/run/ceph/guests/$cluster-$type.$id.$pid.$cctid.asok
    log_file = /var/log/qemu/qemu-guest-$pid.log
    rbd_concurrent_management_ops = 20
    
    
  • 关闭ceph rbd cache
    [client]
    #rbd_cache = true
    #rbd_cache_writethrough_until_flush = true
    admin_socket = /var/run/ceph/guests/$cluster-$type.$id.$pid.$cctid.asok
    #log_file = /var/log/qemu/qemu-guest-$pid.log
    #rbd_concurrent_management_ops = 20
    
  
  
  
• 　　验证cache是否生效
  
  
  
  • 验证 生效命令
    [root@l2-4-30 guests]# ceph --admin-daemo ceph-client.cinder.27346.94699498717184.asok   perf  dump
    {
    "finisher-librbd::TaskFinisher::m_finisher": {
    "queue_len": 0,
    "complete_latency": {
    "avgcount": 0,
    "sum": 0.000000000
    }
    },
    "librbd-1373b25a3be0cd-glance_images-86e30f80-5a93-47ee-88c9-de5c1db6df10": {
    "rd": 32853,
    "rd_bytes": 2060131840,
    "rd_latency": {
    "avgcount": 32853,
    "sum": 42.348302430
    },
    "wr": 0,
    "wr_bytes": 0,
    "wr_latency": {
    "avgcount": 0,
    "sum": 0.000000000
    },
    "discard": 0,
    "discard_bytes": 0,
    "discard_latency": {
    "avgcount": 0,
    "sum": 0.000000000
    },
    "flush": 0,
    "aio_flush": 0,
    "aio_flush_latency": {
    "avgcount": 0,
    "sum": 0.000000000
    },
    "snap_create": 0,
    "snap_remove": 0,
    "snap_rollback": 0,
    "snap_rename": 0,
    "notify": 0,
    "resize": 0,
    "readahead": 103,
    "readahead_bytes": 27451904,
    "invalidate_cache": 0
    },
    "librbd-bf1708496d088e-nova_instances-555153af-f088-4aec-9bae-e4d6389b4d98_disk": {
    "rd": 40116,
    "rd_bytes": 650777600,
    "rd_latency": {
    "avgcount": 40116,
    "sum": 38.547805260
    },
    "wr": 7348,
    "wr_bytes": 253340672,
    "wr_latency": {
    "avgcount": 7348,
    "sum": 55.895274365
    },
    "discard": 0,
    "discard_bytes": 0,
    "discard_latency": {
    "avgcount": 0,
    "sum": 0.000000000
    },
    "flush": 0,
    "aio_flush": 434,
    "aio_flush_latency": {
    "avgcount": 434,
    "sum": 0.540130077
    },
    "snap_create": 0,
    "snap_remove": 0,
    "snap_rollback": 0,
    "snap_rename": 0,
    "notify": 0,
    "resize": 0,
    "readahead": 130,
    "readahead_bytes": 37353472,
    "invalidate_cache": 0
    },
    "objectcacher-librbd-1373b25a3be0cd-glance_images-86e30f80-5a93-47ee-88c9-de5c1db6df10": {
    "cache_ops_hit": 2761,
    "cache_ops_miss": 30195,
    "cache_bytes_hit": 1500713472,
    "cache_bytes_miss": 2063611392,
    "data_read": 2087583744,
    "data_written": 0,
    "data_flushed": 0,
    "data_overwritten_while_flushing": 0,
    "write_ops_blocked": 0,
    "write_bytes_blocked": 0,
    "write_time_blocked": 0.000000000
    },
    "objectcacher-librbd-bf1708496d088e-nova_instances-555153af-f088-4aec-9bae-e4d6389b4d98_disk": {
    "cache_ops_hit": 829,
    "cache_ops_miss": 7979,
    "cache_bytes_hit": 191516672,
    "cache_bytes_miss": 153311232,
    "data_read": 168047616,
    "data_written": 253340672,
    "data_flushed": 253340672,
    "data_overwritten_while_flushing": 0,
    "write_ops_blocked": 0,
    "write_bytes_blocked": 0,
    "write_time_blocked": 0.000000000
    },
    "objecter": {
    "op_active": 0,
    "op_laggy": 0,
    "op_send": 45229,
    "op_send_bytes": 0,
    "op_resend": 0,
    "op_ack": 45105,
    "op_commit": 7893,
    "op": 45229,
    "op_r": 37337,
    "op_w": 7892,
    "op_rmw": 0,
    "op_pg": 0,
    "osdop_stat": 7045,
    "osdop_create": 0,
    "osdop_read": 37319,
    "osdop_write": 7519,
    "osdop_writefull": 0,
    "osdop_append": 0,
    "osdop_zero": 0,
    "osdop_truncate": 0,
    "osdop_delete": 0,
    "osdop_mapext": 0,
    "osdop_sparse_read": 0,
    "osdop_clonerange": 0,
    "osdop_getxattr": 0,
    "osdop_setxattr": 0,
    "osdop_cmpxattr": 0,
    "osdop_rmxattr": 0,
    "osdop_resetxattrs": 0,
    "osdop_tmap_up": 0,
    "osdop_tmap_put": 0,
    "osdop_tmap_get": 0,
    "osdop_call": 1117,
    "osdop_watch": 122,
    "osdop_notify": 1,
    "osdop_src_cmpxattr": 0,
    "osdop_pgls": 0,
    "osdop_pgls_filter": 0,
    "osdop_other": 1,
    "linger_active": 1,
    "linger_send": 2,
    "linger_resend": 0,
    "linger_ping": 121,
    "poolop_active": 0,
    "poolop_send": 0,
    "poolop_resend": 0,
    "poolstat_active": 0,
    "poolstat_send": 0,
    "poolstat_resend": 0,
    "statfs_active": 0,
    "statfs_send": 0,
    "statfs_resend": 0,
    "command_active": 0,
    "command_send": 0,
    "command_resend": 0,
    "map_epoch": 0,
    "map_full": 0,
    "map_inc": 0,
    "osd_sessions": 27966,
    "osd_session_open": 236,
    "osd_session_close": 0,
    "osd_laggy": 1,
    "omap_wr": 0,
    "omap_rd": 0,
    "omap_del": 0
    },
    "throttle-msgr_dispatch_throttler-radosclient": {
    "val": 0,
    "max": 104857600,
    "get": 52511,
    "get_sum": 2152111179,
    "get_or_fail_fail": 0,
    "get_or_fail_success": 0,
    "take": 0,
    "take_sum": 0,
    "put": 52511,
    "put_sum": 2152111179,
    "wait": {
    "avgcount": 0,
    "sum": 0.000000000
    }
    },
    "throttle-objecter_bytes": {
    "val": 0,
    "max": 104857600,
    "get": 45108,
    "get_sum": 2419831296,
    "get_or_fail_fail": 0,
    "get_or_fail_success": 45108,
    "take": 0,
    "take_sum": 0,
    "put": 44725,
    "put_sum": 2419831296,
    "wait": {
    "avgcount": 0,
    "sum": 0.000000000
    }
    },
    "throttle-objecter_ops": {
    "val": 0,
    "max": 1024,
    "get": 45108,
    "get_sum": 45108,
    "get_or_fail_fail": 0,
    "get_or_fail_success": 45108,
    "take": 0,
    "take_sum": 0,
    "put": 45108,
    "put_sum": 45108,
    "wait": {
    "avgcount": 0,
    "sum": 0.000000000
    }
    }
    } 完整配置
    
  
  过滤 cache 字符串
  
      [root@l2-4-30 guests]# ceph --admin-daemo ceph-client.cinder.27346.94699498717184.asok   perf  dump |grep cache -A 5 -B 5
  "snap_rename": 0,
  "notify": 0,
  "resize": 0,
  "readahead": 103,
  "readahead_bytes": 27451904,
  "invalidate_cache": 0
  },
  "librbd-bf1708496d088e-nova_instances-555153af-f088-4aec-9bae-e4d6389b4d98_disk": {
  "rd": 57149,
  "rd_bytes": 1148467200,
  "rd_latency": {
  　　--
  "snap_rename": 0,
  "notify": 0,
  "resize": 0,
  "readahead": 130,
  "readahead_bytes": 37353472,
  "invalidate_cache": 0
  },
  "objectcacher-librbd-1373b25a3be0cd-glance_images-86e30f80-5a93-47ee-88c9-de5c1db6df10": {
  "cache_ops_hit": 2791,
  "cache_ops_miss": 45597,
  "cache_bytes_hit": 1526398976,
  "cache_bytes_miss": 3029461504,
  "data_read": 3057748992,
  "data_written": 0,
  "data_flushed": 0,
  "data_overwritten_while_flushing": 0,
  "write_ops_blocked": 0,
  "write_bytes_blocked": 0,
  "write_time_blocked": 0.000000000
  },
  "objectcacher-librbd-bf1708496d088e-nova_instances-555153af-f088-4aec-9bae-e4d6389b4d98_disk": {
  "cache_ops_hit": 886,
  "cache_ops_miss": 9957,
  "cache_bytes_hit": 248139264,
  "cache_bytes_miss": 225620480,
  "data_read": 240729088,
  "data_written": 462767104,
  "data_flushed": 462767104,
  "data_overwritten_while_flushing": 0,
  "write_ops_blocked": 0,
  
  
  
  
  • 
    验证 无cache命令
    [root@l23-4-40 guests]# ceph --admin-daemo ceph-client.cinder.138869.94547912278016.asok    perf  dump |grep cache -A 5 -B 5
    "snap_rename": 0,
    "notify": 0,
    "resize": 0,
    "readahead": 0,
    "readahead_bytes": 0,
    "invalidate_cache": 0
    },
    "librbd-bee65365887c0b-nova_instances-5cf85c18-6725-4674-88e4-60408c97f487_disk": {
    "rd": 62937,
    "rd_bytes": 1261381632,
    "rd_latency": {
    
                "snap_rename": 0,
    "notify": 0,
    "resize": 0,
    "readahead": 0,
    "readahead_bytes": 0,
    "invalidate_cache": 0
    },
    "objecter": {
    "op_active": 0,
    "op_laggy": 0,
    "op_send": 95327
    }