|
一、关于sharding strategy 及 chunk split mongos> sh.status() --- Sharding Status --- sharding version: { "_id" : 1, "minCompatibleVersion" : 5, "currentVersion" : 6, "clusterId" : ObjectId("5850e59fce8b5f7ab7cfad34") } shards: active mongoses: "3.4.0" : 1 autosplit: Currently enabled: yes balancer: Currently enabled: yes Currently running: no Balancer lock taken at Wed Dec 14 2016 06:24:31 GMT+0000 (UTC) by ConfigServer:Balancer Failed balancer rounds in last 5 attempts: 0 Migration Results for the last 24 hours: No recent migrations databases: mongos> use admin switched to db admin mongos> sh.addShard("shard1/mongo01-jp:27027") { "shardAdded" : "shard1", "ok" : 1 } mongos> sh.addShard("shard2/mongo02-jp:27028") { "shardAdded" : "shard2", "ok" : 1 } mongos> sh.addShard("shard3/mongo03-jp:27029") { "shardAdded" : "shard3", "ok" : 1 } mongos> sh.status() --- Sharding Status --- sharding version: { "_id" : 1, "minCompatibleVersion" : 5, "currentVersion" : 6, "clusterId" : ObjectId("5850e59fce8b5f7ab7cfad34") } shards: { "_id" : "shard1", "host" : "shard1/mongo01-jp:27027,mongo02-jp:27027", "state" : 1 } { "_id" : "shard2", "host" : "shard2/mongo02-jp:27028,mongo03-jp:27028", "state" : 1 } { "_id" : "shard3", "host" : "shard3/mongo01-jp:27029,mongo03-jp:27029", "state" : 1 } active mongoses: "3.4.0" : 1 autosplit: Currently enabled: yes balancer: Currently enabled: yes Currently running: no Balancer lock taken at Wed Dec 14 2016 06:24:31 GMT+0000 (UTC) by ConfigServer:Balancer Failed balancer rounds in last 5 attempts: 0 Migration Results for the last 24 hours: No recent migrations databases: mongos> use config switched to db config mongos> db.chunks.find() mongos> db.settings.find() mongos> db.settings.save({"_id":"chunksize","value":NumberLong(5)}) WriteResult({ "nMatched" : 0, "nUpserted" : 1, "nModified" : 0, "_id" : "chunksize" }) mongos> db.settings.find() { "_id" : "chunksize", "value" : NumberLong(5) } 注: 默认每个chunk大小为64M，支持可以手工设置；较小的chunk可以使数据分布的更均衡，便于迁移，但是带来的问题就是split更加频繁，增加了mongos路由的开支(每个chunk持有的数据量变小，每次query需要访问更多chunks)；较大的chunk不便迁移，但是split次数较少，metadata信息较少，mongos路由简单；如果chunk过大会导致数据分布不均。个人认为64M还是较小，建议增加到256M。spit操作只会有insert或者update触发。 mongos> use admin
switched to db admin mongos> sh.enableSharding("test") { "ok" : 1 } mongos> sh.shardCollection("test.user",{uid:1}) 基于uid range 分片 { "collectionsharded" : "test.user", "ok" : 1 } mongos> use test switched to db test mongos> db.user.ensureIndex({uid:1}) 对于一个空collection，系统会自动创建索引 { "raw" : { "shard1/mongo01-jp:27027,mongo02-jp:27027" : { "createdCollectionAutomatically" : true, "numIndexesBefore" : 1, "numIndexesAfter" : 2, "ok" : 1, "$gleStats" : { "lastOpTime" : { "ts" : Timestamp(1481700282, 2), "t" : NumberLong(1) }, "electionId" : ObjectId("7fffffff0000000000000001") } }, "shard2/mongo02-jp:27028,mongo03-jp:27028" : { "createdCollectionAutomatically" : true, "numIndexesBefore" : 1, "numIndexesAfter" : 2, "ok" : 1, "$gleStats" : { "lastOpTime" : { "ts" : Timestamp(1481700282, 2), "t" : NumberLong(1) }, "electionId" : ObjectId("7fffffff0000000000000001") } }, "shard3/mongo01-jp:27029,mongo03-jp:27029" : { "createdCollectionAutomatically" : false, "numIndexesBefore" : 2, "numIndexesAfter" : 2, "note" : "all indexes already exist", "ok" : 1, "$gleStats" : { "lastOpTime" : { "ts" : Timestamp(1481700282, 1), "t" : NumberLong(1) }, "electionId" : ObjectId("7fffffff0000000000000001") } } }, "ok" : 1 } mongos> for (i=1;i<=100000;i++) db.user.insert({uid:'user'+i,age:(i%15),address:'#'+i+',nongda south road,beijing',preferbooks:['book'+i,'hello world']}) WriteResult({ "nInserted" : 1 }) mongos> sh.status() --- Sharding Status --- sharding version: { "_id" : 1, "minCompatibleVersion" : 5, "currentVersion" : 6, "clusterId" : ObjectId("5850e59fce8b5f7ab7cfad34") } shards: { "_id" : "shard1", "host" : "shard1/mongo01-jp:27027,mongo02-jp:27027", "state" : 1 } { "_id" : "shard2", "host" : "shard2/mongo02-jp:27028,mongo03-jp:27028", "state" : 1 } { "_id" : "shard3", "host" : "shard3/mongo01-jp:27029,mongo03-jp:27029", "state" : 1 } active mongoses: "3.4.0" : 1 autosplit: Currently enabled: yes balancer: Currently enabled: yes Currently running: no Balancer lock taken at Wed Dec 14 2016 06:24:31 GMT+0000 (UTC) by ConfigServer:Balancer Failed balancer rounds in last 5 attempts: 0 Migration Results for the last 24 hours: 8 : Success databases: { "_id" : "test", "primary" : "shard3", "partitioned" : true } test.user shard key: { "uid" : 1 } unique: false balancing: true chunks: shard1 5 shard2 4 shard3 4 { "uid" : { "$minKey" : 1 } } -->> { "uid" : "user10" } on : shard3 Timestamp(9, 1) { "uid" : "user10" } -->> { "uid" : "user17659" } on : shard1 Timestamp(4, 1) { "uid" : "user17659" } -->> { "uid" : "user2" } on : shard1 Timestamp(3, 4) { "uid" : "user2" } -->> { "uid" : "user2832" } on : shard1 Timestamp(6, 0) { "uid" : "user2832" } -->> { "uid" : "user32174" } on : shard3 Timestamp(7, 0) { "uid" : "user32174" } -->> { "uid" : "user36003" } on : shard1 Timestamp(8, 0) { "uid" : "user36003" } -->> { "uid" : "user5211" } on : shard2 Timestamp(8, 1) { "uid" : "user5211" } -->> { "uid" : "user5974" } on : shard2 Timestamp(6, 4) { "uid" : "user5974" } -->> { "uid" : "user6713" } on : shard2 Timestamp(4, 7) { "uid" : "user6713" } -->> { "uid" : "user8" } on : shard2 Timestamp(4, 4) { "uid" : "user8" } -->> { "uid" : "user87659" } on : shard3 Timestamp(8, 2) { "uid" : "user87659" } -->> { "uid" : "user9999" } on : shard3 Timestamp(8, 3) { "uid" : "user9999" } -->> { "uid" : { "$maxKey" : 1 } } on : shard1 Timestamp(9, 0)
mongos> use config switched to db config mongos> db.settings.find() { "_id" : "chunksize", "value" : NumberLong(5) } mongos> db.shards.find() { "_id" : "shard1", "host" : "shard1/mongo01-jp:27027,mongo02-jp:27027", "state" : 1 } { "_id" : "shard2", "host" : "shard2/mongo02-jp:27028,mongo03-jp:27028", "state" : 1 } { "_id" : "shard3", "host" : "shard3/mongo01-jp:27029,mongo03-jp:27029", "state" : 1 } mongos> db.databases.find() { "_id" : "test", "primary" : "shard3", "partitioned" : true } 注: 每个未分片的DB都会有一个primary shard mongos> db.chunks.find() { "_id" : "test.user-uid_MinKey", "lastmod" : Timestamp(9, 1), "lastmodEpoch" : ObjectId("5850f399db3a37e0bc576a2b"), "ns" : "test.user", "min" : { "uid" : { "$minKey" : 1 } }, "max" : { "uid" : "user10" }, "shard" : "shard3" } { "_id" : "test.user-uid_\"user2\"", "lastmod" : Timestamp(6, 0), "lastmodEpoch" : ObjectId("5850f399db3a37e0bc576a2b"), "ns" : "test.user", "min" : { "uid" : "user2" }, "max" : { "uid" : "user2832" }, "shard" : "shard1" } { "_id" : "test.user-uid_\"user8\"", "lastmod" : Timestamp(8, 2), "lastmodEpoch" : ObjectId("5850f399db3a37e0bc576a2b"), "ns" : "test.user", "min" : { "uid" : "user8" }, "max" : { "uid" : "user87659" }, "shard" : "shard3" } { "_id" : "test.user-uid_\"user10\"", "lastmod" : Timestamp(4, 1), "lastmodEpoch" : ObjectId("5850f399db3a37e0bc576a2b"), "ns" : "test.user", "min" : { "uid" : "user10" }, "max" : { "uid" : "user17659" }, "shard" : "shard1" } { "_id" : "test.user-uid_\"user17659\"", "lastmod" : Timestamp(3, 4), "lastmodEpoch" : ObjectId("5850f399db3a37e0bc576a2b"), "ns" : "test.user", "min" : { "uid" : "user17659" }, "max" : { "uid" : "user2" }, "shard" : "shard1" } { "_id" : "test.user-uid_\"user2832\"", "lastmod" : Timestamp(7, 0), "lastmodEpoch" : ObjectId("5850f399db3a37e0bc576a2b"), "ns" : "test.user", "min" : { "uid" : "user2832" }, "max" : { "uid" : "user32174" }, "shard" : "shard3" } { "_id" : "test.user-uid_\"user6713\"", "lastmod" : Timestamp(4, 4), "lastmodEpoch" : ObjectId("5850f399db3a37e0bc576a2b"), "ns" : "test.user", "min" : { "uid" : "user6713" }, "max" : { "uid" : "user8" }, "shard" : "shard2" } { "_id" : "test.user-uid_\"user32174\"", "lastmod" : Timestamp(8, 0), "lastmodEpoch" : ObjectId("5850f399db3a37e0bc576a2b"), "ns" : "test.user", "min" : { "uid" : "user32174" }, "max" : { "uid" : "user36003" }, "shard" : "shard1" } { "_id" : "test.user-uid_\"user5974\"", "lastmod" : Timestamp(4, 7), "lastmodEpoch" : ObjectId("5850f399db3a37e0bc576a2b"), "ns" : "test.user", "min" : { "uid" : "user5974" }, "max" : { "uid" : "user6713" }, "shard" : "shard2" } { "_id" : "test.user-uid_\"user36003\"", "lastmod" : Timestamp(8, 1), "lastmodEpoch" : ObjectId("5850f399db3a37e0bc576a2b"), "ns" : "test.user", "min" : { "uid" : "user36003" }, "max" : { "uid" : "user5211" }, "shard" : "shard2" } { "_id" : "test.user-uid_\"user5211\"", "lastmod" : Timestamp(6, 4), "lastmodEpoch" : ObjectId("5850f399db3a37e0bc576a2b"), "ns" : "test.user", "min" : { "uid" : "user5211" }, "max" : { "uid" : "user5974" }, "shard" : "shard2" } { "_id" : "test.user-uid_\"user87659\"", "lastmod" : Timestamp(8, 3), "lastmodEpoch" : ObjectId("5850f399db3a37e0bc576a2b"), "ns" : "test.user", "min" : { "uid" : "user87659" }, "max" : { "uid" : "user9999" }, "shard" : "shard3" } { "_id" : "test.user-uid_\"user9999\"", "lastmod" : Timestamp(9, 0), "lastmodEpoch" : ObjectId("5850f399db3a37e0bc576a2b"), "ns" : "test.user", "min" : { "uid" : "user9999" }, "max" : { "uid" : { "$maxKey" : 1 } }, "shard" : "shard1" } mongos> use admin switched to db admin mongos> sh.shardCollection("test.user2",{"_id":"hashed"}) 基于hash 分片 { "collectionsharded" : "test.user2", "ok" : 1 } mongos> use test switched to db test mongos> for (i=1;i<=100000;i++) db.user2.insert({uid:'user'+i,age:(i%15),address:'#'+i+',shangdi south road,beijing',preferbooks:['book'+i,'hello world']}) WriteResult({ "nInserted" : 1 }) mongos> sh.status() --- Sharding Status --- sharding version: { "_id" : 1, "minCompatibleVersion" : 5, "currentVersion" : 6, "clusterId" : ObjectId("5850e59fce8b5f7ab7cfad34") } shards: { "_id" : "shard1", "host" : "shard1/mongo01-jp:27027,mongo02-jp:27027", "state" : 1 } { "_id" : "shard2", "host" : "shard2/mongo02-jp:27028,mongo03-jp:27028", "state" : 1 } { "_id" : "shard3", "host" : "shard3/mongo01-jp:27029,mongo03-jp:27029", "state" : 1 } active mongoses: "3.4.0" : 1 autosplit: Currently enabled: yes balancer: Currently enabled: yes Currently running: no Balancer lock taken at Wed Dec 14 2016 06:24:31 GMT+0000 (UTC) by ConfigServer:Balancer Failed balancer rounds in last 5 attempts: 0 Migration Results for the last 24 hours: 11 : Success databases: { "_id" : "test", "primary" : "shard3", "partitioned" : true } test.user shard key: { "uid" : 1 } unique: false balancing: true chunks: shard1 5 shard2 4 shard3 4 { "uid" : { "$minKey" : 1 } } -->> { "uid" : "user10" } on : shard3 Timestamp(9, 1) { "uid" : "user10" } -->> { "uid" : "user17659" } on : shard1 Timestamp(4, 1) { "uid" : "user17659" } -->> { "uid" : "user2" } on : shard1 Timestamp(3, 4) { "uid" : "user2" } -->> { "uid" : "user2832" } on : shard1 Timestamp(6, 0) { "uid" : "user2832" } -->> { "uid" : "user32174" } on : shard3 Timestamp(7, 0) { "uid" : "user32174" } -->> { "uid" : "user36003" } on : shard1 Timestamp(8, 0) { "uid" : "user36003" } -->> { "uid" : "user5211" } on : shard2 Timestamp(8, 1) { "uid" : "user5211" } -->> { "uid" : "user5974" } on : shard2 Timestamp(6, 4) { "uid" : "user5974" } -->> { "uid" : "user6713" } on : shard2 Timestamp(4, 7) { "uid" : "user6713" } -->> { "uid" : "user8" } on : shard2 Timestamp(4, 4) { "uid" : "user8" } -->> { "uid" : "user87659" } on : shard3 Timestamp(8, 2) { "uid" : "user87659" } -->> { "uid" : "user9999" } on : shard3 Timestamp(8, 3) { "uid" : "user9999" } -->> { "uid" : { "$maxKey" : 1 } } on : shard1 Timestamp(9, 0) test.user2 shard key: { "_id" : "hashed" } unique: false balancing: true chunks: shard1 3 shard2 3 shard3 2 { "_id" : { "$minKey" : 1 } } -->> { "_id" : NumberLong("-7785268750491956891") } on : shard2 Timestamp(4, 0) { "_id" : NumberLong("-7785268750491956891") } -->> { "_id" : NumberLong("-6358150028456512613") } on : shard1 Timestamp(4, 1) { "_id" : NumberLong("-6358150028456512613") } -->> { "_id" : NumberLong("-6148914691236517204") } on : shard1 Timestamp(3, 10) { "_id" : NumberLong("-6148914691236517204") } -->> { "_id" : NumberLong("-3074457345618258602") } on : shard1 Timestamp(3, 3) { "_id" : NumberLong("-3074457345618258602") } -->> { "_id" : NumberLong(0) } on : shard2 Timestamp(3, 4) { "_id" : NumberLong(0) } -->> { "_id" : NumberLong("3074457345618258602") } on : shard2 Timestamp(3, 5) { "_id" : NumberLong("3074457345618258602") } -->> { "_id" : NumberLong("6148914691236517204") } on : shard3 Timestamp(3, 6) { "_id" : NumberLong("6148914691236517204") } -->> { "_id" : { "$maxKey" : 1 } } on : shard3 Timestamp(3, 7) 二、关于balancing balancing：如果一个shard上chunks比其他shard更多，即不平衡状态，那么mongos将会自动对chunks迁移以达到平衡，balancing的过程不会影响用户的数据操作。集群中任何mongos实例都可以启动balancing线程，默认balancer是开启状态；Config 数据库（Config servers中）中有个lock表，当balancer活跃时，相应的mongos将会尝试通过修改document方式获取“lock”，如果获取“lock”成功，则此mongos负责balancing工作。大家需要注意，mongos实例的本地系统时间会对lock机制带来影响，需要所有的mongos（包括集群中的所有shards、config servers）的时间保持一致（ntpd指令）。 balancer将chunks从持有chunks最多的shard上迁移到持久chunks最少的shard，每次迁移一个，直到集群相对平衡 (最多chunks节点与最少chunks节点之间相差不超过threshold个)。chunks迁移可能会消耗磁盘空间，那些已经迁移出去的chunks不会立即删除，而是归档到一个特定的目录下，“归档”（archive）特性默认是开启的；此外迁移还会消耗一定的网络带宽，或许会影响到性能，影响用户操作的IO吞吐量。建议每次迁移一个chunk，且只有当“最多”与“最少”的差值达到threshold时才开始balancer；或者指定一个时间区间，balancer只会在此时间段内才会迁移chunks。 threshold：该值目前没有办法修改，当chunks总数< 20时，threshold=2，总数 >= 80时，threshold=8，其他为4。一旦balancing工作启动，只有当chunks分布均衡后才会停止。 默认情况下，mongodb会尽可能的耗尽可用磁盘空间，所以需要关注mongodb对磁盘的消耗量；不过当向集群中添加shard节点时，可以指定当前shard允许使用的最大磁盘空间（max size），当shard的磁盘消耗量达到最大值后，balancer将不会向其再迁移chunks，但这不影响此shard上继续接受write操作。 balancer运行在mongos实例上，控制chunks的分布和迁移，全局只有一个balancer处于active状态，可以通过“sh.getBalancerState()”或者“sh.status()”查看balancer是否开启，可以通过“sh.getBalancerHost()”查看balancer运行在哪个mongos上。 1）可以通过sh.setBalancerState(false)来关闭balancer功能，也可以通过设定为true开启balancer。 2）可以通过sh.startBlancer()或者sh.stopBalancer()来开关闭balancer。 3）可以通过db.locks.find({_id:"balancer"})查看balancer持有锁的情况。 4）可以通过修改settting表中的配置来指定balancer的运行时间区间 。其中start和stop格式为“HH:mm”，不需要指定日期。修改activeWindow配置时需要确保balancer的state为true mongos> sh.getBalancerState() true mongos> sh.setBalancerState(false) { "ok" : 1 } mongos> sh.getBalancerState() false mongos> sh.setBalancerState(true) { "ok" : 1 } mongos> use config switched to db config mongos> db.locks.find({_id:"balancer"}) { "_id" : "balancer", "state" : 2, "ts" : ObjectId("5850de85ce8b5f7ab7cfacdd"), "who" : "ConfigServer:Balancer", "process" : "ConfigServer", "when" : ISODate("2016-12-14T06:24:31.625Z"), "why" : "CSRS Balancer" } mongos> db.settings.find() { "_id" : "chunksize", "value" : NumberLong(5) } { "_id" : "balancer", "stopped" : false, "mode" : "full" } mongos> db.settings.update( ... { _id:"balancer"}, ... { $set: { activeWindow: { start:"23:00",stop:"6:00"} } }, ... { upsert: true } ... ) WriteResult({ "nMatched" : 1, "nUpserted" : 0, "nModified" : 1 }) mongos> db.settings.find() { "_id" : "chunksize", "value" : NumberLong(5) } { "_id" : "balancer", "stopped" : false, "mode" : "full", "activeWindow" : { "start" : "23:00", "stop" : "6:00" } } 参数 _secondaryThrottle与waitForDelete：
_secondaryThrottle表示是对secondary进行节流，默认为true，其语义等同write concern中的{w:2}，即当chunk迁移时（documents复制）destination shard中至少有一个secondary接收成功，balancer才会继续进行下一个chunk；不过开发者可以关闭此参数（同{w:1}），同时与write concern一起使用 _waitForDelete表示balancer是否等待source shard删除已经迁移成功的chunk后才继续处理下一个chunk，默认为false，即不等待。 mongos> use config switched to db config mongos> db.settings.update ( ... { "_id":"balancer"}, ... { $set: {"_secondaryThrottle":false, "writeConncern":{"w":"majority"} } }, ... { upsert: true} ... ) WriteResult({ "nMatched" : 1, "nUpserted" : 0, "nModified" : 1 }) mongos> db.setting.update( ... {"_id":"balancer"}, ... {$set:{"_waitForDelete":true } }, ... {upsert:true} ... ) WriteResult({ "nMatched" : 0, "nUpserted" : 1, "nModified" : 0, "_id" : "balancer" }) mongos>
| 
QQ群⑧：
窥视卡
雷达卡
发表于 2016-12-15 08:55:11
提升卡
置顶卡
沉默卡
喧嚣卡
变色卡
千斤顶
显身卡