Sharding

June 02, 2017

Sharding

N is number of copies of each shard and number of parallel processes spawned by coordinating node
Q is number of shards

coord node waits N/2 +1 responses, before merging as HTTP response to client

recommended is max 10000000 docs per shard
Config N=3, Q=8

shard address	nodes	nodes	nodes	nodes
00000000-1fffffff
20000000-3fffffff
40000000-5fffffff
60000000-7fffffff
80000000-9fffffff
a0000000-bfffffff
c0000000-dfffffff
e0000000-ffffffff

Database configuration for three node cluster with N=3, Q=8

{
"_id": "products",
"_rev": "1-95e1ceac4f9ec6842c40519950ebc4cf",
"shard_suffix": [
46,
49,
52,
57,
54,
52,
48,
56,
48,
49,
55
],
"changelog": [
[
"add",
"00000000-1fffffff",
"node_1@172.17.0.2"
],
[
"add",
"00000000-1fffffff",
"node_2@172.17.0.3"
],
[
"add",
"00000000-1fffffff",
"node_3@172.17.0.4"
],
[
"add",
"20000000-3fffffff",
"node_1@172.17.0.2"
],
[
"add",
"20000000-3fffffff",
"node_2@172.17.0.3"
],
[
"add",
"20000000-3fffffff",
"node_3@172.17.0.4"
],
[
"add",
"40000000-5fffffff",
"node_1@172.17.0.2"
],
[
"add",
"40000000-5fffffff",
"node_2@172.17.0.3"
],
[
"add",
"40000000-5fffffff",
"node_3@172.17.0.4"
],
[
"add",
"60000000-7fffffff",
"node_1@172.17.0.2"
],
[
"add",
"60000000-7fffffff",
"node_2@172.17.0.3"
],
[
"add",
"60000000-7fffffff",
"node_3@172.17.0.4"
],
[
"add",
"80000000-9fffffff",
"node_1@172.17.0.2"
],
[
"add",
"80000000-9fffffff",
"node_2@172.17.0.3"
],
[
"add",
"80000000-9fffffff",
"node_3@172.17.0.4"
],
[
"add",
"a0000000-bfffffff",
"node_1@172.17.0.2"
],
[
"add",
"a0000000-bfffffff",
"node_2@172.17.0.3"
],
[
"add",
"a0000000-bfffffff",
"node_3@172.17.0.4"
],
[
"add",
"c0000000-dfffffff",
"node_1@172.17.0.2"
],
[
"add",
"c0000000-dfffffff",
"node_2@172.17.0.3"
],
[
"add",
"c0000000-dfffffff",
"node_3@172.17.0.4"
],
[
"add",
"e0000000-ffffffff",
"node_1@172.17.0.2"
],
[
"add",
"e0000000-ffffffff",
"node_2@172.17.0.3"
],
[
"add",
"e0000000-ffffffff",
"node_3@172.17.0.4"
]
],
"by_node": {
"node_1@172.17.0.2": [
"00000000-1fffffff",
"20000000-3fffffff",
"40000000-5fffffff",
"60000000-7fffffff",
"80000000-9fffffff",
"a0000000-bfffffff",
"c0000000-dfffffff",
"e0000000-ffffffff"
],
"node_2@172.17.0.3": [
"00000000-1fffffff",
"20000000-3fffffff",
"40000000-5fffffff",
"60000000-7fffffff",
"80000000-9fffffff",
"a0000000-bfffffff",
"c0000000-dfffffff",
"e0000000-ffffffff"
],
"node_3@172.17.0.4": [
"00000000-1fffffff",
"20000000-3fffffff",
"40000000-5fffffff",
"60000000-7fffffff",
"80000000-9fffffff",
"a0000000-bfffffff",
"c0000000-dfffffff",
"e0000000-ffffffff"
]
},
"by_range": {
"00000000-1fffffff": [
"node_1@172.17.0.2",
"node_2@172.17.0.3",
"node_3@172.17.0.4"
],
"20000000-3fffffff": [
"node_1@172.17.0.2",
"node_2@172.17.0.3",
"node_3@172.17.0.4"
],
"40000000-5fffffff": [
"node_1@172.17.0.2",
"node_2@172.17.0.3",
"node_3@172.17.0.4"
],
"60000000-7fffffff": [
"node_1@172.17.0.2",
"node_2@172.17.0.3",
"node_3@172.17.0.4"
],
"80000000-9fffffff": [
"node_1@172.17.0.2",
"node_2@172.17.0.3",
"node_3@172.17.0.4"
],
"a0000000-bfffffff": [
"node_1@172.17.0.2",
"node_2@172.17.0.3",
"node_3@172.17.0.4"
],
"c0000000-dfffffff": [
"node_1@172.17.0.2",
"node_2@172.17.0.3",
"node_3@172.17.0.4"
],
"e0000000-ffffffff": [
"node_1@172.17.0.2",
"node_2@172.17.0.3",
"node_3@172.17.0.4"
]
}
}

Database configuration for ten nodes cluster with N=3, Q=8

{
"_id": "products",
"_rev": "1-d6177c522250c6da73bf2095d1fa0bcb",
"value": {
"rev": "1-d6177c522250c6da73bf2095d1fa0bcb"
},
"key": "products",
"doc": {
"_id": "products",
"_rev": "1-d6177c522250c6da73bf2095d1fa0bcb",
"shard_suffix": [
46,
49,
52,
57,
54,
52,
57,
54,
56,
57,
50
],
"changelog": [
[
"add",
"00000000-1fffffff",
"node_4_10@172.17.0.5"
],
[
"add",
"00000000-1fffffff",
"node_5_10@172.17.0.6"
],
[
"add",
"00000000-1fffffff",
"node_6_10@172.17.0.7"
],
[
"add",
"20000000-3fffffff",
"node_1_10@172.17.0.2"
],
[
"add",
"20000000-3fffffff",
"node_2_10@172.17.0.3"
],
[
"add",
"20000000-3fffffff",
"node_3_10@172.17.0.4"
],
[
"add",
"40000000-5fffffff",
"node_10_10@172.17.0.11"
],
[
"add",
"40000000-5fffffff",
"node_8_10@172.17.0.9"
],
[
"add",
"40000000-5fffffff",
"node_9_10@172.17.0.10"
],
[
"add",
"60000000-7fffffff",
"node_5_10@172.17.0.6"
],
[
"add",
"60000000-7fffffff",
"node_6_10@172.17.0.7"
],
[
"add",
"60000000-7fffffff",
"node_7_10@172.17.0.8"
],
[
"add",
"80000000-9fffffff",
"node_2_10@172.17.0.3"
],
[
"add",
"80000000-9fffffff",
"node_3_10@172.17.0.4"
],
[
"add",
"80000000-9fffffff",
"node_4_10@172.17.0.5"
],
[
"add",
"a0000000-bfffffff",
"node_10_10@172.17.0.11"
],
[
"add",
"a0000000-bfffffff",
"node_1_10@172.17.0.2"
],
[
"add",
"a0000000-bfffffff",
"node_9_10@172.17.0.10"
],
[
"add",
"c0000000-dfffffff",
"node_6_10@172.17.0.7"
],
[
"add",
"c0000000-dfffffff",
"node_7_10@172.17.0.8"
],
[
"add",
"c0000000-dfffffff",
"node_8_10@172.17.0.9"
],
[
"add",
"e0000000-ffffffff",
"node_3_10@172.17.0.4"
],
[
"add",
"e0000000-ffffffff",
"node_4_10@172.17.0.5"
],
[
"add",
"e0000000-ffffffff",
"node_5_10@172.17.0.6"
]
],
"by_node": {
"node_10_10@172.17.0.11": [
"40000000-5fffffff",
"a0000000-bfffffff"
],
"node_1_10@172.17.0.2": [
"20000000-3fffffff",
"a0000000-bfffffff"
],
"node_2_10@172.17.0.3": [
"20000000-3fffffff",
"80000000-9fffffff"
],
"node_3_10@172.17.0.4": [
"20000000-3fffffff",
"80000000-9fffffff",
"e0000000-ffffffff"
],
"node_4_10@172.17.0.5": [
"00000000-1fffffff",
"80000000-9fffffff",
"e0000000-ffffffff"
],
"node_5_10@172.17.0.6": [
"00000000-1fffffff",
"60000000-7fffffff",
"e0000000-ffffffff"
],
"node_6_10@172.17.0.7": [
"00000000-1fffffff",
"60000000-7fffffff",
"c0000000-dfffffff"
],
"node_7_10@172.17.0.8": [
"60000000-7fffffff",
"c0000000-dfffffff"
],
"node_8_10@172.17.0.9": [
"40000000-5fffffff",
"c0000000-dfffffff"
],
"node_9_10@172.17.0.10": [
"40000000-5fffffff",
"a0000000-bfffffff"
]
},
"by_range": {
"00000000-1fffffff": [
"node_4_10@172.17.0.5",
"node_5_10@172.17.0.6",
"node_6_10@172.17.0.7"
],
"20000000-3fffffff": [
"node_1_10@172.17.0.2",
"node_2_10@172.17.0.3",
"node_3_10@172.17.0.4"
],
"40000000-5fffffff": [
"node_10_10@172.17.0.11",
"node_8_10@172.17.0.9",
"node_9_10@172.17.0.10"
],
"60000000-7fffffff": [
"node_5_10@172.17.0.6",
"node_6_10@172.17.0.7",
"node_7_10@172.17.0.8"
],
"80000000-9fffffff": [
"node_2_10@172.17.0.3",
"node_3_10@172.17.0.4",
"node_4_10@172.17.0.5"
],
"a0000000-bfffffff": [
"node_10_10@172.17.0.11",
"node_1_10@172.17.0.2",
"node_9_10@172.17.0.10"
],
"c0000000-dfffffff": [
"node_6_10@172.17.0.7",
"node_7_10@172.17.0.8",
"node_8_10@172.17.0.9"
],
"e0000000-ffffffff": [
"node_3_10@172.17.0.4",
"node_4_10@172.17.0.5",
"node_5_10@172.17.0.6"
]
}
}
}

https://github.com/RipcordSoftware/AvanceDB

Search

Development with Elixir, Phoenix and CouchDB(sorry for drafts...)

Sharding

Comments

Post a Comment

Popular Posts

Configuring nodes to work inside cluster

Building CouchDB 2.0 Docker image