Sharding


N is number of copies of each shard and number of parallel processes spawned by  coordinating node
Q is number of shards

coord node waits N/2 +1 responses, before merging as HTTP response to client

recommended is max 10000000 docs per shard
Config N=3, Q=8

shard address nodes nodes nodes nodes
00000000-1fffffff
20000000-3fffffff
40000000-5fffffff
60000000-7fffffff
80000000-9fffffff
a0000000-bfffffff
c0000000-dfffffff
e0000000-ffffffff

Database configuration for three node cluster with N=3, Q=8


{
  "_id": "products",
  "_rev": "1-95e1ceac4f9ec6842c40519950ebc4cf",
  "shard_suffix": [
    46,
    49,
    52,
    57,
    54,
    52,
    48,
    56,
    48,
    49,
    55
  ],
  "changelog": [
    [
      "add",
      "00000000-1fffffff",
      "node_1@172.17.0.2"
    ],
    [
      "add",
      "00000000-1fffffff",
      "node_2@172.17.0.3"
    ],
    [
      "add",
      "00000000-1fffffff",
      "node_3@172.17.0.4"
    ],
    [
      "add",
      "20000000-3fffffff",
      "node_1@172.17.0.2"
    ],
    [
      "add",
      "20000000-3fffffff",
      "node_2@172.17.0.3"
    ],
    [
      "add",
      "20000000-3fffffff",
      "node_3@172.17.0.4"
    ],
    [
      "add",
      "40000000-5fffffff",
      "node_1@172.17.0.2"
    ],
    [
      "add",
      "40000000-5fffffff",
      "node_2@172.17.0.3"
    ],
    [
      "add",
      "40000000-5fffffff",
      "node_3@172.17.0.4"
    ],
    [
      "add",
      "60000000-7fffffff",
      "node_1@172.17.0.2"
    ],
    [
      "add",
      "60000000-7fffffff",
      "node_2@172.17.0.3"
    ],
    [
      "add",
      "60000000-7fffffff",
      "node_3@172.17.0.4"
    ],
    [
      "add",
      "80000000-9fffffff",
      "node_1@172.17.0.2"
    ],
    [
      "add",
      "80000000-9fffffff",
      "node_2@172.17.0.3"
    ],
    [
      "add",
      "80000000-9fffffff",
      "node_3@172.17.0.4"
    ],
    [
      "add",
      "a0000000-bfffffff",
      "node_1@172.17.0.2"
    ],
    [
      "add",
      "a0000000-bfffffff",
      "node_2@172.17.0.3"
    ],
    [
      "add",
      "a0000000-bfffffff",
      "node_3@172.17.0.4"
    ],
    [
      "add",
      "c0000000-dfffffff",
      "node_1@172.17.0.2"
    ],
    [
      "add",
      "c0000000-dfffffff",
      "node_2@172.17.0.3"
    ],
    [
      "add",
      "c0000000-dfffffff",
      "node_3@172.17.0.4"
    ],
    [
      "add",
      "e0000000-ffffffff",
      "node_1@172.17.0.2"
    ],
    [
      "add",
      "e0000000-ffffffff",
      "node_2@172.17.0.3"
    ],
    [
      "add",
      "e0000000-ffffffff",
      "node_3@172.17.0.4"
    ]
  ],
  "by_node": {
    "node_1@172.17.0.2": [
      "00000000-1fffffff",
      "20000000-3fffffff",
      "40000000-5fffffff",
      "60000000-7fffffff",
      "80000000-9fffffff",
      "a0000000-bfffffff",
      "c0000000-dfffffff",
      "e0000000-ffffffff"
    ],
    "node_2@172.17.0.3": [
      "00000000-1fffffff",
      "20000000-3fffffff",
      "40000000-5fffffff",
      "60000000-7fffffff",
      "80000000-9fffffff",
      "a0000000-bfffffff",
      "c0000000-dfffffff",
      "e0000000-ffffffff"
    ],
    "node_3@172.17.0.4": [
      "00000000-1fffffff",
      "20000000-3fffffff",
      "40000000-5fffffff",
      "60000000-7fffffff",
      "80000000-9fffffff",
      "a0000000-bfffffff",
      "c0000000-dfffffff",
      "e0000000-ffffffff"
    ]
  },
  "by_range": {
    "00000000-1fffffff": [
      "node_1@172.17.0.2",
      "node_2@172.17.0.3",
      "node_3@172.17.0.4"
    ],
    "20000000-3fffffff": [
      "node_1@172.17.0.2",
      "node_2@172.17.0.3",
      "node_3@172.17.0.4"
    ],
    "40000000-5fffffff": [
      "node_1@172.17.0.2",
      "node_2@172.17.0.3",
      "node_3@172.17.0.4"
    ],
    "60000000-7fffffff": [
      "node_1@172.17.0.2",
      "node_2@172.17.0.3",
      "node_3@172.17.0.4"
    ],
    "80000000-9fffffff": [
      "node_1@172.17.0.2",
      "node_2@172.17.0.3",
      "node_3@172.17.0.4"
    ],
    "a0000000-bfffffff": [
      "node_1@172.17.0.2",
      "node_2@172.17.0.3",
      "node_3@172.17.0.4"
    ],
    "c0000000-dfffffff": [
      "node_1@172.17.0.2",
      "node_2@172.17.0.3",
      "node_3@172.17.0.4"
    ],
    "e0000000-ffffffff": [
      "node_1@172.17.0.2",
      "node_2@172.17.0.3",
      "node_3@172.17.0.4"
    ]
  }
}

Database configuration for ten nodes cluster with N=3, Q=8

{
 "_id": "products",
 "_rev": "1-d6177c522250c6da73bf2095d1fa0bcb",
 "value": {
  "rev": "1-d6177c522250c6da73bf2095d1fa0bcb"
 },
 "key": "products",
 "doc": {
  "_id": "products",
  "_rev": "1-d6177c522250c6da73bf2095d1fa0bcb",
  "shard_suffix": [
   46,
   49,
   52,
   57,
   54,
   52,
   57,
   54,
   56,
   57,
   50
  ],
  "changelog": [
   [
    "add",
    "00000000-1fffffff",
    "node_4_10@172.17.0.5"
   ],
   [
    "add",
    "00000000-1fffffff",
    "node_5_10@172.17.0.6"
   ],
   [
    "add",
    "00000000-1fffffff",
    "node_6_10@172.17.0.7"
   ],
   [
    "add",
    "20000000-3fffffff",
    "node_1_10@172.17.0.2"
   ],
   [
    "add",
    "20000000-3fffffff",
    "node_2_10@172.17.0.3"
   ],
   [
    "add",
    "20000000-3fffffff",
    "node_3_10@172.17.0.4"
   ],
   [
    "add",
    "40000000-5fffffff",
    "node_10_10@172.17.0.11"
   ],
   [
    "add",
    "40000000-5fffffff",
    "node_8_10@172.17.0.9"
   ],
   [
    "add",
    "40000000-5fffffff",
    "node_9_10@172.17.0.10"
   ],
   [
    "add",
    "60000000-7fffffff",
    "node_5_10@172.17.0.6"
   ],
   [
    "add",
    "60000000-7fffffff",
    "node_6_10@172.17.0.7"
   ],
   [
    "add",
    "60000000-7fffffff",
    "node_7_10@172.17.0.8"
   ],
   [
    "add",
    "80000000-9fffffff",
    "node_2_10@172.17.0.3"
   ],
   [
    "add",
    "80000000-9fffffff",
    "node_3_10@172.17.0.4"
   ],
   [
    "add",
    "80000000-9fffffff",
    "node_4_10@172.17.0.5"
   ],
   [
    "add",
    "a0000000-bfffffff",
    "node_10_10@172.17.0.11"
   ],
   [
    "add",
    "a0000000-bfffffff",
    "node_1_10@172.17.0.2"
   ],
   [
    "add",
    "a0000000-bfffffff",
    "node_9_10@172.17.0.10"
   ],
   [
    "add",
    "c0000000-dfffffff",
    "node_6_10@172.17.0.7"
   ],
   [
    "add",
    "c0000000-dfffffff",
    "node_7_10@172.17.0.8"
   ],
   [
    "add",
    "c0000000-dfffffff",
    "node_8_10@172.17.0.9"
   ],
   [
    "add",
    "e0000000-ffffffff",
    "node_3_10@172.17.0.4"
   ],
   [
    "add",
    "e0000000-ffffffff",
    "node_4_10@172.17.0.5"
   ],
   [
    "add",
    "e0000000-ffffffff",
    "node_5_10@172.17.0.6"
   ]
  ],
  "by_node": {
   "node_10_10@172.17.0.11": [
    "40000000-5fffffff",
    "a0000000-bfffffff"
   ],
   "node_1_10@172.17.0.2": [
    "20000000-3fffffff",
    "a0000000-bfffffff"
   ],
   "node_2_10@172.17.0.3": [
    "20000000-3fffffff",
    "80000000-9fffffff"
   ],
   "node_3_10@172.17.0.4": [
    "20000000-3fffffff",
    "80000000-9fffffff",
    "e0000000-ffffffff"
   ],
   "node_4_10@172.17.0.5": [
    "00000000-1fffffff",
    "80000000-9fffffff",
    "e0000000-ffffffff"
   ],
   "node_5_10@172.17.0.6": [
    "00000000-1fffffff",
    "60000000-7fffffff",
    "e0000000-ffffffff"
   ],
   "node_6_10@172.17.0.7": [
    "00000000-1fffffff",
    "60000000-7fffffff",
    "c0000000-dfffffff"
   ],
   "node_7_10@172.17.0.8": [
    "60000000-7fffffff",
    "c0000000-dfffffff"
   ],
   "node_8_10@172.17.0.9": [
    "40000000-5fffffff",
    "c0000000-dfffffff"
   ],
   "node_9_10@172.17.0.10": [
    "40000000-5fffffff",
    "a0000000-bfffffff"
   ]
  },
  "by_range": {
   "00000000-1fffffff": [
    "node_4_10@172.17.0.5",
    "node_5_10@172.17.0.6",
    "node_6_10@172.17.0.7"
   ],
   "20000000-3fffffff": [
    "node_1_10@172.17.0.2",
    "node_2_10@172.17.0.3",
    "node_3_10@172.17.0.4"
   ],
   "40000000-5fffffff": [
    "node_10_10@172.17.0.11",
    "node_8_10@172.17.0.9",
    "node_9_10@172.17.0.10"
   ],
   "60000000-7fffffff": [
    "node_5_10@172.17.0.6",
    "node_6_10@172.17.0.7",
    "node_7_10@172.17.0.8"
   ],
   "80000000-9fffffff": [
    "node_2_10@172.17.0.3",
    "node_3_10@172.17.0.4",
    "node_4_10@172.17.0.5"
   ],
   "a0000000-bfffffff": [
    "node_10_10@172.17.0.11",
    "node_1_10@172.17.0.2",
    "node_9_10@172.17.0.10"
   ],
   "c0000000-dfffffff": [
    "node_6_10@172.17.0.7",
    "node_7_10@172.17.0.8",
    "node_8_10@172.17.0.9"
   ],
   "e0000000-ffffffff": [
    "node_3_10@172.17.0.4",
    "node_4_10@172.17.0.5",
    "node_5_10@172.17.0.6"
   ]
  }
 }
}

https://github.com/RipcordSoftware/AvanceDB

Comments

Post a Comment

Popular Posts