T4stUzuh

1#!/usr/bin/env bash
2
3TABLES=$(clickhouse-client -q "SELECT DISTINCT name FROM system.tables WHERE database = 'replica'")
4PROCESSED=$(cat processed_tables)
5
6check_all_data_moved() {
7  local table=$1
8  nrows_old=$(clickhouse-client -q "SELECT sum(rows) FROM system.parts WHERE database = 'replica' AND table = '$table';")
9  nrows_new=$(clickhouse-client -q "SELECT count() FROM replica.${table}_new;")
10  echo "nrows_old: $nrows_old, nrows_new: $nrows_new, diff: $(( nrows_old - nrows_new ))"
11  if [[ "$nrows_old" == "$nrows_new" ]]; then return 0; fi
12  return 1
13}
14
15create_table() {
16  local table=$1
17  query_old=$(clickhouse-client -q "SHOW CREATE TABLE replica.$table")
18  query_new=${query_old//$table/"$table"_new}
19  query_new=${query_new//'CREATE TABLE'/'CREATE TABLE IF NOT EXISTS'}
20  query_new=${query_new//\\n/ }
21  query_new=${query_new//\\/}
22  clickhouse-client -q "DROP TABLE IF EXISTS replica.${table}_new"
23  sleep 5
24  clickhouse-client -q "$query_new"
25}
26
27move_data() {
28  local table=$1
29  clickhouse-client --format=TSVRaw -q "SELECT DISTINCT 'ALTER TABLE replica.${table}_new ATTACH PARTITION ' || partition_id || ' FROM replica.${table};\n' FROM system.parts WHERE database = 'replica' AND table = '$table' ORDER BY partition_id" | clickhouse-client --echo -mn
30}
31
32clear_old_table() {
33  sudo rm -r /var/lib/clickhouse/data/replica/$1
34  sudo rm /var/lib/clickhouse/metadata/replica/$1.sql
35}
36
37drop_new_table() {
38  local table=$1
39  clickhouse-client --format=TSVRaw -q "SELECT DISTINCT 'ALTER TABLE replica.${table}_new DROP PARTITION ' || partition_id || ';\n' FROM system.parts WHERE database = 'replica' AND table = '${table}_new' ORDER BY partition_id" | clickhouse-client -mn
40  clickhouse-client -q "DROP TABLE replica.${table}_new"
41}
42
43process_table() {
44  local table=$1
45  IFS='_' read -r -a array <<< "$table"
46  if [[ "${array[-1]}" == "old" ]] || [[ "${array[-1]}" == "new" ]]; then continue; fi
47  if [[ "$PROCESSED" =~ "${table}" ]]; then continue; fi
48  echo $table
49  create_table $table
50  sleep 5
51  replica_number=$(cat /etc/hostname | cut -d'-' -f4)
52  if [[ "$replica_number" == "r1" ]]; then
53    move_data $table
54  else
55    while [[ $(check_all_data_moved $table) -ne 0 ]]; do sleep 5; done
56  fi
57  if check_all_data_moved $table
58  then
59    echo $table >> processed_tables
60    clear_old_table $table
61  else
62    drop_new_table $table
63  fi
64}
65
66
67process_all() {
68  for table in $TABLES; do
69    process_table $table
70  done
71}
72
73"$@"