· 4 years ago · Apr 20, 2021, 08:44 AM
1TASK [migrate ch db] 17:42failed: [Node1 -> 10.220.21.122] (item=Node1) => {
2 "ansible_loop_var": "item",
3 "changed": true,
4 "cmd": [
5 "./noc",
6 "migrate-ch",
7 "--host",
8 "Node1"
9 ],
10 "delta": "0:00:09.612474",
11 "end": "2021-04-20 13:35:10.640942",
12 "item": "Node1",
13 "rc": 2,
14 "start": "2021-04-20 13:35:01.028468"
15}
16
17STDOUT:
18
192021-04-20 13:35:01,690 [noc.core.ioloop.util] Setting up asyncio event loop policy
202021-04-20 13:35:01,690 [noc.core.ioloop.util] Setting up default event loop
212021-04-20 13:35:02,847 [noc.core.dcs.base] Resolve near service pgbouncer
222021-04-20 13:35:02,854 [noc.core.dcs.base] Resolved near service pgbouncer to ['10.220.21.122:6432']
232021-04-20 13:35:04,024 [py.warnings] /opt/noc/core/profile/base.py:72: UserWarning: None: 'pattern_more' '^---MORE---' pattern must be of binary type. Support for text values will be removed in NOC 20.2
24 warnings.warn(
25
262021-04-20 13:35:05,657 [noc.core.cache.base] Using cache backend: noc.core.cache.memcached.MemcachedCache
272021-04-20 13:35:05,809 [noc.core.dcs.base] [memcached] Starting resolver (near=False)
282021-04-20 13:35:05,813 [noc.core.dcs.base] [memcached] Set active services to: memcached: 10.220.21.122:11211
292021-04-20 13:35:05,814 [noc.core.dcs.base] [memcached] Stopping resolver
302021-04-20 13:35:07,870 [noc.core.dcs.base] [mongo] Starting resolver (near=False)
312021-04-20 13:35:07,876 [noc.core.dcs.base] [mongo] Set active services to: mongo: 10.220.21.122:27017
322021-04-20 13:35:07,876 [noc.core.dcs.base] [mongo] Stopping resolver
332021-04-20 13:35:07,877 [noc.core.mongo.connection] Connecting to MongoDB {'db': 'noc', 'username': 'noc', 'password': '********', 'authentication_source': 'noc', 'replicaSet': 'noc', 'readPreference': 'secondaryPreferred', 'maxIdleTimeMS': 60000, 'host': 'mongodb://noc:********@10.220.21.122:27017/noc'}
34Ensuring Dictionary database
35Ensuring database
362021-04-20 13:35:07,909 [noc.core.clickhouse.ensure] Ensuring BI models:
372021-04-20 13:35:07,909 [noc.core.loader.base] [bi] Loading alarms
382021-04-20 13:35:07,984 [noc.core.clickhouse.ensure] Ensure table alarms
392021-04-20 13:35:08,152 [noc.core.loader.base] [bi] Loading mac
402021-04-20 13:35:08,171 [noc.core.clickhouse.ensure] Ensure table mac
412021-04-20 13:35:08,231 [noc.core.loader.base] [bi] Loading managedobjects
422021-04-20 13:35:08,245 [noc.core.clickhouse.ensure] Ensure table managedobjects
432021-04-20 13:35:08,299 [noc.core.loader.base] [bi] Loading netflow
442021-04-20 13:35:08,315 [noc.core.loader.base] bi not found: netflow
452021-04-20 13:35:08,315 [noc.core.loader.base] [bi] Loading reboots
462021-04-20 13:35:08,327 [noc.core.clickhouse.ensure] Ensure table reboots
472021-04-20 13:35:08,378 [noc.core.loader.base] [bi] Loading routingneighbors
482021-04-20 13:35:08,396 [noc.core.clickhouse.ensure] Ensure table routingneighbors
492021-04-20 13:35:08,435 [noc.core.loader.base] [bi] Loading span
502021-04-20 13:35:08,445 [noc.core.clickhouse.ensure] Ensure table span
512021-04-20 13:35:08,488 [noc.core.loader.base] [bi] Loading syslog
522021-04-20 13:35:08,500 [noc.core.clickhouse.ensure] Ensure table syslog
532021-04-20 13:35:08,543 [noc.core.clickhouse.ensure] Ensuring PM scopes
542021-04-20 13:35:08,576 [noc.core.clickhouse.ensure] Ensure scope sla
552021-04-20 13:35:08,690 [noc.core.clickhouse.ensure] Ensure scope memory
562021-04-20 13:35:08,821 [noc.core.clickhouse.ensure] Ensure scope interface
572021-04-20 13:35:08,914 [noc.core.clickhouse.ensure] Ensure scope cpu
582021-04-20 13:35:10,227 [noc.core.debug] UNHANDLED EXCEPTION (2021-04-20 13:35:09.151690)
59PROCESS: ./commands/migrate-ch.py
60VERSION: 20.4+master.913.24347537
61BRANCH: master CHANGESET: 24347537
62ERROR FINGERPRINT: d41895f4-42e3-54c3-a824-2b7dc1a184fc
63WORKING DIRECTORY: /opt/noc
64EXCEPTION: <class 'noc.core.clickhouse.error.ClickhouseError'> 500: b'Code: 44, e.displayText() = DB::Exception: Cannot add column usage: column with this name already exists (version 21.2.5.5 (official build))\n'
65START OF TRACEBACK
66------------------------------------------------------------------------
67File: core/clickhouse/connect.py (Line: 66)
68Function: execute
69 59 body=post,
70 60 user=self.user,
71 61 password=self.password,
72 62 connect_timeout=config.clickhouse.connect_timeout,
73 63 request_timeout=config.clickhouse.request_timeout,
74 64 )
75 65 if code != 200:
76 66 ==> raise ClickhouseError("%s: %s" % (code, body))
77 67 return [smart_text(row).split("\t") for row in body.splitlines()]
78 68
79 69 def ensure_db(self, db_name=None):
80 70 self.execute(
81 71 post=f"CREATE DATABASE IF NOT EXISTS {db_name or config.clickhouse.db};", nodb=True
82 72 )
83Variables:
84 self = <noc.core.clickhouse.connect.ClickhouseClient object at 0x7f0f275025e0>
85 sql = None
86 args = None
87 nodb = False
88 post =
89('CREATE TABLE IF NOT EXISTS raw_cpu (\n'
90 ' date Date ,\n'
91 ' ts DateTime ,\n'
92 ' metric_type String ,\n'
93 ' managed_object UInt64 ,\n'
94 ' object UInt64 ,\n'
95 ' labels Array(LowCardinality(String)) ,\n'
96 " name LowCardinality(String) MATERIALIZED splitByString('::', arrayFirst(x "
97 "-> startsWith(x, 'noc::cpu::'), labels))[-1] ,\n"
98 ' load_5s UInt8 ,\n'
99 ' load_1min UInt8 ,\n'
100 ' usage UInt8 ,\n'
101 ' load_5min UInt8 ,\n'
102 ' usage UInt8 \n'
103 ') ENGINE = MergeTree() ORDER BY (managed_object, object, date, arrayFirst(x '
104 "-> startsWith(x, 'noc::cpu::'), labels))\n"
105 '\n'
106 'PARTITION BY toYYYYMM(date) PRIMARY KEY (managed_object, object, date)')
107 extra = None
108 qs = ['database=noc']
109 url = 'http://Node1:8123/?database=noc'
110 code = 500
111 headers =
112{'Connection': 'Close',
113 'Content-Type': 'text/plain; charset=UTF-8',
114 'Date': 'Tue, 20 Apr 2021 08:35:08 GMT',
115 'Transfer-Encoding': 'chunked',
116 'X-ClickHouse-Exception-Code': '44',
117 'X-ClickHouse-Server-Display-Name': 'node1.knd-oil.ru',
118 'X-ClickHouse-Summary': '{"read_rows":"0","read_bytes":"0","written_rows":"0","written_bytes":"0","total_rows_to_read":"0"}'}
119 body =
120(b'Code: 44, e.displayText() = DB::Exception: Cannot add column usage: column w'
121 b'ith this name already exists (version 21.2.5.5 (official build))\n')
122 q = <function ClickhouseClient.execute.<locals>.q at 0x7f0ef406f820>
123------------------------------------------------------------------------
124File: pm/models/metricscope.py (Line: 355)
125Function: ensure_table
126 348 ch.rename_table(raw_table, f"{OLD_PM_SCHEMA_TABLE}.{raw_table}")
127 349 # Ensure raw_* table
128 350 if ch.has_table(raw_table):
129 351 # raw_* table exists, check columns
130 352 changed |= ensure_columns(raw_table)
131 353 else:
132 354 # Create new table
133 355 ==> ch.execute(post=self.get_create_sql())
134 356 changed = True
135 357 # For cluster mode check d_* distributed table
136 358 if is_cluster:
137 359 if ch.has_table(dist_table):
138 360 changed |= ensure_columns(dist_table)
139 361 else:
140Variables:
141 connect = <noc.core.clickhouse.connect.ClickhouseClient object at 0x7f0f275025e0>
142 connection = <function connection at 0x7f0f20d52dc0>
143 ensure_column =
144<function MetricScope.ensure_table.<locals>.ensure_column at 0x7f0ef3fcea60>
145 ensure_columns =
146<function MetricScope.ensure_table.<locals>.ensure_columns at 0x7f0ef406f3a0>
147 changed = True
148 is_cluster = False
149 table = 'cpu'
150 raw_table = 'raw_cpu'
151 dist_table = 'd_cpu'
152 ch = <noc.core.clickhouse.connect.ClickhouseClient object at 0x7f0f275025e0>
153 self = <MetricScope: CPU>
154------------------------------------------------------------------------
155File: core/clickhouse/ensure.py (Line: 38)
156Function: ensure_pm_scopes
157 31 def ensure_pm_scopes(connect=None):
158 32 from noc.pm.models.metricscope import MetricScope
159 33
160 34 logger.info("Ensuring PM scopes")
161 35 changed = False
162 36 for s in MetricScope.objects.all():
163 37 logger.info("Ensure scope %s" % s.table_name)
164 38 ==> changed |= s.ensure_table(connect=connect)
165 39 return changed
166 40
167 41
168 42 def ensure_all_pm_scopes():
169 43 from noc.core.clickhouse.connect import connection
170 44
171Variables:
172 connect = <noc.core.clickhouse.connect.ClickhouseClient object at 0x7f0f275025e0>
173 MetricScope = <class 'noc.pm.models.metricscope.MetricScope'>
174 changed = True
175 s = <MetricScope: CPU>
176------------------------------------------------------------------------
177File: commands/migrate-ch.py (Line: 28)
178Function: handle
179 21 self.host = host or None
180 22 self.port = port or None
181 23 self.connect()
182 24 mongo_connect()
183 25 self.create_dictionaries_db()
184 26 self.ensure_db()
185 27 changed = ensure_bi_models(connect=self.connect)
186 28 ==> changed |= ensure_pm_scopes(connect=self.connect)
187 29 if changed:
188 30 self.print("CHANGED")
189 31 else:
190 32 self.print("OK")
191 33
192 34 def connect(self):
193Variables:
194 self = <__main__.Command object at 0x7f0f27502a90>
195 host = 'Node1'
196 port = None
197 args = ()
198 options = {}
199 changed = True
200------------------------------------------------------------------------
201File: core/management/base.py (Line: 78)
202Function: run_from_argv
203 71 self.no_progressbar = cmd_options.pop("no_progressbar", False)
204 72 if enable_profiling:
205 73 # Start profiler
206 74 import yappi
207 75
208 76 yappi.start()
209 77 try:
210 78 ==> return self.handle(*args, **cmd_options) or 0
211 79 except CommandError as e:
212 80 self.print(str(e))
213 81 return 1
214 82 except KeyboardInterrupt:
215 83 self.print("Ctrl+C")
216 84 return 3
217Variables:
218 self = <__main__.Command object at 0x7f0f27502a90>
219 argv = ['--host', 'Node1']
220 parser =
221ArgumentParser(prog='noc migrate-ch', usage=None, description=None, formatter_class=<class 'argparse.HelpFormatter'>, conflict_handler='error', add_help=True)
222 options = Namespace(host='Node1', port=None)
223 cmd_options = {'host': 'Node1', 'port': None}
224 args = ()
225 loglevel = 'info'
226 enable_profiling = False
227 show_metrics = False
228 error_report = <function error_report at 0x7f0f20762040>
229------------------------------------------------------------------------
230END OF TRACEBACK
2312021-04-20 13:35:10,249 [noc.core.debug] Writing CP report to /var/lib/noc/cp/crashinfo/new/d41895f4-42e3-54c3-a824-2b7dc1a184fc.json
232
233
234MSG:
235
236non-zero return code