· 8 years ago · Jun 14, 2018, 12:42 PM
1ASK [migrate : migrate ch db] 05:44
2failed: [noc-node.ins.ru -> 194.226.214.196] (item=noc-node.ins.ru) => {
3 "changed": true,
4 "cmd": [
5 "./noc",
6 "migrate-ch",
7 "--host",
8 "noc-node.ins.ru"
9 ],
10 "delta": "0:00:01.798939",
11 "end": "2018-06-14 15:29:07.897891",
12 "item": "noc-node.ins.ru",
13 "rc": 2,
14 "start": "2018-06-14 15:29:06.098952"
15}
16
17STDOUT:
18
19Ensuring Dictionary database
20Ensuring database
212018-06-14 15:29:06,468 [noc.core.dcs.base] Resolve near service postgres
222018-06-14 15:29:06,470 [noc.core.dcs.base] Resolved near service postgres to ['194.226.214.196:5432']
232018-06-14 15:29:06,726 [noc.core.dcs.base] Running resolver for service mongo
242018-06-14 15:29:06,728 [noc.core.dcs.base] [mongo] Starting resolver
252018-06-14 15:29:06,730 [noc.core.dcs.base] [mongo] Set active services to: mongo: 194.226.214.196:27017
262018-06-14 15:29:06,732 [noc.lib.nosql] Connecting to MongoDB {'username': 'noc', 'authentication_source': 'noc', 'readPreference': 'secondaryPreferred', 'replicaSet': 'noc', 'db': 'noc', 'socketKeepAlive': True, 'host': 'mongodb://noc:********@194.226.214.196:27017/noc', 'password': '********'}
272018-06-14 15:29:06,809 [noc.core.cache.base] Using cache backend: noc.core.cache.mongo.MongoCache
282018-06-14 15:29:07,271 [noc.core.clickhouse.ensure] Ensuring BI models:
292018-06-14 15:29:07,279 [noc.core.clickhouse.ensure] Ensure table span
302018-06-14 15:29:07,286 [noc.core.clickhouse.ensure] Ensure table mac
312018-06-14 15:29:07,291 [noc.core.clickhouse.ensure] Ensure table alarms
322018-06-14 15:29:07,525 [noc.core.debug] UNHANDLED EXCEPTION (2018-06-14 15:29:07.334230)
33PROCESS: ./commands/migrate-ch.py
34VERSION: 15.05.1+microservices.9927.e9bac9f6
35BRANCH: microservices CHANGESET: e9bac9f6
36ERROR FINGERPRINT: c1dde0f3-2709-5967-9424-806a5dfe89a8
37WORKING DIRECTORY: /opt/noc
38EXCEPTION: <class 'noc.core.clickhouse.error.ClickhouseError'> 500: Code: 44, e.displayText() = DB::Exception: Cannot add column services: column with this name already exists, e.what() = DB::Exception
39
40START OF TRACEBACK
41------------------------------------------------------------------------
42File: core/clickhouse/connect.py (Line: 66)
43Function: execute
44 59 body=post,
45 60 user=self.user,
46 61 password=self.password,
47 62 connect_timeout=config.clickhouse.connect_timeout,
48 63 request_timeout=config.clickhouse.request_timeout
49 64 )
50 65 if code != 200:
51 66 ==> raise ClickhouseError("%s: %s" % (code, body))
52 67 return [
53 68 row.split("\t") for row in body.splitlines()
54 69 ]
55 70
56 71 def ensure_db(self):
57 72 self.execute(
58Variables:
59 body =
60'Code: 44, e.displayText() = DB::Exception: Cannot add column services: column with this name already exists, e.what() = DB::Exception\n'
61 qs = ['database=noc']
62 code = 500
63 nodb = False
64 url = 'http://noc-node.ins.ru:8123/?database=noc'
65 self = <noc.core.clickhouse.connect.ClickhouseClient object at 0x7f05e2498390>
66 args = None
67 q = <function q at 0x7f05d4ae3668>
68 headers =
69IOrderedDict([('Date', 'Thu, 14 Jun 2018 12:29:07 GMT'), ('Connection', 'Close'), ('Content-Type', 'text/plain; charset=UTF-8'), ('X-ClickHouse-Server-Display-Name', 'noc-node.ins.ru'), ('Transfer-Encoding', 'chunked')])
70 sql = None
71 post =
72'ALTER TABLE alarms ADD COLUMN services Nested (\nprofile String,\nsummary UInt32 \n) AFTER y'
73------------------------------------------------------------------------
74File: core/clickhouse/model.py (Line: 161)
75Function: ensure_columns
76 154 after = None
77 155 for f in cls._fields_order:
78 156 if f not in existing:
79 157 ch.execute(
80 158 post="ALTER TABLE %s ADD COLUMN %s AFTER %s" % (
81 159 table_name,
82 160 cls._fields[f].get_create_sql(),
83 161 ==> after)
84 162 )
85 163 c = True
86 164 after = f
87 165 return c
88 166
89 167 changed = False
90Variables:
91 c = False
92 ch = <noc.core.clickhouse.connect.ClickhouseClient object at 0x7f05e2498390>
93 name = 'subscribers.summary'
94 f = 'services'
95 after = 'y'
96 existing =
97{'administrative_domain': 'UInt64',
98 'alarm_class': 'UInt64',
99 'alarm_id': 'String',
100 'close_ts': 'DateTime',
101 'container': 'UInt64',
102 'date': 'Date',
103 'direct_services': 'Int64',
104 'direct_subscribers': 'Int64',
105 'duration': 'Int32',
106 'escalation_ts': 'DateTime',
107 'escalation_tt': 'String',
108 'ip': 'UInt32',
109 'managed_object': 'UInt64',
110 'platform': 'UInt64',
111 'pool': 'UInt64',
112 'profile': 'UInt64',
113 'reboots': 'Int16',
114 'reopens': 'Int32',
115 'root': 'String',
116 'segment': 'UInt64',
117 'services.profile': 'Array(String)',
118 'services.summary': 'Array(UInt32)',
119 'severity': 'Int32',
120 'subscribers.profile': 'Array(String)',
121 'subscribers.summary': 'Array(UInt32)',
122 'total_objects': 'Int64',
123 'total_services': 'Int64',
124 'total_subscribers': 'Int64',
125 'ts': 'DateTime',
126 'vendor': 'UInt64',
127 'version': 'UInt64',
128 'x': 'Float64',
129 'y': 'Float64'}
130 table_name = 'alarms'
131 type = 'Array(UInt32)'
132 cls = <class 'noc.bi.models.alarms.Alarms'>
133------------------------------------------------------------------------
134File: core/clickhouse/model.py (Line: 177)
135Function: ensure_table
136 170 # Create new table
137 171 ch.execute(post=cls.get_create_sql())
138 172 if config.clickhouse.cluster:
139 173 # Create distributed
140 174 ch.execute(post=cls.get_create_sql())
141 175 changed = True
142 176 else:
143 177 ==> changed |= ensure_columns(cls._get_raw_db_table())
144 178 # Check for distributed table
145 179 if config.clickhouse.cluster:
146 180 if not ch.has_table(cls._meta.db_table):
147 181 ch.execute(post=cls.get_create_distributed_sql())
148 182 changed = True
149 183 else:
150Variables:
151 ch = <noc.core.clickhouse.connect.ClickhouseClient object at 0x7f05e2498390>
152 changed = False
153 ensure_columns = <function ensure_columns at 0x7f05d4ad11b8>
154 connect = <noc.core.clickhouse.connect.ClickhouseClient object at 0x7f05e2498390>
155 cls = <class 'noc.bi.models.alarms.Alarms'>
156------------------------------------------------------------------------
157File: core/clickhouse/ensure.py (Line: 35)
158Function: ensure_bi_models
159 28 model = Model.get_model_class(mn)
160 29 if model:
161 30 models.add(model)
162 31 # Ensure fields
163 32 changed = False
164 33 for model in models:
165 34 logger.info("Ensure table %s" % model._meta.db_table)
166 35 ==> changed |= model.ensure_table(connect=connect)
167 36 return changed
168 37
169 38
170 39 def ensure_pm_scopes(connect=None):
171 40 from noc.pm.models.metricscope import MetricScope
172 41 logger.info("Ensuring PM scopes")
173Variables:
174 changed = False
175 f = 'alarms.pyc'
176 models =
177set([<class 'noc.bi.models.mac.MAC'>,
178 <class 'noc.bi.models.reboots.Reboots'>,
179 <class 'noc.bi.models.span.Span'>,
180 <class 'noc.bi.models.managedobjects.ManagedObject'>,
181 <class 'noc.bi.models.alarms.Alarms'>])
182 mn = 'alarms'
183 connect = <noc.core.clickhouse.connect.ClickhouseClient object at 0x7f05e2498390>
184 model = <class 'noc.bi.models.alarms.Alarms'>
185 Model = <class 'noc.core.clickhouse.model.Model'>
186------------------------------------------------------------------------
187File: commands/migrate-ch.py (Line: 37)
188Function: handle
189 30
190 31 def handle(self, host=None, port=None, *args, **options):
191 32 self.host = host or None
192 33 self.port = port or None
193 34 self.connect()
194 35 self.create_dictionaries_db()
195 36 self.ensure_db()
196 37 ==> changed = ensure_bi_models(connect=self.connect)
197 38 changed |= ensure_pm_scopes(connect=self.connect)
198 39 if changed:
199 40 self.print("CHANGED")
200 41 else:
201 42 self.print("OK")
202 43
203Variables:
204 self = <__main__.Command object at 0x7f05e08b03d0>
205 host = 'noc-node.ins.ru'
206 args = ()
207 options = {}
208 port = None
209------------------------------------------------------------------------
210File: core/management/base.py (Line: 76)
211Function: run_from_argv
212 69 enable_profiling = cmd_options.pop("enable_profiling", False)
213 70 show_metrics = cmd_options.pop("show_metrics", False)
214 71 if enable_profiling:
215 72 # Start profiler
216 73 import yappi
217 74 yappi.start()
218 75 try:
219 76 ==> return self.handle(*args, **cmd_options) or 0
220 77 except CommandError as e:
221 78 self.print(str(e))
222 79 return 1
223 80 except KeyboardInterrupt:
224 81 self.print("Ctrl+C")
225 82 return 3
226Variables:
227 parser =
228ArgumentParser(prog='migrate-ch.py', usage=None, description=None, version=None, formatter_class=<class 'argparse.HelpFormatter'>, conflict_handler='error', add_help=True)
229 loglevel = 'info'
230 enable_profiling = False
231 args = ()
232 cmd_options = {'host': 'noc-node.ins.ru', 'port': None}
233 argv = ['--host', 'noc-node.ins.ru']
234 self = <__main__.Command object at 0x7f05e08b03d0>
235 show_metrics = False
236 options = Namespace(host='noc-node.ins.ru', port=None)
237------------------------------------------------------------------------
238END OF TRACEBACK
239
240
241MSG:
242
243non-zero return code