· 6 years ago · Oct 03, 2019, 01:48 AM
1# -*- coding: utf-8 -*-
2"""Retrieve candle data.
3
4For complete specs of the endpoint, please check:
5
6 http://developer.oanda.com/rest-live-v20/instrument-ep/
7
8Specs of InstrumentsCandles()
9
10 http://oanda-api-v20.readthedocs.io/en/latest/oandapyV20.endpoints.html
11
12"""
13from indicators import *
14import numpy as np
15from numpy.random import normal
16from numpy import genfromtxt
17import datetime as dt
18import time
19import random
20import csv
21import os
22import sys
23import psutil
24import ray
25import collections
26import argparse
27import json
28from oandapyV20 import API
29from oandapyV20.exceptions import V20Error
30import oandapyV20.endpoints.instruments as instruments
31from oandapyV20.definitions.instruments import CandlestickGranularity
32from exampleauth import exampleAuth
33import re
34import math
35
36np.set_printoptions(suppress=True)
37num_cpus = psutil.cpu_count(logical=True)
38ray.init(num_cpus=num_cpus)
39
40
41def check_date(s):
42 dateFmt = "[\d]{4}-[\d]{2}-[\d]{2}T[\d]{2}:[\d]{2}:[\d]{2}Z"
43 if not re.match(dateFmt, s):
44 raise ValueError("Incorrect date format: ", s)
45
46 return True
47
48
49def erase_files(directory, folder_name):
50 for the_file in os.listdir(directory+folder_name):
51 file_path = os.path.join(directory+folder_name, the_file)
52 if os.path.isfile(file_path):
53 os.unlink(file_path)
54
55
56def using_indicators(bars, params, period):
57 # start = time.time()
58 atr = talib.ATR(bars['high'].values, bars['low'].values,
59 bars['close'].values, timeperiod=14)
60
61 bulls, bears = ash(bars['high'].values, bars['low'].values,
62 bars['close'].values, 0, paramsFixed[period-1,0], paramsFixed[period-1,1], 0, 3)
63 HVI = hawkeye(bars, params[0])
64 # KS = kijun_sen(bars, params[3], params[3])
65
66 # real = talib.ATR(bars['high'].values, bars['low'].values, bars['close'].values, timeperiod=params[0])
67 """ real =williamsR(bars['high'].values, bars['low'].values, bars['close'].values, params[0])
68 feixe_inf = np.full((len(bars['high'])), ((-params[1]/2)-50))
69 feixe_sup = np.full((len(bars['high'])), ((params[1]/2)-50)) """
70
71 """ sar = talib.SAR(bars['high'].values, bars['low'].values, acceleration=params[0], maximum=params[1])
72 emafast = talib.EMA(bars['close'].values, timeperiod=params[2])
73 emaslow = talib.EMA(bars['close'].values, timeperiod=params[3])
74 macd, macd_sign, macd_hist = talib.MACD(
75 bars['close'].values, fastperiod=params[5], slowperiod=params[6], signalperiod=params[4])
76 rsi = talib.RSI(bars['close'].values, timeperiod=params[7])
77 upper, mid, lower = talib.BBANDS(
78 bars['close'].values, timeperiod=params[8], nbdevup=2, nbdevdn=2, matype=0)
79 slowk, slowd = talib.STOCH(bars['high'].values, bars['low'].values, bars['close'].values, fastk_period=params[9], slowk_period=params[10], slowk_matype=0, slowd_period=params[11], slowd_matype=0) """
80
81 # indicators = {'atr':atr,'real':real, 'feixe_inf':feixe_inf,'feixe_sup':feixe_sup}
82 # indicators = {'atr':atr,'rsi':rsi,'emafast':emafast,'emaslow':emaslow,'sar':sar, 'macd_hist':macd_hist,'slowk':slowk,'slowd':slowd,'upper':upper,'mid':mid,'lower':lower,'bulls':bulls,'bears':bears}
83 # indicators = {'atr':atr, 'bulls':bulls,'bears':bears}
84 indicators = {'atr': atr, 'bulls': bulls, 'bears': bears,'HVI_color': HVI[0], 'HVI_volume': HVI[1]}
85 # indicators = {'atr':atr, 'bulls':bulls,'bears':bears, 'HVI_color': HVI[0], 'HVI_volume': HVI[1], 'KS': KS}
86 # indicators = {'atr':atr, 'bulls':bulls,'bears':bears, 'real':real, 'feixe_inf':feixe_inf,'feixe_sup':feixe_sup}
87
88 # end = time.time()
89 # print('indicators: ' + str(end-start))
90 # print('--------------')
91 return indicators
92# //////////////////// fim using_indicators
93
94
95def condition_trade(bars, indicators):
96 # start = time.time()
97 # williams
98 """ buy = crossed_above(indicators['real'], indicators['feixe_inf'])
99 sell = crossed_below(indicators['real'], indicators['feixe_sup']) """
100
101 # 150 doleta
102 """ buy = crossed_above(indicators['bulls'], indicators['bears']) & (indicators['macd_hist'] > 0) & (indicators['rsi'] > 50) & (bars['close'].values > indicators['emafast']) & (
103 bars['close'].values > indicators['emaslow']) & (bars['close'].values > indicators['mid']) & (indicators['sar'] < bars['low'].values) & (indicators['slowk'] > indicators['slowd'])
104 sell = crossed_below(indicators['bulls'], indicators['bears']) & (indicators['macd_hist'] < 0) & (indicators['rsi'] < 50) & (bars['close'].values < indicators['emafast']) & (
105 bars['close'].values < indicators['emaslow']) & (bars['close'].values < indicators['mid']) & (indicators['sar'] > bars['high'].values) & (indicators['slowk'] < indicators['slowd']) """
106
107 # ash hvi e KS
108 """ buy = crossed_above(indicators['bulls'], indicators['bears']) & ((indicators['HVI_color'] == 'green') | (indicators['HVI_color']== 'gray') | (indicators['HVI_color'] == 'blue')) & (bars['volume'].values > indicators['HVI_volume']) & (bars['close'].values > indicators['KS'])
109 sell = crossed_below(indicators['bulls'], indicators['bears']) & ((indicators['HVI_color'] == 'red') | (indicators['HVI_color'] == 'gray') | (indicators['HVI_color'] == 'blue')) & (bars['volume'].values > indicators['HVI_volume']) & (bars['close'].values < indicators['KS']) """
110
111 # ash hvi e KS
112 """ buy = crossed_above(indicators['bulls'], indicators['bears']) & ((indicators['HVI_color'] == 'green') | (
113 indicators['HVI_color'] == 'gray') | (indicators['HVI_color'] == 'blue')) & (bars['volume'].values > indicators['HVI_volume'])
114 sell = crossed_below(indicators['bulls'], indicators['bears']) & ((indicators['HVI_color'] == 'red') | (
115 indicators['HVI_color'] == 'gray') | (indicators['HVI_color'] == 'blue')) & (bars['volume'].values > indicators['HVI_volume']) """
116
117
118 #ash puro
119 buy = crossed_above(indicators['bulls'], indicators['bears'])
120 sell = crossed_below(indicators['bulls'], indicators['bears'])
121
122 """ #ash puro
123 buy = crossed_above(indicators['bulls'], indicators['bears']) & (indicators['real'] > indicators['feixe_sup'])
124 sell = crossed_below(indicators['bulls'], indicators['bears']) & (indicators['real'] < indicators['feixe_inf']) """
125 # end = time.time()
126 # print('condition: ' + str(end-start))
127 # print('--------------')
128
129 return buy, sell
130# //////////////////// fim condition_trade
131
132
133def on_calc_move(m, results_temp, validation, directory, atr, fator_pips, per_composto, period, mesInicial, ano_op, currency, cStreak):
134 global bars
135
136 data = np.array([m['data_ent'], m['data_saida'], m['action'], m['v_ent'], m['v_saida'], m['lucro']], dtype=object)
137 results_temp.append(data)
138 if validation:
139 signal = '1' if m['lucro'] >= 0 else '0'
140 date = m['data_saida']
141
142 if lstreak:
143 if signal == '0':
144 for cur in cStreak:
145 if date in bars[cur].index.values:
146 idx = bars[cur].index.get_loc(date)
147 else:
148 continue
149 for i in range(40):
150 bars[cur].iloc[idx+i+1,bars[cur].columns.get_loc('losses')] = bars[cur].iloc[idx+i+1,bars[cur].columns.get_loc('losses')] + 1 if bars[cur].iloc[idx+i+1,bars[cur].columns.get_loc('losses')] < 5 else bars[cur].iloc[idx+i+1,bars[cur].columns.get_loc('losses')]
151 #bars[cur].iloc[idx:idx+nCandlesV]['wins'] = 0
152 else:
153 for cur in cStreak:
154 if date in bars[cur].index.values:
155 idx = bars[cur].index.get_loc(date)
156 else:
157 continue
158 for i in range(40):
159 bars[cur].iloc[idx+i+1,bars[cur].columns.get_loc('losses')] = bars[cur].iloc[idx+i+1,bars[cur].columns.get_loc('losses')] - 1 if bars[cur].iloc[idx+i+1,bars[cur].columns.get_loc('losses')] > 0 else 0
160 #bars[cur].iloc[idx:idx+nCandlesV]['wins'] = 0
161
162
163
164 with open(directory+'/moves/base_validation_'+strat+'period'+str(period)+'mes'+str(mesInicial)+'-'+str(ano_op)+'.csv', 'a') as filedata:
165 writer = csv.writer(filedata, delimiter=',')
166 # DATA , TIPO, VALOR ENT, VALOR SAIDA, LUCRO, PRECISION TP HOLD
167 data = [period, m['data_ent'], m['data_saida'], m['action'], m['v_ent'], m['v_saida'], m['lucro'], atr, signal]
168 writer.writerow(data)
169 else:
170 per_composto = (per_composto * risco / ((atr*fator_pips)*c_prejATR)) * (m['lucro'] * fator_pips) + per_composto
171 return results_temp
172# //////////////////// fim on_calc_move
173
174
175def trade_exit(bars_temp, currency, indicators, c_buy, c_sell, results_temp, validation, directory, fator_pips, per_composto, period, mesInicial, ano_op):
176 buys = np.where(c_buy == True)[0]
177
178 cBase = currency[:3]
179 cDev = currency[3:]
180
181 cStreak = []
182 if validation:
183 cStreak = [key for key, value in bars.items() if cBase in key or cDev in key]
184
185 # start = time.time()
186 # [0] Open - [1] High - [2] Low - [3] Close - [4] Data - [5] ATR
187 dates = bars_temp.index.values
188 bars_temp = np.array([bars_temp['open'].values, bars_temp['high'].values, bars_temp['low'].values, bars_temp['close'].values, dates, indicators['atr']], dtype=object)
189 for i in buys:
190 # shift até o primeiro buy e assim, um loop entre todos os outros buys
191 if (i + 2) >= len(bars_temp[0]) or i == 0:
192 continue
193 m = {'action':'buy', 'v_ent':bars_temp[3][i], 'v_saida_low':bars_temp[2][i+1], 'v_saida_high':bars_temp[1][i+1], 'data_ent':bars_temp[4][i]}
194 # shift para o primeiro candle apos
195 for j in range(i+1,len(bars_temp[0])-1):
196 if ((m['v_saida_low'] - m['v_ent']) <= -(bars_temp[5][i] * c_prejATR)):
197 m['v_saida'] = -bars_temp[5][i]*c_prejATR
198 break
199 elif ((m['v_saida_high'] - m['v_ent']) >= bars_temp[5][i] * c_lucATR):
200 m['v_saida'] = (bars_temp[5][i] * c_lucATR)
201 break
202 else:
203 if j+1 < len(bars_temp[0][j:]):
204 m['v_saida_low'] = bars_temp[2][j+1]
205 m['v_saida_high'] = bars_temp[1][j+1]
206 else:
207 m['v_saida'] = m['v_ent']
208 m['v_ent'] = False
209 break
210
211 m['data_saida'] = bars_temp[4][j]
212 m['lucro'] = m['v_saida']
213 if m['v_ent']:
214 results_temp = on_calc_move(m, results_temp, validation, directory, bars_temp[5][i], fator_pips, per_composto, period, mesInicial, ano_op, currency, cStreak)
215
216 sells = np.where(c_sell == True)[0]
217 for i in sells:
218 # shift até o primeiro buy e assim, um loop entre todos os outros buys
219 if (i + 2) >= len(bars_temp[0]) or i == 0:
220 continue
221 m = {'action':'sell', 'v_ent':bars_temp[3][i], 'v_saida_low':bars_temp[2][i+1], 'v_saida_high':bars_temp[1][i+1], 'data_ent':bars_temp[4][i]}
222 # shift para o primeiro candle apos
223 for j in range(i+1,len(bars_temp[0])-1):
224 if ((m['v_ent'] - m['v_saida_high']) <= -(bars_temp[5][i] * c_prejATR)):
225 m['v_saida'] = -bars_temp[5][i]*c_prejATR
226 break
227 elif ((m['v_ent'] - m['v_saida_low']) >= bars_temp[5][i] * c_lucATR):
228 m['v_saida'] = (bars_temp[5][i] * c_lucATR)
229 break
230 else:
231 if j+1 < len(bars_temp[0][j:]):
232 m['v_saida_low'] = bars_temp[2][j+1]
233 m['v_saida_high'] = bars_temp[1][j+1]
234 else:
235 m['v_saida'] = m['v_ent']
236 m['v_ent'] = False
237 break
238
239 m['data_saida'] = bars_temp[4][j]
240 m['lucro'] = m['v_saida']
241 if m['v_ent']:
242 results_temp = on_calc_move(m, results_temp, validation, directory, bars_temp[5][i], fator_pips, per_composto, period, mesInicial, ano_op, currency, cStreak)
243 #end = time.time()
244 # print('trade exit: ' + str(end-start))
245 # print('--------------')
246 return results_temp
247# //////////////////// fim trade_exit
248
249
250def on_evaluate(params, bars_temp, currency, per_composto, period, mesInicial, ano_op, validation=False):
251 indicators = using_indicators(bars_temp[currency], params, period)
252 directory = pasta_raiz+'/'+currency
253 results_temp = []
254
255 if 'jpy' in currency:
256 fator_pips = 100
257 else:
258 fator_pips = 10000
259
260 safe_i = nCandlesSafeStart
261 # variavel safe de inicio do laço de repetição dos preços
262 j = 1
263
264 # /////////////////////////////// INICIO DO DRIP //////////////////////////////////////////////////
265 # Corte dos bars iniciais que precisam de load
266 bars_temp[currency] = bars_temp[currency].iloc[safe_i+j:]
267 for key in indicators:
268 indicators[key] = indicators[key][safe_i+j:] if validation == False else indicators[key][safe_i+j:safe_i+j+nCandlesV]
269
270 # estrategias de trade
271
272 [c_buy, c_sell] = condition_trade(bars_temp[currency].iloc[:nCandlesV] if validation==True else bars_temp[currency], indicators)
273
274 # estrategias de saída
275 results_temp = trade_exit(bars_temp[currency], currency, indicators, c_buy, c_sell, results_temp, validation, directory, fator_pips, per_composto, period, mesInicial, ano_op)
276
277 # calculo dos moves
278 results_temp = np.asarray(results_temp)
279 if(len(results_temp) > 0):
280 target = results_temp[:, 5]
281 # total
282 total = len(target)
283 # lucro
284 lucro_total = sum(target)*fator_pips
285 # negativos
286 negatives = sum(1 for i in target if i < 0)
287 # positivos
288 positives = int(total) - negatives
289 # pernegatives
290 per_positives = positives/len(target)*100
291 else:
292 target = total = lucro_total = negatives = positives = per_positives = 0
293
294 return np.array([currency, lucro_total, positives, negatives, total, per_positives, per_composto], dtype=object)
295# //////////////////// fim on_evaluate
296
297#@ray.remote
298def fitness_function(bars_temp, earnings, params, news_currencies, period, mesInicial, ano_op, validation=False):
299 results = []
300
301
302 # 0 open - 1 close - 2 high - 3 low
303 mvalidation = minicial = high = low = earnings[1]
304 per_composto = float(minicial)
305 # start = time.time()
306 for currency in currencies:
307 # pego os candles de teste ou validacao pra fazer o evaluate
308 result_currency = on_evaluate(params, bars_temp, currency, per_composto, period, mesInicial, ano_op, validation)
309 per_composto = result_currency[6]
310 results.append(result_currency)
311
312 # verifico quais moedas triggaram o atr e faco um metodo para cancelar as operacoes
313 # end = time.time()
314 # print(end-start)
315 if(validation):
316 global bars
317 for currency in currencies:
318 fileNameBase = pasta_raiz+'/'+currency+'/moves/base_validation_'+strat+'period'+str(period)+'mes'+str(mesInicial)+'-'+str(ano_op)+'.csv'
319 fileNameCuts = pasta_raiz+'/'+currency+'/moves/cuts_validation_'+strat+'period'+str(period)+'mes'+str(mesInicial)+'-'+str(ano_op)+'.csv'
320
321 if 'jpy' in currency:
322 fator_pips = 100
323 else:
324 fator_pips = 10000
325 bars_temp[currency] = bars[currency].iloc[(nCandlesT-nCandlesSafeStart):nCandlesT+nCandlesV+nCandlesSafeT]
326 bars_temp[currency] = bars_temp[currency].iloc[nCandlesSafeStart+1:nCandlesSafeStart+1+nCandlesV]
327
328 if lstreak:
329 for d in range(nCandlesV):
330 # CONTROLE DE LOSE STREAK
331 if(bars_temp[currency].iloc[d]['losses'] >= 5):
332 try:
333 df = pd.read_csv(fileNameBase, header=None)
334 except:
335 df = pd.DataFrame()
336 if not df.empty:
337 with open(fileNameBase, "r") as f:
338 data = list(csv.reader(f))
339
340 with open(fileNameBase, "w") as f:
341 writer = csv.writer(f)
342 for row in data:
343 if row[1] != bars_temp[currency].iloc[d].name:
344 writer.writerow(row)
345 else:
346 with open(fileNameCuts, 'a') as filedata:
347 writer_cut = csv.writer(filedata)
348 writer_cut.writerow(row)
349
350 try:
351 dados = np.loadtxt(fileNameBase, delimiter=',', dtype=object)
352 except IOError:
353 dados = lucro_total = positives = negatives = total = per_positives = 0
354
355
356 if type(dados) is not int:
357 if dados.size == 0:
358 dados = lucro_total = positives = negatives = total = per_positives = 0
359 else:
360 dados = np.asarray([dados]) if dados.ndim == 1 else dados
361 target = dados[:, 6].astype(float)
362 # total
363 total = len(target)
364 # lucro
365 lucro_total = sum(target)*fator_pips
366 # negativos
367 negatives = sum(1 for i in target if i < 0)
368 # positivos
369 positives = int(total) - negatives
370 # pernegatives
371 per_positives = positives/len(target)*100
372 # acerto_total = (positives/total)*100
373
374 for z in range(len(target)):
375 mvalidation = (mvalidation * risco / ((float(dados[z,7])*fator_pips)*1.5)) * (float(dados[z,6]) * fator_pips) + mvalidation
376 low = mvalidation if mvalidation < low else low
377 high = mvalidation if mvalidation > high else high
378
379
380 with open ('registry/allmoves_'+strat+'.csv', 'a') as filedata:
381 writer = csv.writer(filedata, delimiter=',')
382 # DATA , TIPO, VALOR ENT, VALOR SAIDA, LUCRO, PRECISION TP HOLD
383 for row in dados:
384 data = [str(row[0]),str(currency),str(row[1]),str(row[2]),str(row[3]),str(row[4]),str(row[5]),str(row[6]),str(row[7]),str(row[8])]
385 writer.writerow(data)
386
387
388 with open ('registry/periods/csvzao_validation_'+strat+'period'+str(period)+'mes'+str(mesInicial)+'-'+str(ano_op)+'.csv', 'a') as filedata:
389 writer = csv.writer(filedata, delimiter=',')
390 # DATA , TIPO, VALOR ENT, VALOR SAIDA, LUCRO, PRECISION TP HOLD
391 data = [currency, lucro_total, positives, negatives, total, per_positives, mvalidation]
392 writer.writerow(data)
393 per_composto = (mvalidation - minicial) / minicial
394 else:
395 per_composto = (per_composto - minicial) / minicial
396
397 results = np.asarray(results)
398 earnings = [minicial, mvalidation, high, low]
399
400 lucro_total = sum(results[:,1])
401 qtd_operacao = sum(results[:,4])
402 winrate = ((sum(results[:,2])/qtd_operacao) * 100) if qtd_operacao > 0 else 0
403
404 peso1 = per_composto if per_composto > 0 else per_composto*1.5
405 peso1 = peso1 * 2 if winrate > 60 else 0
406 peso2 = (winrate/55) if winrate > 60 else (winrate/45) if winrate > 70 else -(100/(winrate+0.1))
407 # peso3 = sharpe_ratio*5 if peso2 > 0 and sharpe_ratio > 0 else 0
408 peso2 = peso2 + (qtd_operacao/(100)) if winrate > 69 else peso2
409 # peso3 = qtd_operacao/1000
410
411
412 fitness = sum([peso2])
413 # print('\n','%.2f' % fitness, ' ', params, 'Perc Total: ', '%.2f' % per_composto, 'Lucro Total: ', '%.2f' % lucro_total, 'Winrate: ', '%.2f' % winrate, 'Qtd Op: ', qtd_operacao, 'Sharpe Ratio: ', '%.2f' % sharpe_ratio)
414 return fitness, winrate, qtd_operacao, lucro_total, per_composto, earnings
415# //////////////////// fim fitness_function
416
417
418#######################################################################################################################
419""" -------------------------------------------- PARAMS ----------------------------------------------------- """
420#######################################################################################################################
421candle = 'H4'
422start = '2008-01-01T00:00:00Z'
423end = '2019-09-20T00:00:00Z'
424strat = "oanda_h4_ash"
425weight = 1.2
426c1 = 2.29618
427c2 = 2.29618
428nRuns = 1
429nCandlesT = 312*5
430nCandlesV = 80
431nCandlesSafeStart = 150
432nCandlesSafeT = 40
433minicial = 10000
434n_iterations = 1
435n_particles = 2
436risco = 0.02
437c_prejATR = 1.5
438c_lucATR = 1
439oanda_count = False
440lstreak = True
441
442##### OANDA PARAMETERS
443""""
444usage: candle-data [-h] [--nice] [--count COUNT] --granularity
445 {S5,S10,S15,S30,M1,M2,M4,M5,M10,M15,M30,H1,H2,H3,H4,H6,H8,H12,D,W,M}
446 [--price {M,B,A,BA,MBA}] [--from FROM] [--to TO]
447 [--instruments [INSTRUMENTS]]"""
448
449accountID, token = exampleAuth()
450api = API(access_token=token)
451
452bars = {'AUDCAD':0, 'AUDCHF':0, 'AUDJPY':0, 'AUDNZD':0, 'AUDUSD':0,'CADCHF':0, 'CADJPY':0, 'CHFJPY':0, 'EURAUD':0, 'EURCAD':0, 'EURCHF':0, 'EURGBP':0, 'EURJPY':0, 'EURNZD':0, 'EURUSD':0, 'GBPAUD':0, 'GBPCAD':0, 'GBPCHF':0, 'GBPJPY':0, 'GBPNZD':0, 'GBPUSD':0, 'NZDCAD':0, 'NZDCHF':0, 'NZDJPY':0, 'NZDUSD':0, 'USDCAD':0, 'USDCHF':0, 'USDJPY':0}
453currencies = list(bars.keys())
454
455oanda_start_year = start[:4]
456oanda_end_year = end[:4]
457
458#separarei as requisicoes de 2 em 2 anos, pq 5000 candles em 4H da 2,6 anos
459qnt_req_oanda = math.ceil((int(oanda_end_year) - int(oanda_start_year)) / 2)
460
461print('Extracting data from broker...')
462for j in range(qnt_req_oanda):
463 oanda_start_year = int(start[:4]) + (j * 2)
464 oanda_start = str(oanda_start_year) + start[4:]
465 oanda_end_year = oanda_start_year + 1
466 if j < qnt_req_oanda-1:
467 oanda_end = str(oanda_end_year) + '-12-31T00:00:00Z'
468 else:
469 oanda_end = str(oanda_end_year) + end[4:]
470
471
472 oanda_params = {}
473 oanda_params.update({"granularity": candle})
474 if not oanda_count:
475 if check_date(start):
476 oanda_params.update({"from": oanda_start})
477 if check_date(end):
478 oanda_params.update({"to": oanda_end})
479 else:
480 oanda_params.update({"count": oanda_count})
481 oanda_params.update({"price": 'B'})
482
483
484
485 # aqui é feita uma juncao de cada currencie e cada periodo de requisicao montando um dict só
486 print('start: ' + oanda_start)
487 print('end: ' + oanda_end)
488 for i in range(len(bars)):
489 r = instruments.InstrumentsCandles(instrument=currencies[i][:3]+'_'+currencies[i][3:], params=oanda_params)
490 oanda_bars = api.request(r)
491
492 oanda_time = [x['time'] for x in oanda_bars['candles']]
493 oanda_data = [x['bid'] for x in oanda_bars['candles']]
494 for idx, x in enumerate(oanda_bars['candles']):
495 oanda_data[idx].update({'volume': x['volume']})
496
497 bars_temp = pd.DataFrame(oanda_data, index = oanda_time)
498 bars_temp = bars_temp.rename(columns={'c': 'close', 'h': 'high', 'l': 'low', 'o': 'open'})
499 bars_temp[['close', 'high', 'low','open']] = bars_temp[['close', 'high', 'low','open']].apply(pd.to_numeric)
500 bars_temp['losses'] = bars_temp['wins'] = 0
501
502 if j > 0:
503 bars[currencies[i]] = bars[currencies[i]].append(bars_temp)
504 else:
505 bars[currencies[i]] = bars_temp
506
507
508
509print('Data extractred. Running algo...')
510#######################################################################################################################
511""" -------------------------------------------- ON START ----------------------------------------------------- """
512#######################################################################################################################
513period = 1
514pasta_raiz = 'testes'+candle
515
516
517
518# bars = get_data(candle, start, end)
519news_currencies = []
520news_currencies.append(0)
521
522earnings = [minicial, minicial, minicial, minicial]
523# tratativa para remover algumas moedas
524qnt_p = 1
525
526news_currencies = []
527
528paramsFixed = np.loadtxt('registry/paramsFixed.csv', dtype = object, delimiter=',', skiprows=1)
529paramsFixed = paramsFixed[:, [10,11]].astype(float)
530#paramsFixed = 0
531
532for currency in currencies:
533 directory_real = pasta_raiz+'/'+currency
534 if not os.path.exists(directory_real):
535 os.makedirs(directory_real)
536 os.makedirs(directory_real+'/moves')
537 # os.makedirs(directory+'/plots')
538 else:
539 erase_files(directory_real, '/moves')
540
541
542#######################################################################################################################
543""" --------------------------------------------- ON RUN ------------------------------------------------------ """
544#######################################################################################################################
545with open('registry/csvzaozao_'+strat+'.csv', 'w') as filedata:
546 writer = csv.writer(filedata, delimiter=',')
547 data = ['Periodo', 'Per Total', 'Moves', 'N acertos', 'N erros',
548 'Per acertos', 'Open', 'Close', 'High', 'Low', 'HVI']
549 writer.writerow(data)
550
551with open('registry/allmoves_'+strat+'.csv', 'w') as filedata:
552 writer = csv.writer(filedata, delimiter=',')
553 data = ['Periodo', 'Currency', 'Entrada', 'Saida', 'Ação',
554 'V Entrada', 'V Saida', 'Lucro', 'ATRi', 'Sinal']
555 writer.writerow(data)
556
557# Definição de critério de parada do Algo, a data da ultima validação tem que ser igual a ultima data de candle
558first_index = next(iter(bars))
559cur_algo = bars[first_index].index[0]
560
561oanda_date_format = "%Y-%m-%dT%H:%M:%S.%f000Z"
562cur_algo = dt.datetime.strptime(cur_algo, oanda_date_format)
563runAlgo = True
564
565# as datas de validação no inicio sao o primeiro indice do candle
566ano_op = cur_algo.year
567mesInicial = cur_algo.month
568while runAlgo:
569 best_fitness = np.array([float(0) for _ in range(nRuns)])
570 best_params = ([np.array([0] * qnt_p)
571 for _ in range(nRuns)])
572 data_currencies_teste = {}
573 data_currencies_validation = {}
574 hist_solutions = []
575
576 # cortes nos dados
577 for currency in currencies:
578 # eu corto a validação em: nteste - safeStart até nteste + validacao + safeStart
579 data_currencies_validation[currency] = bars[currency].iloc[(nCandlesT-nCandlesSafeStart):nCandlesT+nCandlesV+nCandlesSafeT]
580 # o corte no teste é: inicio até nteste
581 data_currencies_teste[currency] = bars[currency].iloc[:nCandlesT]
582
583 # defino as datas de ano e mes da validação, essas datas tbm serão usadas para checar se é o fim do Algo
584 first_index = next(iter(data_currencies_validation))
585 cur_algo = dt.datetime.strptime(data_currencies_validation[first_index].index[nCandlesSafeStart], oanda_date_format)
586 ano_op = cur_algo.year
587 mesInicial = cur_algo.month
588
589 for r in range(0, nRuns):
590 particle_position_vector = []
591 hist_gbest_fitness = []
592 for i in range(n_particles):
593
594 # param1 = random.randint(4, 25) # Williams Period
595 # param2 = random.randint(20, 90) # Feixe
596
597 # param4 = random.randint(5,40) #KS
598
599 # parametros KS + BB
600 # param1 = random.uniform(0.8,2) # ATR lucro
601 # param2 = random.uniform(0.8,2) # ATR preju
602 # param2 = param1 * 2 if param2 > (2 * param1) else param2
603
604 param1 = random.randint(10,120) #hawk
605 # param2 = random.randint(4, 25) # ASH SMOOTH
606 # param3 = param2 + random.randint(4, 25) # ASH PERIOD
607
608 # param2 = random.randint(4, 25) # ASH SMOOTH SAIDA
609 # param1 = param2 + random.randint(4, 25) # ASH PERIOD SAIDA
610
611 """ # DEDO NO BRIOCO
612 param1 = random.uniform(0.01,0.05) # step
613 param2 = random.uniform(0.1,0.5) # maximum
614 param3 = random.randint(4,15) # EMAFAST
615 param4 = random.randint(40,60) #EMASLOW
616 param5 = random.randint(5,12) # MACD1
617 param6 = param5 + random.randint(2,8) # MACD2
618 param7 = param6 + random.randint(2,8) #MACD3
619 param8 = random.randint(6,20) #RSI
620 param9 = random.randint(10,30) # BB
621 param10 = random.randint(3,8) # STOCH FASTK
622 param11 = random.randint(2,7) # STOCH SLOWK
623 param12 = random.randint(2,7) # STOCH SLOWD """
624
625 # fim parametros
626
627 # paramsFixed
628 particle_position_vector.append(np.array([param1]))
629 hist_solutions.append(particle_position_vector[i])
630
631 param_fitness = 6
632
633 pbest_position = particle_position_vector
634 params_fitness = np.array([float(0) for _ in range(param_fitness)])
635 pbest_fitness_value = np.array(
636 [params_fitness for _ in range(n_particles)], dtype=object)
637 gbest_fitness_value = np.array(
638 [float(-10000) for _ in range(param_fitness)], dtype=object)
639 gbest_position = np.array([float(0) for _ in range(qnt_p)])
640
641 velocity_vector = ([np.array([0] * qnt_p) for _ in range(n_particles)])
642 iteration = 0
643 while iteration < n_iterations:
644
645 """ fitness_cadidate = ray.get([fitness_function.remote(data_currencies_teste, earnings, particle_position_vector[i], news_currencies, period, mesInicial, ano_op) for i in range(n_particles)])
646
647 for i in range(n_particles):
648 if(pbest_fitness_value[i][0] < fitness_cadidate[i][0]):
649 pbest_fitness_value[i] = fitness_cadidate[i]
650 pbest_position[i] = particle_position_vector[i]
651
652 if(gbest_fitness_value[0] < fitness_cadidate[i][0]):
653 gbest_fitness_value = fitness_cadidate[i]
654 gbest_position = particle_position_vector[i] """
655
656 for i in range(n_particles):
657 fitness_cadidate = fitness_function(data_currencies_teste, earnings, particle_position_vector[i], news_currencies, period, mesInicial, ano_op)
658 # fitness_cadidate = fitness_function(particle_position_vector[i], data_currencies_teste, currencies, teste, gnrt_csv, period, 0,paramsFixed,news_currencies,[], earnings)
659
660 if(pbest_fitness_value[i][0] < fitness_cadidate[0]):
661 pbest_fitness_value[i] = fitness_cadidate
662 pbest_position[i] = particle_position_vector[i]
663
664 if(gbest_fitness_value[0] < fitness_cadidate[0]):
665 gbest_fitness_value = fitness_cadidate
666 gbest_position = particle_position_vector[i]
667
668 # if(abs(gbest_fitness_value - target) < target_error):
669 # break
670
671 for i in range(n_particles):
672 r1 = np.array([random.uniform(0, 1)] * qnt_p)
673 r2 = np.array([random.uniform(0, 1)] * qnt_p)
674
675 velocity_vector[i] = (weight*velocity_vector[i]) + (np.array([c1] * qnt_p)) * (r1 * (pbest_position[i] -
676 particle_position_vector[i])) + (np.array([c2] * qnt_p)) * (r2 * (gbest_position-particle_position_vector[i]))
677 particle_position_vector[i] = velocity_vector[i] + \
678 particle_position_vector[i]
679
680
681 # ASH
682 """ particle_position_vector[i][:][particle_position_vector[i][:] < 4] = 4
683 particle_position_vector[i][:][particle_position_vector[i][:] > 60] = 60 """
684
685
686 """ #WILLIAMS
687 particle_position_vector[i][0] = 4 if particle_position_vector[i][0] < 4 else particle_position_vector[i][0]
688 particle_position_vector[i][1] = 20 if particle_position_vector[i][1] < 20 else particle_position_vector[i][1]
689 particle_position_vector[i][1] = 90 if particle_position_vector[i][1] > 90 else particle_position_vector[i][1] """
690
691 """ #DEDO NO BRIOCO
692 # SAR
693 particle_position_vector[i][0] = 0.01 if particle_position_vector[i][0] < 0.01 else particle_position_vector[i][0]
694 particle_position_vector[i][0] = 0.05 if particle_position_vector[i][0] > 0.05 else particle_position_vector[i][0]
695 particle_position_vector[i][0] = np.around(particle_position_vector[i][0], decimals=3)
696
697 particle_position_vector[i][1] = 0.1 if particle_position_vector[i][1] < 0.1 else particle_position_vector[i][1]
698 particle_position_vector[i][1] = 0.5 if particle_position_vector[i][1] > 0.5 else particle_position_vector[i][1]
699 particle_position_vector[i][1] = np.around(particle_position_vector[i][1], decimals=3)
700
701 # ALLINT
702 particle_position_vector[i][2:] = particle_position_vector[i][2:].astype(int)
703
704 particle_position_vector[i][2] = 4 if particle_position_vector[i][2] < 4 else particle_position_vector[i][2]
705 particle_position_vector[i][2] = 15 if particle_position_vector[i][2] > 15 else particle_position_vector[i][2]
706 particle_position_vector[i][3] = 40 if particle_position_vector[i][3] < 40 else particle_position_vector[i][3]
707 particle_position_vector[i][3] = 60 if particle_position_vector[i][3] > 60 else particle_position_vector[i][3]
708
709
710 particle_position_vector[i][4:9][particle_position_vector[i][4:9] < 4] = 4
711 particle_position_vector[i][4:9][particle_position_vector[i][4:9] > 30] = 30
712
713 # EMASLOW > EMAFAST
714 particle_position_vector[i][3] = (particle_position_vector[i][2] + 3) if (particle_position_vector[i][3] - 2) <= particle_position_vector[i][2] else particle_position_vector[i][3]
715
716 # MACD SLOW PERIOD AND FAST
717 particle_position_vector[i][5] = (particle_position_vector[i][4] + 3) if (particle_position_vector[i][5] - 2) <= particle_position_vector[i][4] else particle_position_vector[i][5]
718 particle_position_vector[i][6] = (particle_position_vector[i][5] + 3) if (particle_position_vector[i][6] - 2) <= particle_position_vector[i][5] else particle_position_vector[i][6]
719
720 # STOCH 9 10 11
721 particle_position_vector[i][9:][particle_position_vector[i][9:] < 2] = 2
722 particle_position_vector[i][9:][particle_position_vector[i][9:] > 8] = 8
723 particle_position_vector[i][9] = (particle_position_vector[i][10] + 2) if (particle_position_vector[i][9] - 2) <= particle_position_vector[i][10] else particle_position_vector[i][9]
724 particle_position_vector[i][9] = (particle_position_vector[i][11] + 2) if (particle_position_vector[i][9] - 2) <= particle_position_vector[i][11] else particle_position_vector[i][9]
725 particle_position_vector[i][11] = particle_position_vector[i][10] if particle_position_vector[i][11] < particle_position_vector[i][10] else particle_position_vector[i][11] """
726
727 # ATR
728 # particle_position_vector[i][:2][particle_position_vector[i][:2] < 0.8] = 0.8
729 # particle_position_vector[i][:2][particle_position_vector[i][:2] > 2] = 2
730 # particle_position_vector[i][:2] = np.around(particle_position_vector[i][:2], decimals=1)
731
732 # HVI
733 particle_position_vector[i][0] = particle_position_vector[i][0].astype(int)
734 particle_position_vector[i][0] = 5 if particle_position_vector[i][0] < 5 else particle_position_vector[i][0]
735 particle_position_vector[i][0] = 150 if particle_position_vector[i][0] > 150 else particle_position_vector[i][0]
736
737
738 """ particle_position_vector[i][0:5] = np.round(new_position[0:5], decimals=1)
739 particle_position_vector[i][0:5] = particle_position_vector[i][0:5].astype(int)
740 particle_position_vector[i][0:5][particle_position_vector[i][0:5] < 4] = 4
741 particle_position_vector[i][5:][particle_position_vector[i][5:] < 0.8] = 0.8
742 particle_position_vector[i][6:][particle_position_vector[i][6:] > 2] = 2
743 particle_position_vector[i][5:] = np.around(particle_position_vector[i][5:], decimals=1) """
744
745 # controle preju e ganho ATR
746 # particle_position_vector[i][1] = (particle_position_vector[i][0] * 2) if particle_position_vector[i][1] > (2 * particle_position_vector[i][0]) else particle_position_vector[i][1]
747
748
749 particle_position_vector[i][:] = particle_position_vector[i][:].astype(int)
750
751 while (hist_solutions == particle_position_vector[i]).all(1).any():
752 print('anti stagnation')
753 indicador = random.randint(0,len(particle_position_vector[0])-1)
754 minV = min([row[indicador] for row in hist_solutions])
755 maxV = max([row[indicador] for row in hist_solutions])
756 particle_position_vector[i][indicador] = random.randint(5,150)
757
758 hist_solutions.append(particle_position_vector[i])
759
760 print('-')
761 print("The best position is ", gbest_position, "in iteration number ",
762 iteration, ' fitness: ', gbest_fitness_value[0])
763 print("Melhor Per: ", gbest_fitness_value[4], "Winrate: ", gbest_fitness_value[1],
764 ' Qtde Op: ', gbest_fitness_value[2], 'periodo: ', period, 'run: ', r)
765 print('-')
766 hist_gbest_fitness.append(gbest_fitness_value)
767 fitness_over_iterations = [row[0] for row in hist_gbest_fitness]
768 repeats = max(collections.Counter(
769 fitness_over_iterations).values())
770 if repeats > 3:
771 print('Muitas repetições, saindo...')
772 break
773
774 iteration = iteration + 1
775
776 print("The best position is ", gbest_position,
777 "in iteration number ", iteration)
778
779 ##FIM DOS RUNS!##
780
781 best_fitness[r] = gbest_fitness_value[0]
782 best_params[r] = gbest_position
783
784 ######GERAÇÃO DOS DADOS PARA OS ANOS DE VALIDACAO########
785
786 with open('registry/periods/csvzao_validation_'+strat+'period'+str(period)+'mes'+str(mesInicial)+'-'+str(ano_op)+'.csv', 'w') as filedata:
787 writer = csv.writer(filedata, delimiter=',')
788 # DATA , TIPO, VALOR ENT, VALOR SAIDA, LUCRO, PRECISION TP HOLD
789 data = ['Moeda', 'Lucro Total', 'Qtd Positivos',
790 'Qtd Negativos', 'Qtd Total', '% Positivos', '% Perlucro']
791 writer.writerow(data)
792
793 indexB = np.argmax(best_fitness)
794 period_best_params = best_params[indexB]
795
796 # fitness_cadidate = ray.get(fitness_function.remote(data_currencies_validation, earnings, period_best_params, news_currencies, period, mesInicial, ano_op, validation=True))
797 fitness_cadidate = fitness_function(data_currencies_validation, earnings, period_best_params, news_currencies, period, mesInicial, ano_op, validation=True)
798
799
800 earnings = fitness_cadidate[5]
801
802 dados = genfromtxt('registry/periods/csvzao_validation_'+strat+'period'+str(
803 period)+'mes'+str(mesInicial)+'-'+str(ano_op)+'.csv', delimiter=',')
804 lucros = dados[1:, 1]
805 positivos = sum(dados[1:, 2])
806 negativos = sum(dados[1:, 3])
807 total = sum(dados[1:, 4])
808
809 lucro_total = sum(lucros)
810 acerto_total = (positivos/total)*100
811 with open('registry/periods/csvzao_validation_'+strat+'period'+str(period)+'mes'+str(mesInicial)+'-'+str(ano_op)+'.csv', 'a') as filedata:
812 writer = csv.writer(filedata, delimiter=',')
813 # DATA , TIPO, VALOR ENT, VALOR SAIDA, LUCRO, PRECISION TP HOLD
814 data = ['Total', lucro_total, positivos, negativos,
815 total, acerto_total, fitness_cadidate[4]]
816 writer.writerow(data)
817
818 with open('registry/csvzaozao_'+strat+'.csv', 'a') as filedata:
819 writer = csv.writer(filedata, delimiter=',')
820 data = [str(period), fitness_cadidate[4], total, positivos, negativos, acerto_total, str(
821 earnings[0]), str(earnings[1]), str(earnings[2]), str(earnings[3]), str(period_best_params[0])]
822 writer.writerow(data)
823
824 # mesCheck = int(data_currencies_validation[currencies[0]][len(data_currencies_validation[currencies[0]])-1,5][5:7])
825 # anoCheck = int(data_currencies_validation[currencies[0]][len(data_currencies_validation[currencies[0]])-1,5][0:4])
826
827 period = period + 1
828
829 for currency in currencies:
830 bars[currency] = bars[currency].iloc[nCandlesV:]