ymNWRzWq

1// Copyright (c) 2017, SUMOKOIN
2// Copyright (c) 2014-2016, The Monero Project
3//
4// All rights reserved.
5//
6// Redistribution and use in source and binary forms, with or without modification, are
7// permitted provided that the following conditions are met:
8//
9// 1. Redistributions of source code must retain the above copyright notice, this list of
10//    conditions and the following disclaimer.
11//
12// 2. Redistributions in binary form must reproduce the above copyright notice, this list
13//    of conditions and the following disclaimer in the documentation and/or other
14//    materials provided with the distribution.
15//
16// 3. Neither the name of the copyright holder nor the names of its contributors may be
17//    used to endorse or promote products derived from this software without specific
18//    prior written permission.
19//
20// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
21// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
22// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
23// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
24// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
25// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
27// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
28// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29//
30// Parts of this file are originally copyright (c) 2012-2013 The Cryptonote developers
31
32#include <algorithm>
33#include <cstdio>
34#include <boost/archive/binary_oarchive.hpp>
35#include <boost/archive/binary_iarchive.hpp>
36#include <boost/filesystem.hpp>
37
38#include "include_base_utils.h"
39#include "cryptonote_basic_impl.h"
40#include "tx_pool.h"
41#include "blockchain.h"
42#include "blockchain_db/blockchain_db.h"
43#include "cryptonote_format_utils.h"
44#include "cryptonote_boost_serialization.h"
45#include "cryptonote_config.h"
46#include "miner.h"
47#include "misc_language.h"
48#include "profile_tools.h"
49#include "file_io_utils.h"
50#include "common/int-util.h"
51#include "common/boost_serialization_helper.h"
52#include "warnings.h"
53#include "crypto/hash.h"
54#include "cryptonote_core/checkpoints.h"
55#include "cryptonote_core/cryptonote_core.h"
56#include "ringct/rctSigs.h"
57#include "common/perf_timer.h"
58#if defined(PER_BLOCK_CHECKPOINT)
59#include "blocks/blocks.h"
60#endif
61
62//#include "serialization/json_archive.h"
63
64/* TODO:
65 *  Clean up code:
66 *    Possibly change how outputs are referred to/indexed in blockchain and wallets
67 *
68 */
69
70using namespace cryptonote;
71using epee::string_tools::pod_to_hex;
72
73DISABLE_VS_WARNINGS(4267)
74
75// used to overestimate the block reward when estimating a per kB to use
76#define BLOCK_REWARD_OVERESTIMATE   ((uint64_t)(16000000000))
77#define MAINNET_HARDFORK_V3_HEIGHT  ((uint64_t)(116520))
78#define MAINNET_HARDFORK_V4_HEIGHT  ((uint64_t)(170000))
79#define MAINNET_HARDFORK_V5_HEIGHT  ((uint64_t)(170046))
80
81static const struct {
82  uint8_t version;
83  uint64_t height;
84  uint8_t threshold;
85  time_t time;
86} mainnet_hard_forks[] = {
87  { 1, 1, 0, 1482806500 },
88  { 2, 21300, 0, 1497657600 },
89  { 3, MAINNET_HARDFORK_V3_HEIGHT, 0, 1522800000 },
90  { 4, MAINNET_HARDFORK_V4_HEIGHT, 0, 1529728956 },
91  { 4, MAINNET_HARDFORK_V5_HEIGHT, 0, 1529729198 }
92};
93static const uint64_t mainnet_hard_fork_version_1_till = (uint64_t)-1;
94
95static const struct {
96  uint8_t version;
97  uint64_t height;
98  uint8_t threshold;
99  time_t time;
100} testnet_hard_forks[] = {
101  { 1, 1, 0, 1482806500 },
102  { 2, 5150, 0, 1497181713 },
103  { 3, 103580, 0, 1522540800 } // April 01, 2018
104};
105static const uint64_t testnet_hard_fork_version_1_till = (uint64_t)-1;
106
107//------------------------------------------------------------------
108Blockchain::Blockchain(tx_memory_pool& tx_pool) :
109  m_db(), m_tx_pool(tx_pool), m_hardfork(NULL), m_timestamps_and_difficulties_height(0), m_current_block_cumul_sz_limit(0), m_is_in_checkpoint_zone(false),
110  m_is_blockchain_storing(false), m_enforce_dns_checkpoints(false), m_max_prepare_blocks_threads(4), m_db_blocks_per_sync(1), m_db_sync_mode(db_async), m_fast_sync(true), m_show_time_stats(false), m_sync_counter(0), m_cancel(false)
111{
112  LOG_PRINT_L3("Blockchain::" << __func__);
113}
114//------------------------------------------------------------------
115bool Blockchain::have_tx(const crypto::hash &id) const
116{
117  LOG_PRINT_L3("Blockchain::" << __func__);
118  CRITICAL_REGION_LOCAL(m_blockchain_lock);
119  return m_db->tx_exists(id);
120}
121//------------------------------------------------------------------
122bool Blockchain::have_tx_keyimg_as_spent(const crypto::key_image &key_im) const
123{
124  LOG_PRINT_L3("Blockchain::" << __func__);
125  CRITICAL_REGION_LOCAL(m_blockchain_lock);
126  return  m_db->has_key_image(key_im);
127}
128//------------------------------------------------------------------
129// This function makes sure that each "input" in an input (mixins) exists
130// and collects the public key for each from the transaction it was included in
131// via the visitor passed to it.
132template <class visitor_t>
133bool Blockchain::scan_outputkeys_for_indexes(size_t tx_version, const txin_to_key& tx_in_to_key, visitor_t &vis, const crypto::hash &tx_prefix_hash, uint64_t* pmax_related_block_height) const
134{
135  LOG_PRINT_L3("Blockchain::" << __func__);
136
137  // ND: Disable locking and make method private.
138  //CRITICAL_REGION_LOCAL(m_blockchain_lock);
139
140  // verify that the input has key offsets (that it exists properly, really)
141  if(!tx_in_to_key.key_offsets.size())
142    return false;
143
144  // cryptonote_format_utils uses relative offsets for indexing to the global
145  // outputs list.  that is to say that absolute offset #2 is absolute offset
146  // #1 plus relative offset #2.
147  // TODO: Investigate if this is necessary / why this is done.
148  std::vector<uint64_t> absolute_offsets = relative_output_offsets_to_absolute(tx_in_to_key.key_offsets);
149  std::vector<output_data_t> outputs;
150
151  bool found = false;
152  auto it = m_scan_table.find(tx_prefix_hash);
153  if (it != m_scan_table.end())
154  {
155    auto its = it->second.find(tx_in_to_key.k_image);
156    if (its != it->second.end())
157    {
158      outputs = its->second;
159      found = true;
160    }
161  }
162
163  if (!found)
164  {
165    try
166    {
167      m_db->get_output_key(tx_in_to_key.amount, absolute_offsets, outputs);
168    }
169    catch (...)
170    {
171      LOG_PRINT_L0("Output does not exist! amount = " << tx_in_to_key.amount);
172      return false;
173    }
174  }
175  else
176  {
177    // check for partial results and add the rest if needed;
178    if (outputs.size() < absolute_offsets.size() && outputs.size() > 0)
179    {
180      LOG_PRINT_L1("Additional outputs needed: " << absolute_offsets.size() - outputs.size());
181      std::vector < uint64_t > add_offsets;
182      std::vector<output_data_t> add_outputs;
183      for (size_t i = outputs.size(); i < absolute_offsets.size(); i++)
184        add_offsets.push_back(absolute_offsets[i]);
185      try
186      {
187        m_db->get_output_key(tx_in_to_key.amount, add_offsets, add_outputs);
188      }
189      catch (...)
190      {
191        LOG_PRINT_L0("Output does not exist! amount = " << tx_in_to_key.amount);
192        return false;
193      }
194      outputs.insert(outputs.end(), add_outputs.begin(), add_outputs.end());
195    }
196  }
197
198  size_t count = 0;
199  for (const uint64_t& i : absolute_offsets)
200  {
201    try
202    {
203      output_data_t output_index;
204      try
205      {
206        // get tx hash and output index for output
207        if (count < outputs.size())
208          output_index = outputs.at(count);
209        else
210          output_index = m_db->get_output_key(tx_in_to_key.amount, i);
211
212        // call to the passed boost visitor to grab the public key for the output
213        if (!vis.handle_output(output_index.unlock_time, output_index.pubkey, output_index.commitment))
214        {
215          LOG_PRINT_L0("Failed to handle_output for output no = " << count << ", with absolute offset " << i);
216          return false;
217        }
218      }
219      catch (...)
220      {
221        LOG_PRINT_L0("Output does not exist! amount = " << tx_in_to_key.amount << ", absolute_offset = " << i);
222        return false;
223      }
224
225      // if on last output and pmax_related_block_height not null pointer
226      if(++count == absolute_offsets.size() && pmax_related_block_height)
227      {
228        // set *pmax_related_block_height to tx block height for this output
229        auto h = output_index.height;
230        if(*pmax_related_block_height < h)
231        {
232          *pmax_related_block_height = h;
233        }
234      }
235
236    }
237    catch (const OUTPUT_DNE& e)
238    {
239      LOG_PRINT_L0("Output does not exist: " << e.what());
240      return false;
241    }
242    catch (const TX_DNE& e)
243    {
244      LOG_PRINT_L0("Transaction does not exist: " << e.what());
245      return false;
246    }
247
248  }
249
250  return true;
251}
252//------------------------------------------------------------------
253uint64_t Blockchain::get_current_blockchain_height() const
254{
255  LOG_PRINT_L3("Blockchain::" << __func__);
256  CRITICAL_REGION_LOCAL(m_blockchain_lock);
257  return m_db->height();
258}
259//------------------------------------------------------------------
260//FIXME: possibly move this into the constructor, to avoid accidentally
261//       dereferencing a null BlockchainDB pointer
262bool Blockchain::init(BlockchainDB* db, const bool testnet, const cryptonote::test_options *test_options)
263{
264  LOG_PRINT_L3("Blockchain::" << __func__);
265  CRITICAL_REGION_LOCAL(m_blockchain_lock);
266
267  bool fakechain = test_options != NULL;
268
269  if (db == nullptr)
270  {
271    LOG_ERROR("Attempted to init Blockchain with null DB");
272    return false;
273  }
274  if (!db->is_open())
275  {
276    LOG_ERROR("Attempted to init Blockchain with unopened DB");
277    return false;
278  }
279
280  m_db = db;
281
282  m_testnet = testnet;
283  if (m_hardfork == nullptr)
284  {
285    if (fakechain)
286      m_hardfork = new HardFork(*db, 1, 0);
287    else if (m_testnet)
288      m_hardfork = new HardFork(*db, 1, testnet_hard_fork_version_1_till);
289    else
290      m_hardfork = new HardFork(*db, 1, mainnet_hard_fork_version_1_till);
291  }
292  if (fakechain)
293  {
294    for (size_t n = 0; test_options->hard_forks[n].first; ++n)
295      m_hardfork->add_fork(test_options->hard_forks[n].first, test_options->hard_forks[n].second, 0, n + 1);
296  }
297  else if (m_testnet)
298  {
299    for (size_t n = 0; n < sizeof(testnet_hard_forks) / sizeof(testnet_hard_forks[0]); ++n)
300      m_hardfork->add_fork(testnet_hard_forks[n].version, testnet_hard_forks[n].height, testnet_hard_forks[n].threshold, testnet_hard_forks[n].time);
301  }
302  else
303  {
304    for (size_t n = 0; n < sizeof(mainnet_hard_forks) / sizeof(mainnet_hard_forks[0]); ++n)
305      m_hardfork->add_fork(mainnet_hard_forks[n].version, mainnet_hard_forks[n].height, mainnet_hard_forks[n].threshold, mainnet_hard_forks[n].time);
306  }
307  m_hardfork->init();
308
309  m_db->set_hard_fork(m_hardfork);
310
311  // if the blockchain is new, add the genesis block
312  // this feels kinda kludgy to do it this way, but can be looked at later.
313  // TODO: add function to create and store genesis block,
314  //       taking testnet into account
315  if(!m_db->height())
316  {
317    LOG_PRINT_L0("Blockchain not loaded, generating genesis block.");
318    block bl = boost::value_initialized<block>();
319    block_verification_context bvc = boost::value_initialized<block_verification_context>();
320    if (m_testnet)
321    {
322      generate_genesis_block(bl, config::testnet::GENESIS_TX, config::testnet::GENESIS_NONCE);
323    }
324    else
325    {
326      generate_genesis_block(bl, config::GENESIS_TX, config::GENESIS_NONCE);
327    }
328    add_new_block(bl, bvc);
329    CHECK_AND_ASSERT_MES(!bvc.m_verifivation_failed, false, "Failed to add genesis block to blockchain");
330  }
331  // TODO: if blockchain load successful, verify blockchain against both
332  //       hard-coded and runtime-loaded (and enforced) checkpoints.
333  else
334  {
335  }
336
337  if (!fakechain)
338  {
339    // ensure we fixup anything we found and fix in the future
340    m_db->fixup();
341  }
342
343  m_db->block_txn_start(true);
344  // check how far behind we are
345  uint64_t top_block_timestamp = m_db->get_top_block_timestamp();
346  uint64_t timestamp_diff = time(NULL) - top_block_timestamp;
347
348  // genesis block has no timestamp, could probably change it to have timestamp of 1341378000...
349  if(!top_block_timestamp)
350    timestamp_diff = time(NULL) - 1341378000;
351
352  // create general purpose async service queue
353
354  m_async_work_idle = std::unique_ptr < boost::asio::io_service::work > (new boost::asio::io_service::work(m_async_service));
355  // we only need 1
356  m_async_pool.create_thread(boost::bind(&boost::asio::io_service::run, &m_async_service));
357
358#if defined(PER_BLOCK_CHECKPOINT)
359  if (!fakechain)
360    load_compiled_in_block_hashes();
361#endif
362
363  LOG_PRINT_GREEN("Blockchain initialized. last block: " << m_db->height() - 1 << ", " << epee::misc_utils::get_time_interval_string(timestamp_diff) << " time ago, current difficulty: " << get_difficulty_for_next_block(), LOG_LEVEL_0);
364  m_db->block_txn_stop();
365
366  return true;
367}
368//------------------------------------------------------------------
369bool Blockchain::init(BlockchainDB* db, HardFork*& hf, const bool testnet)
370{
371  if (hf != nullptr)
372    m_hardfork = hf;
373  bool res = init(db, testnet, NULL);
374  if (hf == nullptr)
375    hf = m_hardfork;
376  return res;
377}
378//------------------------------------------------------------------
379bool Blockchain::store_blockchain()
380{
381  LOG_PRINT_L3("Blockchain::" << __func__);
382  // lock because the rpc_thread command handler also calls this
383  CRITICAL_REGION_LOCAL(m_db->m_synchronization_lock);
384
385  TIME_MEASURE_START(save);
386  // TODO: make sure sync(if this throws that it is not simply ignored higher
387  // up the call stack
388  try
389  {
390    m_db->sync();
391  }
392  catch (const std::exception& e)
393  {
394    LOG_PRINT_L0(std::string("Error syncing blockchain db: ") + e.what() + "-- shutting down now to prevent issues!");
395    throw;
396  }
397  catch (...)
398  {
399    LOG_PRINT_L0("There was an issue storing the blockchain, shutting down now to prevent issues!");
400    throw;
401  }
402
403  TIME_MEASURE_FINISH(save);
404  if(m_show_time_stats)
405    LOG_PRINT_L0("Blockchain stored OK, took: " << save << " ms");
406  return true;
407}
408//------------------------------------------------------------------
409bool Blockchain::deinit()
410{
411  LOG_PRINT_L3("Blockchain::" << __func__);
412
413  LOG_PRINT_L1("Stopping blockchain read/write activity");
414
415 // stop async service
416  m_async_work_idle.reset();
417  m_async_pool.join_all();
418  m_async_service.stop();
419
420  // as this should be called if handling a SIGSEGV, need to check
421  // if m_db is a NULL pointer (and thus may have caused the illegal
422  // memory operation), otherwise we may cause a loop.
423  if (m_db == NULL)
424  {
425    throw new DB_ERROR("The db pointer is null in Blockchain, the blockchain may be corrupt!");
426  }
427
428  try
429  {
430    m_db->close();
431    LOG_PRINT_L1("Local blockchain read/write activity stopped successfully");
432  }
433  catch (const std::exception& e)
434  {
435    LOG_ERROR(std::string("Error closing blockchain db: ") + e.what());
436  }
437  catch (...)
438  {
439    LOG_ERROR("There was an issue closing/storing the blockchain, shutting down now to prevent issues!");
440  }
441
442  delete m_hardfork;
443  delete m_db;
444  return true;
445}
446//------------------------------------------------------------------
447// This function tells BlockchainDB to remove the top block from the
448// blockchain and then returns all transactions (except the miner tx, of course)
449// from it to the tx_pool
450block Blockchain::pop_block_from_blockchain()
451{
452  LOG_PRINT_L3("Blockchain::" << __func__);
453  CRITICAL_REGION_LOCAL(m_blockchain_lock);
454
455  m_timestamps_and_difficulties_height = 0;
456
457  block popped_block;
458  std::vector<transaction> popped_txs;
459
460  try
461  {
462    m_db->pop_block(popped_block, popped_txs);
463  }
464  // anything that could cause this to throw is likely catastrophic,
465  // so we re-throw
466  catch (const std::exception& e)
467  {
468    LOG_ERROR("Error popping block from blockchain: " << e.what());
469    throw;
470  }
471  catch (...)
472  {
473    LOG_ERROR("Error popping block from blockchain, throwing!");
474    throw;
475  }
476
477  // return transactions from popped block to the tx_pool
478  for (transaction& tx : popped_txs)
479  {
480    if (!is_coinbase(tx))
481    {
482      cryptonote::tx_verification_context tvc = AUTO_VAL_INIT(tvc);
483
484      // FIXME: HardFork
485      // Besides the below, popping a block should also remove the last entry
486      // in hf_versions.
487      //
488      // FIXME: HardFork
489      // This is not quite correct, as we really want to add the txes
490      // to the pool based on the version determined after all blocks
491      // are popped.
492      uint8_t version = get_current_hard_fork_version();
493
494      // We assume that if they were in a block, the transactions are already
495      // known to the network as a whole. However, if we had mined that block,
496      // that might not be always true. Unlikely though, and always relaying
497      // these again might cause a spike of traffic as many nodes re-relay
498      // all the transactions in a popped block when a reorg happens.
499      bool r = m_tx_pool.add_tx(tx, tvc, true, true, version);
500      if (!r)
501      {
502        LOG_ERROR("Error returning transaction to tx_pool");
503      }
504    }
505  }
506  update_next_cumulative_size_limit();
507  m_tx_pool.on_blockchain_dec(m_db->height()-1, get_tail_id());
508
509  return popped_block;
510}
511//------------------------------------------------------------------
512bool Blockchain::reset_and_set_genesis_block(const block& b)
513{
514  LOG_PRINT_L3("Blockchain::" << __func__);
515  CRITICAL_REGION_LOCAL(m_blockchain_lock);
516  m_alternative_chains.clear();
517  m_db->reset();
518  m_hardfork->init();
519
520  block_verification_context bvc = boost::value_initialized<block_verification_context>();
521  add_new_block(b, bvc);
522  return bvc.m_added_to_main_chain && !bvc.m_verifivation_failed;
523}
524//------------------------------------------------------------------
525crypto::hash Blockchain::get_tail_id(uint64_t& height) const
526{
527  LOG_PRINT_L3("Blockchain::" << __func__);
528  CRITICAL_REGION_LOCAL(m_blockchain_lock);
529  height = m_db->height() - 1;
530  return get_tail_id();
531}
532//------------------------------------------------------------------
533crypto::hash Blockchain::get_tail_id() const
534{
535  LOG_PRINT_L3("Blockchain::" << __func__);
536  CRITICAL_REGION_LOCAL(m_blockchain_lock);
537  return m_db->top_block_hash();
538}
539//------------------------------------------------------------------
540/*TODO: this function was...poorly written.  As such, I'm not entirely
541 *      certain on what it was supposed to be doing.  Need to look into this,
542 *      but it doesn't seem terribly important just yet.
543 *
544 * puts into list <ids> a list of hashes representing certain blocks
545 * from the blockchain in reverse chronological order
546 *
547 * the blocks chosen, at the time of this writing, are:
548 *   the most recent 11
549 *   powers of 2 less recent from there, so 13, 17, 25, etc...
550 *
551 */
552bool Blockchain::get_short_chain_history(std::list<crypto::hash>& ids) const
553{
554  LOG_PRINT_L3("Blockchain::" << __func__);
555  CRITICAL_REGION_LOCAL(m_blockchain_lock);
556  uint64_t i = 0;
557  uint64_t current_multiplier = 1;
558  uint64_t sz = m_db->height();
559
560  if(!sz)
561    return true;
562
563  m_db->block_txn_start(true);
564  bool genesis_included = false;
565  uint64_t current_back_offset = 1;
566  while(current_back_offset < sz)
567  {
568    ids.push_back(m_db->get_block_hash_from_height(sz - current_back_offset));
569
570    if(sz-current_back_offset == 0)
571    {
572      genesis_included = true;
573    }
574    if(i < 10)
575    {
576      ++current_back_offset;
577    }
578    else
579    {
580      current_multiplier *= 2;
581      current_back_offset += current_multiplier;
582    }
583    ++i;
584  }
585
586  if (!genesis_included)
587  {
588    ids.push_back(m_db->get_block_hash_from_height(0));
589  }
590  m_db->block_txn_stop();
591
592  return true;
593}
594//------------------------------------------------------------------
595crypto::hash Blockchain::get_block_id_by_height(uint64_t height) const
596{
597  LOG_PRINT_L3("Blockchain::" << __func__);
598  CRITICAL_REGION_LOCAL(m_blockchain_lock);
599  try
600  {
601    return m_db->get_block_hash_from_height(height);
602  }
603  catch (const BLOCK_DNE& e)
604  {
605  }
606  catch (const std::exception& e)
607  {
608    LOG_PRINT_L0(std::string("Something went wrong fetching block hash by height: ") + e.what());
609    throw;
610  }
611  catch (...)
612  {
613    LOG_PRINT_L0(std::string("Something went wrong fetching block hash by height"));
614    throw;
615  }
616  return null_hash;
617}
618//------------------------------------------------------------------
619bool Blockchain::get_block_by_hash(const crypto::hash &h, block &blk) const
620{
621  LOG_PRINT_L3("Blockchain::" << __func__);
622  CRITICAL_REGION_LOCAL(m_blockchain_lock);
623
624  // try to find block in main chain
625  try
626  {
627    blk = m_db->get_block(h);
628    return true;
629  }
630  // try to find block in alternative chain
631  catch (const BLOCK_DNE& e)
632  {
633    blocks_ext_by_hash::const_iterator it_alt = m_alternative_chains.find(h);
634    if (m_alternative_chains.end() != it_alt)
635    {
636      blk = it_alt->second.bl;
637      return true;
638    }
639  }
640  catch (const std::exception& e)
641  {
642    LOG_PRINT_L0(std::string("Something went wrong fetching block by hash: ") + e.what());
643    throw;
644  }
645  catch (...)
646  {
647    LOG_PRINT_L0(std::string("Something went wrong fetching block hash by hash"));
648    throw;
649  }
650
651  return false;
652}
653//------------------------------------------------------------------
654//FIXME: this function does not seem to be called from anywhere, but
655// if it ever is, should probably change std::list for std::vector
656void Blockchain::get_all_known_block_ids(std::list<crypto::hash> &main, std::list<crypto::hash> &alt, std::list<crypto::hash> &invalid) const
657{
658  LOG_PRINT_L3("Blockchain::" << __func__);
659  CRITICAL_REGION_LOCAL(m_blockchain_lock);
660
661  for (auto& a : m_db->get_hashes_range(0, m_db->height() - 1))
662  {
663    main.push_back(a);
664  }
665
666  for (const blocks_ext_by_hash::value_type &v: m_alternative_chains)
667    alt.push_back(v.first);
668
669  for (const blocks_ext_by_hash::value_type &v: m_invalid_blocks)
670    invalid.push_back(v.first);
671}
672//------------------------------------------------------------------
673// This function aggregates the cumulative difficulties and timestamps of the
674// last DIFFICULTY_BLOCKS_COUNT blocks and passes them to next_difficulty,
675// returning the result of that call.  Ignores the genesis block, and can use
676// less blocks than desired if there aren't enough.
677difficulty_type Blockchain::get_difficulty_for_next_block()
678{
679  LOG_PRINT_L3("Blockchain::" << __func__);
680  CRITICAL_REGION_LOCAL(m_blockchain_lock);
681  std::vector<uint64_t> timestamps;
682  std::vector<difficulty_type> difficulties;
683  auto height = m_db->height();
684  
685  if (HARD_FORK_CLAMP == 1)
686  {
687  // Reset network hashrate to 2.0 MHz when hardfork v3 comes
688  if (!m_testnet && (uint64_t)height >= MAINNET_HARDFORK_V3_HEIGHT && (uint64_t)height <= MAINNET_HARDFORK_V3_HEIGHT + (uint64_t)DIFFICULTY_BLOCKS_COUNT_V2){
689    return (difficulty_type) 480000000;
690  }
691  // Reset network hashrate to 2.0 Hz when hardfork v4 comes
692  if (!m_testnet && (uint64_t)height >= MAINNET_HARDFORK_V4_HEIGHT && (uint64_t)height <= MAINNET_HARDFORK_V4_HEIGHT + (uint64_t)DIFFICULTY_BLOCKS_COUNT_V2){
693    return (difficulty_type) 250;
694  }
695  // Reset network hashrate to 2.0 MHz when hardfork v5 comes
696  if (!m_testnet && (uint64_t)height >= MAINNET_HARDFORK_V5_HEIGHT + 1 && (uint64_t)height <= MAINNET_HARDFORK_V5_HEIGHT + (uint64_t)DIFFICULTY_BLOCKS_COUNT_V2){
697    return (difficulty_type) 480000000;
698  }
699  }
700  size_t difficult_block_count = get_current_hard_fork_version() < 2 ? DIFFICULTY_BLOCKS_COUNT : DIFFICULTY_BLOCKS_COUNT_V2;
701
702  // ND: Speedup
703  // 1. Keep a list of the last 735 (or less) blocks that is used to compute difficulty,
704  //    then when the next block difficulty is queried, push the latest height data and
705  //    pop the oldest one from the list. This only requires 1x read per height instead
706  //    of doing 735 (DIFFICULTY_BLOCKS_COUNT).
707  if (m_timestamps_and_difficulties_height != 0 && ((height - m_timestamps_and_difficulties_height) == 1))
708  {
709    uint64_t index = height - 1;
710    m_timestamps.push_back(m_db->get_block_timestamp(index));
711    m_difficulties.push_back(m_db->get_block_cumulative_difficulty(index));
712
713    while (m_timestamps.size() > difficult_block_count)
714      m_timestamps.erase(m_timestamps.begin());
715    while (m_difficulties.size() > difficult_block_count)
716      m_difficulties.erase(m_difficulties.begin());
717
718    m_timestamps_and_difficulties_height = height;
719    timestamps = m_timestamps;
720    difficulties = m_difficulties;
721  }
722  else
723  {
724    size_t offset = height - std::min < size_t >(height, static_cast<size_t>(difficult_block_count));
725    if (offset == 0)
726      ++offset;
727
728    timestamps.clear();
729    difficulties.clear();
730    for (; offset < height; offset++)
731    {
732      timestamps.push_back(m_db->get_block_timestamp(offset));
733      difficulties.push_back(m_db->get_block_cumulative_difficulty(offset));
734    }
735
736    m_timestamps_and_difficulties_height = height;
737    m_timestamps = timestamps;
738    m_difficulties = difficulties;
739  }
740  size_t target = DIFFICULTY_TARGET;
741  return get_current_hard_fork_version() < 2 ?
742    next_difficulty(timestamps, difficulties, target) :
743    next_difficulty_v2(timestamps, difficulties, target);
744}
745//------------------------------------------------------------------
746// This function removes blocks from the blockchain until it gets to the
747// position where the blockchain switch started and then re-adds the blocks
748// that had been removed.
749bool Blockchain::rollback_blockchain_switching(std::list<block>& original_chain, uint64_t rollback_height)
750{
751  LOG_PRINT_L3("Blockchain::" << __func__);
752  CRITICAL_REGION_LOCAL(m_blockchain_lock);
753
754  // fail if rollback_height passed is too high
755  if (rollback_height > m_db->height())
756  {
757    return true;
758  }
759
760  m_timestamps_and_difficulties_height = 0;
761
762  // remove blocks from blockchain until we get back to where we should be.
763  while (m_db->height() != rollback_height)
764  {
765    pop_block_from_blockchain();
766  }
767
768  // make sure the hard fork object updates its current version
769  m_hardfork->reorganize_from_chain_height(rollback_height);
770
771  //return back original chain
772  for (auto& bl : original_chain)
773  {
774    block_verification_context bvc = boost::value_initialized<block_verification_context>();
775    bool r = handle_block_to_main_chain(bl, bvc);
776    CHECK_AND_ASSERT_MES(r && bvc.m_added_to_main_chain, false, "PANIC! failed to add (again) block while chain switching during the rollback!");
777  }
778
779  m_hardfork->reorganize_from_chain_height(rollback_height);
780
781  LOG_PRINT_L1("Rollback to height " << rollback_height << " was successful.");
782  if (original_chain.size())
783  {
784    LOG_PRINT_L1("Restoration to previous blockchain successful as well.");
785  }
786  return true;
787}
788//------------------------------------------------------------------
789// This function attempts to switch to an alternate chain, returning
790// boolean based on success therein.
791bool Blockchain::switch_to_alternative_blockchain(std::list<blocks_ext_by_hash::iterator>& alt_chain, bool discard_disconnected_chain)
792{
793  LOG_PRINT_L3("Blockchain::" << __func__);
794  CRITICAL_REGION_LOCAL(m_blockchain_lock);
795
796  m_timestamps_and_difficulties_height = 0;
797
798  // if empty alt chain passed (not sure how that could happen), return false
799  CHECK_AND_ASSERT_MES(alt_chain.size(), false, "switch_to_alternative_blockchain: empty chain passed");
800
801  // verify that main chain has front of alt chain's parent block
802  if (!m_db->block_exists(alt_chain.front()->second.bl.prev_id))
803  {
804    LOG_ERROR("Attempting to move to an alternate chain, but it doesn't appear to connect to the main chain!");
805    return false;
806  }
807
808  // pop blocks from the blockchain until the top block is the parent
809  // of the front block of the alt chain.
810  std::list<block> disconnected_chain;
811  while (m_db->top_block_hash() != alt_chain.front()->second.bl.prev_id)
812  {
813    block b = pop_block_from_blockchain();
814    disconnected_chain.push_front(b);
815  }
816
817  auto split_height = m_db->height();
818
819  //connecting new alternative chain
820  for(auto alt_ch_iter = alt_chain.begin(); alt_ch_iter != alt_chain.end(); alt_ch_iter++)
821  {
822    auto ch_ent = *alt_ch_iter;
823    block_verification_context bvc = boost::value_initialized<block_verification_context>();
824
825    // add block to main chain
826    bool r = handle_block_to_main_chain(ch_ent->second.bl, bvc);
827
828    // if adding block to main chain failed, rollback to previous state and
829    // return false
830    if(!r || !bvc.m_added_to_main_chain)
831    {
832      LOG_PRINT_L1("Failed to switch to alternative blockchain");
833
834      // rollback_blockchain_switching should be moved to two different
835      // functions: rollback and apply_chain, but for now we pretend it is
836      // just the latter (because the rollback was done above).
837      rollback_blockchain_switching(disconnected_chain, split_height);
838
839      // FIXME: Why do we keep invalid blocks around?  Possibly in case we hear
840      // about them again so we can immediately dismiss them, but needs some
841      // looking into.
842      add_block_as_invalid(ch_ent->second, get_block_hash(ch_ent->second.bl));
843      LOG_PRINT_L1("The block was inserted as invalid while connecting new alternative chain, block_id: " << get_block_hash(ch_ent->second.bl));
844      m_alternative_chains.erase(*alt_ch_iter++);
845
846      for(auto alt_ch_to_orph_iter = alt_ch_iter; alt_ch_to_orph_iter != alt_chain.end(); )
847      {
848        add_block_as_invalid((*alt_ch_to_orph_iter)->second, (*alt_ch_to_orph_iter)->first);
849        m_alternative_chains.erase(*alt_ch_to_orph_iter++);
850      }
851      return false;
852    }
853  }
854
855  // if we're to keep the disconnected blocks, add them as alternates
856  if(!discard_disconnected_chain)
857  {
858    //pushing old chain as alternative chain
859    for (auto& old_ch_ent : disconnected_chain)
860    {
861      block_verification_context bvc = boost::value_initialized<block_verification_context>();
862      bool r = handle_alternative_block(old_ch_ent, get_block_hash(old_ch_ent), bvc);
863      if(!r)
864      {
865        LOG_PRINT_L1("Failed to push ex-main chain blocks to alternative chain ");
866        // previously this would fail the blockchain switching, but I don't
867        // think this is bad enough to warrant that.
868      }
869    }
870  }
871
872  //removing alt_chain entries from alternative chains container
873  for (auto ch_ent: alt_chain)
874  {
875    m_alternative_chains.erase(ch_ent);
876  }
877
878  m_hardfork->reorganize_from_chain_height(split_height);
879
880  LOG_PRINT_GREEN("REORGANIZE SUCCESS! on height: " << split_height << ", new blockchain size: " << m_db->height(), LOG_LEVEL_0);
881  return true;
882}
883//------------------------------------------------------------------
884// This function calculates the difficulty target for the block being added to
885// an alternate chain.
886difficulty_type Blockchain::get_next_difficulty_for_alternative_chain(const std::list<blocks_ext_by_hash::iterator>& alt_chain, block_extended_info& bei) const
887{
888  LOG_PRINT_L3("Blockchain::" << __func__);
889  std::vector<uint64_t> timestamps;
890  std::vector<difficulty_type> cumulative_difficulties;
891  size_t difficult_block_count = get_current_hard_fork_version() < 2 ? DIFFICULTY_BLOCKS_COUNT : DIFFICULTY_BLOCKS_COUNT_V2;
892
893  // if the alt chain isn't long enough to calculate the difficulty target
894  // based on its blocks alone, need to get more blocks from the main chain
895  if (alt_chain.size() < difficult_block_count)
896  {
897    CRITICAL_REGION_LOCAL(m_blockchain_lock);
898
899    // Figure out start and stop offsets for main chain blocks
900    size_t main_chain_stop_offset = alt_chain.size() ? alt_chain.front()->second.height : bei.height;
901    size_t main_chain_count = difficult_block_count - std::min(static_cast<size_t>(difficult_block_count), alt_chain.size());
902    main_chain_count = std::min(main_chain_count, main_chain_stop_offset);
903    size_t main_chain_start_offset = main_chain_stop_offset - main_chain_count;
904
905    if (!main_chain_start_offset)
906      ++main_chain_start_offset; //skip genesis block
907
908    // get difficulties and timestamps from relevant main chain blocks
909    for (; main_chain_start_offset < main_chain_stop_offset; ++main_chain_start_offset)
910    {
911      timestamps.push_back(m_db->get_block_timestamp(main_chain_start_offset));
912      cumulative_difficulties.push_back(m_db->get_block_cumulative_difficulty(main_chain_start_offset));
913    }
914
915    // make sure we haven't accidentally grabbed too many blocks...maybe don't need this check?
916    CHECK_AND_ASSERT_MES((alt_chain.size() + timestamps.size()) <= difficult_block_count, false, "Internal error, alt_chain.size()[" << alt_chain.size() << "] + vtimestampsec.size()[" << timestamps.size() << "] NOT <= DIFFICULTY_WINDOW[]" << difficult_block_count);
917
918    for (auto it : alt_chain)
919    {
920      timestamps.push_back(it->second.bl.timestamp);
921      cumulative_difficulties.push_back(it->second.cumulative_difficulty);
922    }
923  }
924  // if the alt chain is long enough for the difficulty calc, grab difficulties
925  // and timestamps from it alone
926  else
927  {
928    timestamps.resize(static_cast<size_t>(difficult_block_count));
929    cumulative_difficulties.resize(static_cast<size_t>(difficult_block_count));
930    size_t count = 0;
931    size_t max_i = timestamps.size() - 1;
932    // get difficulties and timestamps from most recent blocks in alt chain
933    BOOST_REVERSE_FOREACH(auto it, alt_chain)
934    {
935      timestamps[max_i - count] = it->second.bl.timestamp;
936      cumulative_difficulties[max_i - count] = it->second.cumulative_difficulty;
937      count++;
938      if (count >= difficult_block_count)
939        break;
940    }
941  }
942
943  // FIXME: This will fail if fork activation heights are subject to voting
944  size_t target = DIFFICULTY_TARGET;
945
946  // calculate the difficulty target for the block and return it
947  return get_current_hard_fork_version() < 2 ?
948    next_difficulty(timestamps, cumulative_difficulties, target) :
949    next_difficulty_v2(timestamps, cumulative_difficulties, target);
950}
951//------------------------------------------------------------------
952// This function does a sanity check on basic things that all miner
953// transactions have in common, such as:
954//   one input, of type txin_gen, with height set to the block's height
955//   correct miner tx unlock time
956//   a non-overflowing tx amount (dubious necessity on this check)
957bool Blockchain::prevalidate_miner_transaction(const block& b, uint64_t height)
958{
959  LOG_PRINT_L3("Blockchain::" << __func__);
960  CHECK_AND_ASSERT_MES(b.miner_tx.vin.size() == 1, false, "coinbase transaction in the block has no inputs");
961  CHECK_AND_ASSERT_MES(b.miner_tx.vin[0].type() == typeid(txin_gen), false, "coinbase transaction in the block has the wrong type");
962  if(boost::get<txin_gen>(b.miner_tx.vin[0]).height != height)
963  {
964    LOG_PRINT_RED_L1("The miner transaction in block has invalid height: " << boost::get<txin_gen>(b.miner_tx.vin[0]).height << ", expected: " << height);
965    return false;
966  }
967  CHECK_AND_ASSERT_MES(b.miner_tx.unlock_time == height + CRYPTONOTE_MINED_MONEY_UNLOCK_WINDOW, false, "coinbase transaction transaction has the wrong unlock time=" << b.miner_tx.unlock_time << ", expected " << height + CRYPTONOTE_MINED_MONEY_UNLOCK_WINDOW);
968
969  //check outs overflow
970  //NOTE: not entirely sure this is necessary, given that this function is
971  //      designed simply to make sure the total amount for a transaction
972  //      does not overflow a uint64_t, and this transaction *is* a uint64_t...
973  if(!check_outs_overflow(b.miner_tx))
974  {
975    LOG_PRINT_RED_L1("miner transaction has money overflow in block " << get_block_hash(b));
976    return false;
977  }
978
979  return true;
980}
981//------------------------------------------------------------------
982// This function validates the miner transaction reward
983bool Blockchain::validate_miner_transaction(const block& b, size_t cumulative_block_size, uint64_t fee, uint64_t& base_reward, uint64_t already_generated_coins, bool &partial_block_reward, uint8_t version)
984{
985  LOG_PRINT_L3("Blockchain::" << __func__);
986  //validate reward
987  uint64_t money_in_use = 0;
988  for (auto& o: b.miner_tx.vout)
989    money_in_use += o.amount;
990  partial_block_reward = false;
991
992  std::vector<size_t> last_blocks_sizes;
993  get_last_n_blocks_sizes(last_blocks_sizes, CRYPTONOTE_REWARD_BLOCKS_WINDOW);
994
995  if (!get_block_reward(epee::misc_utils::median(last_blocks_sizes), cumulative_block_size, already_generated_coins, base_reward, m_db->height()))
996  {
997    LOG_PRINT_L1("block size " << cumulative_block_size << " is bigger than allowed for this blockchain");
998    return false;
999  }
1000  if(base_reward + fee < money_in_use)
1001  {
1002    LOG_PRINT_L1("coinbase transaction spend too much money (" << print_money(money_in_use) << "). Block reward is " << print_money(base_reward + fee) << "(" << print_money(base_reward) << "+" << print_money(fee) << ")");
1003    return false;
1004  }
1005  
1006  // from hard fork 2, since a miner can claim less than the full block reward, we update the base_reward
1007  // to show the amount of coins that were actually generated, the remainder will be pushed back for later
1008  // emission. This modifies the emission curve very slightly.
1009  CHECK_AND_ASSERT_MES(money_in_use - fee <= base_reward, false, "base reward calculation bug");
1010  if(base_reward + fee != money_in_use)
1011    partial_block_reward = true;
1012  base_reward = money_in_use - fee;
1013  
1014  return true;
1015}
1016//------------------------------------------------------------------
1017// get the block sizes of the last <count> blocks, and return by reference <sz>.
1018void Blockchain::get_last_n_blocks_sizes(std::vector<size_t>& sz, size_t count) const
1019{
1020  LOG_PRINT_L3("Blockchain::" << __func__);
1021  CRITICAL_REGION_LOCAL(m_blockchain_lock);
1022  auto h = m_db->height();
1023
1024  // this function is meaningless for an empty blockchain...granted it should never be empty
1025  if(h == 0)
1026    return;
1027
1028  m_db->block_txn_start(true);
1029  // add size of last <count> blocks to vector <sz> (or less, if blockchain size < count)
1030  size_t start_offset = h - std::min<size_t>(h, count);
1031  for(size_t i = start_offset; i < h; i++)
1032  {
1033    sz.push_back(m_db->get_block_size(i));
1034  }
1035  m_db->block_txn_stop();
1036}
1037//------------------------------------------------------------------
1038uint64_t Blockchain::get_current_cumulative_blocksize_limit() const
1039{
1040  LOG_PRINT_L3("Blockchain::" << __func__);
1041  return m_current_block_cumul_sz_limit;
1042}
1043//------------------------------------------------------------------
1044//TODO: This function only needed minor modification to work with BlockchainDB,
1045//      and *works*.  As such, to reduce the number of things that might break
1046//      in moving to BlockchainDB, this function will remain otherwise
1047//      unchanged for the time being.
1048//
1049// This function makes a new block for a miner to mine the hash for
1050//
1051// FIXME: this codebase references #if defined(DEBUG_CREATE_BLOCK_TEMPLATE)
1052// in a lot of places.  That flag is not referenced in any of the code
1053// nor any of the makefiles, howeve.  Need to look into whether or not it's
1054// necessary at all.
1055bool Blockchain::create_block_template(block& b, const account_public_address& miner_address, difficulty_type& diffic, uint64_t& height, const blobdata& ex_nonce)
1056{
1057  LOG_PRINT_L3("Blockchain::" << __func__);
1058  size_t median_size;
1059  uint64_t already_generated_coins;
1060  uint64_t cal_height;
1061
1062  CRITICAL_REGION_BEGIN(m_blockchain_lock);
1063  height = m_db->height();
1064
1065  b.major_version = m_hardfork->get_current_version();
1066  b.minor_version = m_hardfork->get_ideal_version();
1067  b.prev_id = get_tail_id();
1068  b.timestamp = time(NULL);
1069
1070  diffic = get_difficulty_for_next_block();
1071  CHECK_AND_ASSERT_MES(diffic, false, "difficulty overhead.");
1072
1073  median_size = m_current_block_cumul_sz_limit / 2;
1074
1075  cal_height = height - height % COIN_EMISSION_HEIGHT_INTERVAL;
1076  already_generated_coins = cal_height ? m_db->get_block_already_generated_coins(cal_height - 1) : 0;
1077  
1078  CRITICAL_REGION_END();
1079
1080  size_t txs_size;
1081  uint64_t fee;
1082  uint8_t hf_version = m_hardfork->get_current_version();
1083  if (!m_tx_pool.fill_block_template(b, median_size, already_generated_coins, txs_size, fee, height))
1084  {
1085    return false;
1086  }
1087#if defined(DEBUG_CREATE_BLOCK_TEMPLATE)
1088  size_t real_txs_size = 0;
1089  uint64_t real_fee = 0;
1090  CRITICAL_REGION_BEGIN(m_tx_pool.m_transactions_lock);
1091  for(crypto::hash &cur_hash: b.tx_hashes)
1092  {
1093    auto cur_res = m_tx_pool.m_transactions.find(cur_hash);
1094    if (cur_res == m_tx_pool.m_transactions.end())
1095    {
1096      LOG_ERROR("Creating block template: error: transaction not found");
1097      continue;
1098    }
1099    tx_memory_pool::tx_details &cur_tx = cur_res->second;
1100    real_txs_size += cur_tx.blob_size;
1101    real_fee += cur_tx.fee;
1102    if (cur_tx.blob_size != get_object_blobsize(cur_tx.tx))
1103    {
1104      LOG_ERROR("Creating block template: error: invalid transaction size");
1105    }
1106    if (cur_tx.tx.version == 1)
1107    {
1108      uint64_t inputs_amount;
1109      if (!get_inputs_money_amount(cur_tx.tx, inputs_amount))
1110      {
1111        LOG_ERROR("Creating block template: error: cannot get inputs amount");
1112      }
1113      else if (cur_tx.fee != inputs_amount - get_outs_money_amount(cur_tx.tx))
1114      {
1115        LOG_ERROR("Creating block template: error: invalid fee");
1116      }
1117    }
1118    else
1119    {
1120      if (cur_tx.fee != cur_tx.tx.rct_signatures.txnFee)
1121      {
1122        LOG_ERROR("Creating block template: error: invalid fee");
1123      }
1124    }
1125  }
1126  if (txs_size != real_txs_size)
1127  {
1128    LOG_ERROR("Creating block template: error: wrongly calculated transaction size");
1129  }
1130  if (fee != real_fee)
1131  {
1132    LOG_ERROR("Creating block template: error: wrongly calculated fee");
1133  }
1134  CRITICAL_REGION_END();
1135  LOG_PRINT_L1("Creating block template: height " << height <<
1136      ", median size " << median_size <<
1137      ", already generated coins " << already_generated_coins <<
1138      ", transaction size " << txs_size <<
1139      ", fee " << fee);
1140#endif
1141
1142  /*
1143   two-phase miner transaction generation: we don't know exact block size until we prepare block, but we don't know reward until we know
1144   block size, so first miner transaction generated with fake amount of money, and with phase we know think we know expected block size
1145   */
1146  //make blocks coin-base tx looks close to real coinbase tx to get truthful blob size
1147  size_t max_outs = 1;
1148  bool r = construct_miner_tx(height, median_size, already_generated_coins, txs_size, fee, miner_address, b.miner_tx, ex_nonce, max_outs, hf_version);
1149  CHECK_AND_ASSERT_MES(r, false, "Failed to construct miner tx, first chance");
1150  size_t cumulative_size = txs_size + get_object_blobsize(b.miner_tx);
1151#if defined(DEBUG_CREATE_BLOCK_TEMPLATE)
1152  LOG_PRINT_L1("Creating block template: miner tx size " << get_object_blobsize(b.miner_tx) <<
1153      ", cumulative size " << cumulative_size);
1154#endif
1155  for (size_t try_count = 0; try_count != 10; ++try_count)
1156  {
1157    r = construct_miner_tx(height, median_size, already_generated_coins, cumulative_size, fee, miner_address, b.miner_tx, ex_nonce, max_outs, hf_version);
1158
1159    CHECK_AND_ASSERT_MES(r, false, "Failed to construct miner tx, second chance");
1160    size_t coinbase_blob_size = get_object_blobsize(b.miner_tx);
1161    if (coinbase_blob_size > cumulative_size - txs_size)
1162    {
1163      cumulative_size = txs_size + coinbase_blob_size;
1164#if defined(DEBUG_CREATE_BLOCK_TEMPLATE)
1165      LOG_PRINT_L1("Creating block template: miner tx size " << coinbase_blob_size <<
1166          ", cumulative size " << cumulative_size << " is greater then before");
1167#endif
1168      continue;
1169    }
1170
1171    if (coinbase_blob_size < cumulative_size - txs_size)
1172    {
1173      size_t delta = cumulative_size - txs_size - coinbase_blob_size;
1174#if defined(DEBUG_CREATE_BLOCK_TEMPLATE)
1175      LOG_PRINT_L1("Creating block template: miner tx size " << coinbase_blob_size <<
1176          ", cumulative size " << txs_size + coinbase_blob_size <<
1177          " is less then before, adding " << delta << " zero bytes");
1178#endif
1179      b.miner_tx.extra.insert(b.miner_tx.extra.end(), delta, 0);
1180      //here  could be 1 byte difference, because of extra field counter is varint, and it can become from 1-byte len to 2-bytes len.
1181      if (cumulative_size != txs_size + get_object_blobsize(b.miner_tx))
1182      {
1183        CHECK_AND_ASSERT_MES(cumulative_size + 1 == txs_size + get_object_blobsize(b.miner_tx), false, "unexpected case: cumulative_size=" << cumulative_size << " + 1 is not equal txs_cumulative_size=" << txs_size << " + get_object_blobsize(b.miner_tx)=" << get_object_blobsize(b.miner_tx));
1184        b.miner_tx.extra.resize(b.miner_tx.extra.size() - 1);
1185        if (cumulative_size != txs_size + get_object_blobsize(b.miner_tx))
1186        {
1187          //fuck, not lucky, -1 makes varint-counter size smaller, in that case we continue to grow with cumulative_size
1188          LOG_PRINT_RED("Miner tx creation has no luck with delta_extra size = " << delta << " and " << delta - 1 , LOG_LEVEL_2);
1189          cumulative_size += delta - 1;
1190          continue;
1191        }
1192        LOG_PRINT_GREEN("Setting extra for block: " << b.miner_tx.extra.size() << ", try_count=" << try_count, LOG_LEVEL_1);
1193      }
1194    }
1195    CHECK_AND_ASSERT_MES(cumulative_size == txs_size + get_object_blobsize(b.miner_tx), false, "unexpected case: cumulative_size=" << cumulative_size << " is not equal txs_cumulative_size=" << txs_size << " + get_object_blobsize(b.miner_tx)=" << get_object_blobsize(b.miner_tx));
1196#if defined(DEBUG_CREATE_BLOCK_TEMPLATE)
1197    LOG_PRINT_L1("Creating block template: miner tx size " << coinbase_blob_size <<
1198        ", cumulative size " << cumulative_size << " is now good");
1199#endif
1200    return true;
1201  }
1202  LOG_ERROR("Failed to create_block_template with " << 10 << " tries");
1203  return false;
1204}
1205//------------------------------------------------------------------
1206// for an alternate chain, get the timestamps from the main chain to complete
1207// the needed number of timestamps for the BLOCKCHAIN_TIMESTAMP_CHECK_WINDOW.
1208bool Blockchain::complete_timestamps_vector(uint64_t start_top_height, std::vector<uint64_t>& timestamps)
1209{
1210  LOG_PRINT_L3("Blockchain::" << __func__);
1211  size_t blockchain_timestamp_check_window = get_current_hard_fork_version() < 2 ? 
1212                                                BLOCKCHAIN_TIMESTAMP_CHECK_WINDOW : 
1213                                                BLOCKCHAIN_TIMESTAMP_CHECK_WINDOW_V2;
1214  if (timestamps.size() >= blockchain_timestamp_check_window)
1215    return true;
1216
1217  CRITICAL_REGION_LOCAL(m_blockchain_lock);
1218  size_t need_elements = blockchain_timestamp_check_window - timestamps.size();
1219  CHECK_AND_ASSERT_MES(start_top_height < m_db->height(), false, "internal error: passed start_height not < " << " m_db->height() -- " << start_top_height << " >= " << m_db->height());
1220  size_t stop_offset = start_top_height > need_elements ? start_top_height - need_elements : 0;
1221  while (start_top_height != stop_offset)
1222  {
1223    timestamps.push_back(m_db->get_block_timestamp(start_top_height));
1224    --start_top_height;
1225  }
1226  return true;
1227}
1228//------------------------------------------------------------------
1229// If a block is to be added and its parent block is not the current
1230// main chain top block, then we need to see if we know about its parent block.
1231// If its parent block is part of a known forked chain, then we need to see
1232// if that chain is long enough to become the main chain and re-org accordingly
1233// if so.  If not, we need to hang on to the block in case it becomes part of
1234// a long forked chain eventually.
1235bool Blockchain::handle_alternative_block(const block& b, const crypto::hash& id, block_verification_context& bvc)
1236{
1237  LOG_PRINT_L3("Blockchain::" << __func__);
1238  CRITICAL_REGION_LOCAL(m_blockchain_lock);
1239  m_timestamps_and_difficulties_height = 0;
1240  uint64_t block_height = get_block_height(b);
1241  if(0 == block_height)
1242  {
1243    LOG_PRINT_L1("Block with id: " << epee::string_tools::pod_to_hex(id) << " (as alternative), but miner tx says height is 0.");
1244    bvc.m_verifivation_failed = true;
1245    return false;
1246  }
1247  // this basically says if the blockchain is smaller than the first
1248  // checkpoint then alternate blocks are allowed.  Alternatively, if the
1249  // last checkpoint *before* the end of the current chain is also before
1250  // the block to be added, then this is fine.
1251  if (!m_checkpoints.is_alternative_block_allowed(get_current_blockchain_height(), block_height))
1252  {
1253    LOG_PRINT_RED_L1("Block with id: " << id << std::endl << " can't be accepted for alternative chain, block height: " << block_height << std::endl << " blockchain height: " << get_current_blockchain_height());
1254    bvc.m_verifivation_failed = true;
1255    return false;
1256  }
1257
1258  //block is not related with head of main chain
1259  //first of all - look in alternative chains container
1260  auto it_prev = m_alternative_chains.find(b.prev_id);
1261  bool parent_in_main = m_db->block_exists(b.prev_id);
1262  if(it_prev != m_alternative_chains.end() || parent_in_main)
1263  {
1264    //we have new block in alternative chain
1265
1266    //build alternative subchain, front -> mainchain, back -> alternative head
1267    blocks_ext_by_hash::iterator alt_it = it_prev; //m_alternative_chains.find()
1268    std::list<blocks_ext_by_hash::iterator> alt_chain;
1269    std::vector<uint64_t> timestamps;
1270    while(alt_it != m_alternative_chains.end())
1271    {
1272      alt_chain.push_front(alt_it);
1273      timestamps.push_back(alt_it->second.bl.timestamp);
1274      alt_it = m_alternative_chains.find(alt_it->second.bl.prev_id);
1275    }
1276
1277    // if block to be added connects to known blocks that aren't part of the
1278    // main chain -- that is, if we're adding on to an alternate chain
1279    if(alt_chain.size())
1280    {
1281      // make sure alt chain doesn't somehow start past the end of the main chain
1282      CHECK_AND_ASSERT_MES(m_db->height() > alt_chain.front()->second.height, false, "main blockchain wrong height");
1283
1284      // make sure that the blockchain contains the block that should connect
1285      // this alternate chain with it.
1286      if (!m_db->block_exists(alt_chain.front()->second.bl.prev_id))
1287      {
1288        LOG_PRINT_L1("alternate chain does not appear to connect to main chain...");
1289        return false;
1290      }
1291
1292      // make sure block connects correctly to the main chain
1293      auto h = m_db->get_block_hash_from_height(alt_chain.front()->second.height - 1);
1294      CHECK_AND_ASSERT_MES(h == alt_chain.front()->second.bl.prev_id, false, "alternative chain has wrong connection to main chain");
1295      complete_timestamps_vector(m_db->get_block_height(alt_chain.front()->second.bl.prev_id), timestamps);
1296    }
1297    // if block not associated with known alternate chain
1298    else
1299    {
1300      // if block parent is not part of main chain or an alternate chain,
1301      // we ignore it
1302      CHECK_AND_ASSERT_MES(parent_in_main, false, "internal error: broken imperative condition: parent_in_main");
1303
1304      complete_timestamps_vector(m_db->get_block_height(b.prev_id), timestamps);
1305    }
1306
1307    // verify that the block's timestamp is within the acceptable range
1308    // (not earlier than the median of the last X blocks)
1309    if(!check_block_timestamp(timestamps, b))
1310    {
1311      LOG_PRINT_RED_L1("Block with id: " << id << std::endl << " for alternative chain, has invalid timestamp: " << b.timestamp);
1312      bvc.m_verifivation_failed = true;
1313      return false;
1314    }
1315
1316    // FIXME: consider moving away from block_extended_info at some point
1317    block_extended_info bei = boost::value_initialized<block_extended_info>();
1318    bei.bl = b;
1319    bei.height = alt_chain.size() ? it_prev->second.height + 1 : m_db->get_block_height(b.prev_id) + 1;
1320
1321    bool is_a_checkpoint;
1322    if(!m_checkpoints.check_block(bei.height, id, is_a_checkpoint))
1323    {
1324      LOG_ERROR("CHECKPOINT VALIDATION FAILED");
1325      bvc.m_verifivation_failed = true;
1326      return false;
1327    }
1328
1329    // Check the block's hash against the difficulty target for its alt chain
1330    m_is_in_checkpoint_zone = false;
1331    difficulty_type current_diff = get_next_difficulty_for_alternative_chain(alt_chain, bei);
1332    CHECK_AND_ASSERT_MES(current_diff, false, "!!!!!!! DIFFICULTY OVERHEAD !!!!!!!");
1333    crypto::hash proof_of_work = null_hash;
1334	get_block_longhash(bei.bl, m_pow_ctx, proof_of_work);
1335    if(!check_hash(proof_of_work, current_diff))
1336    {
1337      LOG_PRINT_RED_L1("Block with id: " << id << std::endl << " for alternative chain, does not have enough proof of work: " << proof_of_work << std::endl << " expected difficulty: " << current_diff);
1338      bvc.m_verifivation_failed = true;
1339      return false;
1340    }
1341
1342    if(!prevalidate_miner_transaction(b, bei.height))
1343    {
1344      LOG_PRINT_RED_L1("Block with id: " << epee::string_tools::pod_to_hex(id) << " (as alternative) has incorrect miner transaction.");
1345      bvc.m_verifivation_failed = true;
1346      return false;
1347    }
1348
1349    // FIXME:
1350    // this brings up an interesting point: consider allowing to get block
1351    // difficulty both by height OR by hash, not just height.
1352    difficulty_type main_chain_cumulative_difficulty = m_db->get_block_cumulative_difficulty(m_db->height() - 1);
1353    if (alt_chain.size())
1354    {
1355      bei.cumulative_difficulty = it_prev->second.cumulative_difficulty;
1356    }
1357    else
1358    {
1359      // passed-in block's previous block's cumulative difficulty, found on the main chain
1360      bei.cumulative_difficulty = m_db->get_block_cumulative_difficulty(m_db->get_block_height(b.prev_id));
1361    }
1362    bei.cumulative_difficulty += current_diff;
1363
1364    // add block to alternate blocks storage,
1365    // as well as the current "alt chain" container
1366    auto i_res = m_alternative_chains.insert(blocks_ext_by_hash::value_type(id, bei));
1367    CHECK_AND_ASSERT_MES(i_res.second, false, "insertion of new alternative block returned as it already exist");
1368    alt_chain.push_back(i_res.first);
1369
1370    // FIXME: is it even possible for a checkpoint to show up not on the main chain?
1371    if(is_a_checkpoint)
1372    {
1373      //do reorganize!
1374      LOG_PRINT_GREEN("###### REORGANIZE on height: " << alt_chain.front()->second.height << " of " << m_db->height() - 1 << ", checkpoint is found in alternative chain on height " << bei.height, LOG_LEVEL_0);
1375
1376      bool r = switch_to_alternative_blockchain(alt_chain, true);
1377
1378      if(r) bvc.m_added_to_main_chain = true;
1379      else bvc.m_verifivation_failed = true;
1380
1381      return r;
1382    }
1383    else if(main_chain_cumulative_difficulty < bei.cumulative_difficulty) //check if difficulty bigger then in main chain
1384    {
1385      //do reorganize!
1386      LOG_PRINT_GREEN("###### REORGANIZE on height: " << alt_chain.front()->second.height << " of " << m_db->height() - 1 << " with cum_difficulty " << m_db->get_block_cumulative_difficulty(m_db->height() - 1) << std::endl << " alternative blockchain size: " << alt_chain.size() << " with cum_difficulty " << bei.cumulative_difficulty, LOG_LEVEL_0);
1387
1388      bool r = switch_to_alternative_blockchain(alt_chain, false);
1389      if (r)
1390        bvc.m_added_to_main_chain = true;
1391      else
1392        bvc.m_verifivation_failed = true;
1393      return r;
1394    }
1395    else
1396    {
1397      LOG_PRINT_BLUE("----- BLOCK ADDED AS ALTERNATIVE ON HEIGHT " << bei.height << std::endl << "id:\t" << id << std::endl << "PoW:\t" << proof_of_work << std::endl << "difficulty:\t" << current_diff, LOG_LEVEL_0);
1398      return true;
1399    }
1400  }
1401  else
1402  {
1403    //block orphaned
1404    bvc.m_marked_as_orphaned = true;
1405    LOG_PRINT_RED_L1("Block recognized as orphaned and rejected, id = " << id);
1406  }
1407
1408  return true;
1409}
1410//------------------------------------------------------------------
1411bool Blockchain::get_blocks(uint64_t start_offset, size_t count, std::list<block>& blocks, std::list<transaction>& txs) const
1412{
1413  LOG_PRINT_L3("Blockchain::" << __func__);
1414  CRITICAL_REGION_LOCAL(m_blockchain_lock);
1415  if(start_offset > m_db->height())
1416    return false;
1417
1418  if (!get_blocks(start_offset, count, blocks))
1419  {
1420    return false;
1421  }
1422
1423  for(const block& blk : blocks)
1424  {
1425    std::list<crypto::hash> missed_ids;
1426    get_transactions(blk.tx_hashes, txs, missed_ids);
1427    CHECK_AND_ASSERT_MES(!missed_ids.size(), false, "has missed transactions in own block in main blockchain");
1428  }
1429
1430  return true;
1431}
1432//------------------------------------------------------------------
1433bool Blockchain::get_blocks(uint64_t start_offset, size_t count, std::list<block>& blocks) const
1434{
1435  LOG_PRINT_L3("Blockchain::" << __func__);
1436  CRITICAL_REGION_LOCAL(m_blockchain_lock);
1437  if(start_offset > m_db->height())
1438    return false;
1439
1440  for(size_t i = start_offset; i < start_offset + count && i < m_db->height();i++)
1441  {
1442    blocks.push_back(m_db->get_block_from_height(i));
1443  }
1444  return true;
1445}
1446//------------------------------------------------------------------
1447//TODO: This function *looks* like it won't need to be rewritten
1448//      to use BlockchainDB, as it calls other functions that were,
1449//      but it warrants some looking into later.
1450//
1451//FIXME: This function appears to want to return false if any transactions
1452//       that belong with blocks are missing, but not if blocks themselves
1453//       are missing.
1454bool Blockchain::handle_get_objects(NOTIFY_REQUEST_GET_OBJECTS::request& arg, NOTIFY_RESPONSE_GET_OBJECTS::request& rsp)
1455{
1456  LOG_PRINT_L3("Blockchain::" << __func__);
1457  CRITICAL_REGION_LOCAL(m_blockchain_lock);
1458  m_db->block_txn_start(true);
1459  rsp.current_blockchain_height = get_current_blockchain_height();
1460  std::list<block> blocks;
1461  get_blocks(arg.blocks, blocks, rsp.missed_ids);
1462
1463  for (const auto& bl: blocks)
1464  {
1465    std::list<crypto::hash> missed_tx_ids;
1466    std::list<transaction> txs;
1467
1468    // FIXME: s/rsp.missed_ids/missed_tx_id/ ?  Seems like rsp.missed_ids
1469    //        is for missed blocks, not missed transactions as well.
1470    get_transactions(bl.tx_hashes, txs, missed_tx_ids);
1471
1472    if (missed_tx_ids.size() != 0)
1473    {
1474      LOG_ERROR("Error retrieving blocks, missed " << missed_tx_ids.size()
1475          << " transactions for block with hash: " << get_block_hash(bl)
1476          << std::endl
1477      );
1478
1479      // append missed transaction hashes to response missed_ids field,
1480      // as done below if any standalone transactions were requested
1481      // and missed.
1482      rsp.missed_ids.splice(rsp.missed_ids.end(), missed_tx_ids);
1483	  m_db->block_txn_stop();
1484      return false;
1485    }
1486
1487    rsp.blocks.push_back(block_complete_entry());
1488    block_complete_entry& e = rsp.blocks.back();
1489    //pack block
1490    e.block = t_serializable_object_to_blob(bl);
1491    //pack transactions
1492    for (transaction& tx: txs)
1493      e.txs.push_back(t_serializable_object_to_blob(tx));
1494  }
1495  //get another transactions, if need
1496  std::list<transaction> txs;
1497  get_transactions(arg.txs, txs, rsp.missed_ids);
1498  //pack aside transactions
1499  for (const auto& tx: txs)
1500    rsp.txs.push_back(t_serializable_object_to_blob(tx));
1501
1502  m_db->block_txn_stop();
1503  return true;
1504}
1505//------------------------------------------------------------------
1506bool Blockchain::get_alternative_blocks(std::list<block>& blocks) const
1507{
1508  LOG_PRINT_L3("Blockchain::" << __func__);
1509  CRITICAL_REGION_LOCAL(m_blockchain_lock);
1510
1511  for (const auto& alt_bl: m_alternative_chains)
1512  {
1513    blocks.push_back(alt_bl.second.bl);
1514  }
1515  return true;
1516}
1517//------------------------------------------------------------------
1518size_t Blockchain::get_alternative_blocks_count() const
1519{
1520  LOG_PRINT_L3("Blockchain::" << __func__);
1521  CRITICAL_REGION_LOCAL(m_blockchain_lock);
1522  return m_alternative_chains.size();
1523}
1524//------------------------------------------------------------------
1525// This function adds the output specified by <amount, i> to the result_outs container
1526// unlocked and other such checks should be done by here.
1527void Blockchain::add_out_to_get_random_outs(COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::outs_for_amount& result_outs, uint64_t amount, size_t i) const
1528{
1529  LOG_PRINT_L3("Blockchain::" << __func__);
1530  CRITICAL_REGION_LOCAL(m_blockchain_lock);
1531
1532  COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::out_entry& oen = *result_outs.outs.insert(result_outs.outs.end(), COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::out_entry());
1533  oen.global_amount_index = i;
1534  output_data_t data = m_db->get_output_key(amount, i);
1535  oen.out_key = data.pubkey;
1536}
1537//------------------------------------------------------------------
1538// This function takes an RPC request for mixins and creates an RPC response
1539// with the requested mixins.
1540// TODO: figure out why this returns boolean / if we should be returning false
1541// in some cases
1542bool Blockchain::get_random_outs_for_amounts(const COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::request& req, COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::response& res) const
1543{
1544  LOG_PRINT_L3("Blockchain::" << __func__);
1545  CRITICAL_REGION_LOCAL(m_blockchain_lock);
1546
1547  // for each amount that we need to get mixins for, get <n> random outputs
1548  // from BlockchainDB where <n> is req.outs_count (number of mixins).
1549  for (uint64_t amount : req.amounts)
1550  {
1551    auto num_outs = m_db->get_num_outputs(amount);
1552    // ensure we don't include outputs that aren't yet eligible to be used
1553    // outpouts are sorted by height
1554    while (num_outs > 0)
1555    {
1556      const tx_out_index toi = m_db->get_output_tx_and_index(amount, num_outs - 1);
1557      const uint64_t height = m_db->get_tx_block_height(toi.first);
1558      if (height + CRYPTONOTE_DEFAULT_TX_SPENDABLE_AGE <= m_db->height())
1559        break;
1560      --num_outs;
1561    }
1562
1563    // create outs_for_amount struct and populate amount field
1564    COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::outs_for_amount& result_outs = *res.outs.insert(res.outs.end(), COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::outs_for_amount());
1565    result_outs.amount = amount;
1566
1567    std::unordered_set<uint64_t> seen_indices;
1568
1569    // if there aren't enough outputs to mix with (or just enough),
1570    // use all of them.  Eventually this should become impossible.
1571    if (num_outs <= req.outs_count)
1572    {
1573      for (uint64_t i = 0; i < num_outs; i++)
1574      {
1575        // get tx_hash, tx_out_index from DB
1576        tx_out_index toi = m_db->get_output_tx_and_index(amount, i);
1577
1578        // if tx is unlocked, add output to result_outs
1579        if (is_tx_spendtime_unlocked(m_db->get_tx_unlock_time(toi.first)))
1580        {
1581          add_out_to_get_random_outs(result_outs, amount, i);
1582        }
1583
1584      }
1585    }
1586    else
1587    {
1588      // while we still need more mixins
1589      while (result_outs.outs.size() < req.outs_count)
1590      {
1591        // if we've gone through every possible output, we've gotten all we can
1592        if (seen_indices.size() == num_outs)
1593        {
1594          break;
1595        }
1596
1597        // get a random output index from the DB.  If we've already seen it,
1598        // return to the top of the loop and try again, otherwise add it to the
1599        // list of output indices we've seen.
1600
1601        // triangular distribution over [a,b) with a=0, mode c=b=up_index_limit
1602        uint64_t r = crypto::rand<uint64_t>() % ((uint64_t)1 << 53);
1603        double frac = std::sqrt((double)r / ((uint64_t)1 << 53));
1604        uint64_t i = (uint64_t)(frac*num_outs);
1605        // just in case rounding up to 1 occurs after sqrt
1606        if (i == num_outs)
1607          --i;
1608
1609        if (seen_indices.count(i))
1610        {
1611          continue;
1612        }
1613        seen_indices.emplace(i);
1614
1615        // get tx_hash, tx_out_index from DB
1616        tx_out_index toi = m_db->get_output_tx_and_index(amount, i);
1617
1618        // if the output's transaction is unlocked, add the output's index to
1619        // our list.
1620        if (is_tx_spendtime_unlocked(m_db->get_tx_unlock_time(toi.first)))
1621        {
1622          add_out_to_get_random_outs(result_outs, amount, i);
1623        }
1624      }
1625    }
1626  }
1627  return true;
1628}
1629//------------------------------------------------------------------
1630// This function adds the ringct output at index i to the list
1631// unlocked and other such checks should be done by here.
1632void Blockchain::add_out_to_get_rct_random_outs(std::list<COMMAND_RPC_GET_RANDOM_RCT_OUTPUTS::out_entry>& outs, uint64_t amount, size_t i) const
1633{
1634  LOG_PRINT_L3("Blockchain::" << __func__);
1635  CRITICAL_REGION_LOCAL(m_blockchain_lock);
1636
1637  COMMAND_RPC_GET_RANDOM_RCT_OUTPUTS::out_entry& oen = *outs.insert(outs.end(), COMMAND_RPC_GET_RANDOM_RCT_OUTPUTS::out_entry());
1638  oen.amount = amount;
1639  oen.global_amount_index = i;
1640  output_data_t data = m_db->get_output_key(amount, i);
1641  oen.out_key = data.pubkey;
1642  oen.commitment = data.commitment;
1643}
1644//------------------------------------------------------------------
1645// This function takes an RPC request for mixins and creates an RPC response
1646// with the requested mixins.
1647// TODO: figure out why this returns boolean / if we should be returning false
1648// in some cases
1649bool Blockchain::get_random_rct_outs(const COMMAND_RPC_GET_RANDOM_RCT_OUTPUTS::request& req, COMMAND_RPC_GET_RANDOM_RCT_OUTPUTS::response& res) const
1650{
1651  LOG_PRINT_L3("Blockchain::" << __func__);
1652  CRITICAL_REGION_LOCAL(m_blockchain_lock);
1653
1654  // for each amount that we need to get mixins for, get <n> random outputs
1655  // from BlockchainDB where <n> is req.outs_count (number of mixins).
1656  auto num_outs = m_db->get_num_outputs(0);
1657  // ensure we don't include outputs that aren't yet eligible to be used
1658  // outpouts are sorted by height
1659  while (num_outs > 0)
1660  {
1661    const tx_out_index toi = m_db->get_output_tx_and_index(0, num_outs - 1);
1662    const uint64_t height = m_db->get_tx_block_height(toi.first);
1663    if (height + CRYPTONOTE_DEFAULT_TX_SPENDABLE_AGE <= m_db->height())
1664      break;
1665    --num_outs;
1666  }
1667
1668  std::unordered_set<uint64_t> seen_indices;
1669
1670  // if there aren't enough outputs to mix with (or just enough),
1671  // use all of them.  Eventually this should become impossible.
1672  if (num_outs <= req.outs_count)
1673  {
1674    for (uint64_t i = 0; i < num_outs; i++)
1675    {
1676      // get tx_hash, tx_out_index from DB
1677      tx_out_index toi = m_db->get_output_tx_and_index(0, i);
1678
1679      // if tx is unlocked, add output to result_outs
1680      if (is_tx_spendtime_unlocked(m_db->get_tx_unlock_time(toi.first)))
1681      {
1682        add_out_to_get_rct_random_outs(res.outs, 0, i);
1683      }
1684    }
1685  }
1686  else
1687  {
1688    // while we still need more mixins
1689    while (res.outs.size() < req.outs_count)
1690    {
1691      // if we've gone through every possible output, we've gotten all we can
1692      if (seen_indices.size() == num_outs)
1693      {
1694        break;
1695      }
1696
1697      // get a random output index from the DB.  If we've already seen it,
1698      // return to the top of the loop and try again, otherwise add it to the
1699      // list of output indices we've seen.
1700
1701      // triangular distribution over [a,b) with a=0, mode c=b=up_index_limit
1702      uint64_t r = crypto::rand<uint64_t>() % ((uint64_t)1 << 53);
1703      double frac = std::sqrt((double)r / ((uint64_t)1 << 53));
1704      uint64_t i = (uint64_t)(frac*num_outs);
1705      // just in case rounding up to 1 occurs after sqrt
1706      if (i == num_outs)
1707        --i;
1708
1709      if (seen_indices.count(i))
1710      {
1711        continue;
1712      }
1713      seen_indices.emplace(i);
1714
1715      // get tx_hash, tx_out_index from DB
1716      tx_out_index toi = m_db->get_output_tx_and_index(0, i);
1717
1718      // if the output's transaction is unlocked, add the output's index to
1719      // our list.
1720      if (is_tx_spendtime_unlocked(m_db->get_tx_unlock_time(toi.first)))
1721      {
1722        add_out_to_get_rct_random_outs(res.outs, 0, i);
1723      }
1724    }
1725  }
1726
1727  if (res.outs.size() < req.outs_count)
1728    return false;
1729#if 0
1730  // if we do not have enough RCT inputs, we can pick from the non RCT ones
1731  // which will have a zero mask
1732  if (res.outs.size() < req.outs_count)
1733  {
1734    LOG_PRINT_L0("Out of RCT inputs (" << res.outs.size() << "/" << req.outs_count << "), using regular ones");
1735
1736    // TODO: arbitrary selection, needs better
1737    COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::request req2 = AUTO_VAL_INIT(req2);
1738    COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::response res2 = AUTO_VAL_INIT(res2);
1739    req2.outs_count = req.outs_count - res.outs.size();
1740    static const uint64_t amounts[] = {1, 10, 20, 50, 100, 200, 500, 1000, 10000};
1741    for (uint64_t a: amounts)
1742      req2.amounts.push_back(a);
1743    if (!get_random_outs_for_amounts(req2, res2))
1744      return false;
1745
1746    // pick random ones from there
1747    while (res.outs.size() < req.outs_count)
1748    {
1749      int list_idx = rand() % (sizeof(amounts)/sizeof(amounts[0]));
1750      if (!res2.outs[list_idx].outs.empty())
1751      {
1752        const COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::out_entry oe = res2.outs[list_idx].outs.back();
1753        res2.outs[list_idx].outs.pop_back();
1754        add_out_to_get_rct_random_outs(res.outs, res2.outs[list_idx].amount, oe.global_amount_index);
1755      }
1756    }
1757  }
1758#endif
1759
1760  return true;
1761}
1762//------------------------------------------------------------------
1763bool Blockchain::get_outs(const COMMAND_RPC_GET_OUTPUTS_BIN::request& req, COMMAND_RPC_GET_OUTPUTS_BIN::response& res) const
1764{
1765  LOG_PRINT_L3("Blockchain::" << __func__);
1766  CRITICAL_REGION_LOCAL(m_blockchain_lock);
1767
1768  res.outs.clear();
1769  res.outs.reserve(req.outputs.size());
1770  for (const auto &i: req.outputs)
1771  {
1772    // get tx_hash, tx_out_index from DB
1773    const output_data_t od = m_db->get_output_key(i.amount, i.index);
1774    tx_out_index toi = m_db->get_output_tx_and_index(i.amount, i.index);
1775    bool unlocked = is_tx_spendtime_unlocked(m_db->get_tx_unlock_time(toi.first));
1776
1777    res.outs.push_back({od.pubkey, od.commitment, unlocked});
1778  }
1779  return true;
1780}
1781//------------------------------------------------------------------
1782// This function takes a list of block hashes from another node
1783// on the network to find where the split point is between us and them.
1784// This is used to see what to send another node that needs to sync.
1785bool Blockchain::find_blockchain_supplement(const std::list<crypto::hash>& qblock_ids, uint64_t& starter_offset) const
1786{
1787  LOG_PRINT_L3("Blockchain::" << __func__);
1788  CRITICAL_REGION_LOCAL(m_blockchain_lock);
1789
1790  // make sure the request includes at least the genesis block, otherwise
1791  // how can we expect to sync from the client that the block list came from?
1792  if(!qblock_ids.size() /*|| !req.m_total_height*/)
1793  {
1794    LOG_PRINT_L1("Client sent wrong NOTIFY_REQUEST_CHAIN: m_block_ids.size()=" << qblock_ids.size() << /*", m_height=" << req.m_total_height <<*/ ", dropping connection");
1795    return false;
1796  }
1797
1798  m_db->block_txn_start(true);
1799  // make sure that the last block in the request's block list matches
1800  // the genesis block
1801  auto gen_hash = m_db->get_block_hash_from_height(0);
1802  if(qblock_ids.back() != gen_hash)
1803  {
1804    LOG_PRINT_L1("Client sent wrong NOTIFY_REQUEST_CHAIN: genesis block mismatch: " << std::endl << "id: " << qblock_ids.back() << ", " << std::endl << "expected: " << gen_hash << "," << std::endl << " dropping connection");
1805	m_db->block_txn_abort();
1806    return false;
1807  }
1808
1809  // Find the first block the foreign chain has that we also have.
1810  // Assume qblock_ids is in reverse-chronological order.
1811  auto bl_it = qblock_ids.begin();
1812  uint64_t split_height = 0;
1813  for(; bl_it != qblock_ids.end(); bl_it++)
1814  {
1815    try
1816    {
1817      if (m_db->block_exists(*bl_it, &split_height))
1818        break;
1819    }
1820    catch (const std::exception& e)
1821    {
1822      LOG_PRINT_L1("Non-critical error trying to find block by hash in BlockchainDB, hash: " << *bl_it);
1823	  m_db->block_txn_abort();
1824      return false;
1825    }
1826  }
1827  m_db->block_txn_stop();
1828
1829  // this should be impossible, as we checked that we share the genesis block,
1830  // but just in case...
1831  if(bl_it == qblock_ids.end())
1832  {
1833    LOG_PRINT_L1("Internal error handling connection, can't find split point");
1834    return false;
1835  }
1836
1837  //we start to put block ids INCLUDING last known id, just to make other side be sure
1838  starter_offset = split_height;
1839  return true;
1840}
1841//------------------------------------------------------------------
1842uint64_t Blockchain::block_difficulty(uint64_t i) const
1843{
1844  LOG_PRINT_L3("Blockchain::" << __func__);
1845  CRITICAL_REGION_LOCAL(m_blockchain_lock);
1846  try
1847  {
1848    return m_db->get_block_difficulty(i);
1849  }
1850  catch (const BLOCK_DNE& e)
1851  {
1852    LOG_PRINT_L0("Attempted to get block difficulty for height above blockchain height");
1853  }
1854  return 0;
1855}
1856//------------------------------------------------------------------
1857//TODO: return type should be void, throw on exception
1858//       alternatively, return true only if no blocks missed
1859template<class t_ids_container, class t_blocks_container, class t_missed_container>
1860bool Blockchain::get_blocks(const t_ids_container& block_ids, t_blocks_container& blocks, t_missed_container& missed_bs) const
1861{
1862  LOG_PRINT_L3("Blockchain::" << __func__);
1863  CRITICAL_REGION_LOCAL(m_blockchain_lock);
1864
1865  for (const auto& block_hash : block_ids)
1866  {
1867    try
1868    {
1869      blocks.push_back(m_db->get_block(block_hash));
1870    }
1871    catch (const BLOCK_DNE& e)
1872    {
1873      missed_bs.push_back(block_hash);
1874    }
1875    catch (const std::exception& e)
1876    {
1877      return false;
1878    }
1879  }
1880  return true;
1881}
1882//------------------------------------------------------------------
1883//TODO: return type should be void, throw on exception
1884//       alternatively, return true only if no transactions missed
1885template<class t_ids_container, class t_tx_container, class t_missed_container>
1886bool Blockchain::get_transactions(const t_ids_container& txs_ids, t_tx_container& txs, t_missed_container& missed_txs) const
1887{
1888  LOG_PRINT_L3("Blockchain::" << __func__);
1889  CRITICAL_REGION_LOCAL(m_blockchain_lock);
1890
1891  for (const auto& tx_hash : txs_ids)
1892  {
1893    try
1894    {
1895      txs.push_back(m_db->get_tx(tx_hash));
1896    }
1897    catch (const TX_DNE& e)
1898    {
1899      missed_txs.push_back(tx_hash);
1900    }
1901    catch (const std::exception& e)
1902    {
1903      return false;
1904    }
1905  }
1906  return true;
1907}
1908//------------------------------------------------------------------
1909void Blockchain::print_blockchain(uint64_t start_index, uint64_t end_index) const
1910{
1911  LOG_PRINT_L3("Blockchain::" << __func__);
1912  std::stringstream ss;
1913  CRITICAL_REGION_LOCAL(m_blockchain_lock);
1914  auto h = m_db->height();
1915  if(start_index > h)
1916  {
1917    LOG_PRINT_L1("Wrong starter index set: " << start_index << ", expected max index " << h);
1918    return;
1919  }
1920
1921  for(size_t i = start_index; i <= h && i != end_index; i++)
1922  {
1923    ss << "height " << i << ", timestamp " << m_db->get_block_timestamp(i) << ", cumul_dif " << m_db->get_block_cumulative_difficulty(i) << ", size " << m_db->get_block_size(i) << "\nid\t\t" << m_db->get_block_hash_from_height(i) << "\ndifficulty\t\t" << m_db->get_block_difficulty(i) << ", nonce " << m_db->get_block_from_height(i).nonce << ", tx_count " << m_db->get_block_from_height(i).tx_hashes.size() << std::endl;
1924  }
1925  LOG_PRINT_L1("Current blockchain:" << std::endl << ss.str());
1926  LOG_PRINT_L0("Blockchain printed with log level 1");
1927}
1928//------------------------------------------------------------------
1929void Blockchain::print_blockchain_index() const
1930{
1931  LOG_PRINT_L3("Blockchain::" << __func__);
1932  std::stringstream ss;
1933  CRITICAL_REGION_LOCAL(m_blockchain_lock);
1934  auto height = m_db->height();
1935  if (height != 0)
1936  {
1937    for(uint64_t i = 0; i <= height; i++)
1938    {
1939      ss << "height: " << i << ", hash: " << m_db->get_block_hash_from_height(i);
1940    }
1941  }
1942
1943  LOG_PRINT_L0("Current blockchain index:" << std::endl << ss.str());
1944}
1945//------------------------------------------------------------------
1946//TODO: remove this function and references to it
1947void Blockchain::print_blockchain_outs(const std::string& file) const
1948{
1949  LOG_PRINT_L3("Blockchain::" << __func__);
1950  return;
1951}
1952//------------------------------------------------------------------
1953// Find the split point between us and foreign blockchain and return
1954// (by reference) the most recent common block hash along with up to
1955// BLOCKS_IDS_SYNCHRONIZING_DEFAULT_COUNT additional (more recent) hashes.
1956bool Blockchain::find_blockchain_supplement(const std::list<crypto::hash>& qblock_ids, NOTIFY_RESPONSE_CHAIN_ENTRY::request& resp) const
1957{
1958  LOG_PRINT_L3("Blockchain::" << __func__);
1959  CRITICAL_REGION_LOCAL(m_blockchain_lock);
1960
1961  // if we can't find the split point, return false
1962  if(!find_blockchain_supplement(qblock_ids, resp.start_height))
1963  {
1964    return false;
1965  }
1966
1967  resp.total_height = get_current_blockchain_height();
1968  size_t count = 0;
1969  for(size_t i = resp.start_height; i < resp.total_height && count < BLOCKS_IDS_SYNCHRONIZING_DEFAULT_COUNT; i++, count++)
1970  {
1971    resp.m_block_ids.push_back(m_db->get_block_hash_from_height(i));
1972  }
1973  return true;
1974}
1975//------------------------------------------------------------------
1976//FIXME: change argument to std::vector, low priority
1977// find split point between ours and foreign blockchain (or start at
1978// blockchain height <req_start_block>), and return up to max_count FULL
1979// blocks by reference.
1980bool Blockchain::find_blockchain_supplement(const uint64_t req_start_block, const std::list<crypto::hash>& qblock_ids, std::list<std::pair<block, std::list<transaction> > >& blocks, uint64_t& total_height, uint64_t& start_height, size_t max_count) const
1981{
1982  LOG_PRINT_L3("Blockchain::" << __func__);
1983  CRITICAL_REGION_LOCAL(m_blockchain_lock);
1984
1985  // if a specific start height has been requested
1986  if(req_start_block > 0)
1987  {
1988    // if requested height is higher than our chain, return false -- we can't help
1989    if (req_start_block >= m_db->height())
1990    {
1991      return false;
1992    }
1993    start_height = req_start_block;
1994  }
1995  else
1996  {
1997    if(!find_blockchain_supplement(qblock_ids, start_height))
1998    {
1999      return false;
2000    }
2001  }
2002
2003  total_height = get_current_blockchain_height();
2004  size_t count = 0;
2005  for(size_t i = start_height; i < total_height && count < max_count; i++, count++)
2006  {
2007    blocks.resize(blocks.size()+1);
2008    blocks.back().first = m_db->get_block_from_height(i);
2009    std::list<crypto::hash> mis;
2010    get_transactions(blocks.back().first.tx_hashes, blocks.back().second, mis);
2011    CHECK_AND_ASSERT_MES(!mis.size(), false, "internal error, transaction from block not found");
2012  }
2013  return true;
2014}
2015//------------------------------------------------------------------
2016bool Blockchain::add_block_as_invalid(const block& bl, const crypto::hash& h)
2017{
2018  LOG_PRINT_L3("Blockchain::" << __func__);
2019  block_extended_info bei = AUTO_VAL_INIT(bei);
2020  bei.bl = bl;
2021  return add_block_as_invalid(bei, h);
2022}
2023//------------------------------------------------------------------
2024bool Blockchain::add_block_as_invalid(const block_extended_info& bei, const crypto::hash& h)
2025{
2026  LOG_PRINT_L3("Blockchain::" << __func__);
2027  CRITICAL_REGION_LOCAL(m_blockchain_lock);
2028  auto i_res = m_invalid_blocks.insert(std::map<crypto::hash, block_extended_info>::value_type(h, bei));
2029  CHECK_AND_ASSERT_MES(i_res.second, false, "at insertion invalid by tx returned status existed");
2030  LOG_PRINT_L1("BLOCK ADDED AS INVALID: " << h << std::endl << ", prev_id=" << bei.bl.prev_id << ", m_invalid_blocks count=" << m_invalid_blocks.size());
2031  return true;
2032}
2033//------------------------------------------------------------------
2034bool Blockchain::have_block(const crypto::hash& id) const
2035{
2036  LOG_PRINT_L3("Blockchain::" << __func__);
2037  CRITICAL_REGION_LOCAL(m_blockchain_lock);
2038
2039  if(m_db->block_exists(id))
2040  {
2041    LOG_PRINT_L3("block exists in main chain");
2042    return true;
2043  }
2044
2045  if(m_alternative_chains.count(id))
2046  {
2047    LOG_PRINT_L3("block found in m_alternative_chains");
2048    return true;
2049  }
2050
2051  if(m_invalid_blocks.count(id))
2052  {
2053    LOG_PRINT_L3("block found in m_invalid_blocks");
2054    return true;
2055  }
2056
2057  return false;
2058}
2059//------------------------------------------------------------------
2060bool Blockchain::handle_block_to_main_chain(const block& bl, block_verification_context& bvc)
2061{
2062    LOG_PRINT_L3("Blockchain::" << __func__);
2063    crypto::hash id = get_block_hash(bl);
2064    return handle_block_to_main_chain(bl, id, bvc);
2065}
2066//------------------------------------------------------------------
2067size_t Blockchain::get_total_transactions() const
2068{
2069  LOG_PRINT_L3("Blockchain::" << __func__);
2070  CRITICAL_REGION_LOCAL(m_blockchain_lock);
2071  return m_db->get_tx_count();
2072}
2073//------------------------------------------------------------------
2074// This function checks each input in the transaction <tx> to make sure it
2075// has not been used already, and adds its key to the container <keys_this_block>.
2076//
2077// This container should be managed by the code that validates blocks so we don't
2078// have to store the used keys in a given block in the permanent storage only to
2079// remove them later if the block fails validation.
2080bool Blockchain::check_for_double_spend(const transaction& tx, key_images_container& keys_this_block) const
2081{
2082  LOG_PRINT_L3("Blockchain::" << __func__);
2083  CRITICAL_REGION_LOCAL(m_blockchain_lock);
2084  struct add_transaction_input_visitor: public boost::static_visitor<bool>
2085  {
2086    key_images_container& m_spent_keys;
2087    BlockchainDB* m_db;
2088    add_transaction_input_visitor(key_images_container& spent_keys, BlockchainDB* db) :
2089      m_spent_keys(spent_keys), m_db(db)
2090    {
2091    }
2092    bool operator()(const txin_to_key& in) const
2093    {
2094      const crypto::key_image& ki = in.k_image;
2095
2096      // attempt to insert the newly-spent key into the container of
2097      // keys spent this block.  If this fails, the key was spent already
2098      // in this block, return false to flag that a double spend was detected.
2099      //
2100      // if the insert into the block-wide spent keys container succeeds,
2101      // check the blockchain-wide spent keys container and make sure the
2102      // key wasn't used in another block already.
2103      auto r = m_spent_keys.insert(ki);
2104      if(!r.second || m_db->has_key_image(ki))
2105      {
2106        //double spend detected
2107        return false;
2108      }
2109
2110      // if no double-spend detected, return true
2111      return true;
2112    }
2113
2114    bool operator()(const txin_gen& tx) const
2115    {
2116      return true;
2117    }
2118    bool operator()(const txin_to_script& tx) const
2119    {
2120      return false;
2121    }
2122    bool operator()(const txin_to_scripthash& tx) const
2123    {
2124      return false;
2125    }
2126  };
2127
2128  for (const txin_v& in : tx.vin)
2129  {
2130    if(!boost::apply_visitor(add_transaction_input_visitor(keys_this_block, m_db), in))
2131    {
2132      LOG_ERROR("Double spend detected!");
2133      return false;
2134    }
2135  }
2136
2137  return true;
2138}
2139//------------------------------------------------------------------
2140bool Blockchain::get_tx_outputs_gindexs(const crypto::hash& tx_id, std::vector<uint64_t>& indexs) const
2141{
2142  LOG_PRINT_L3("Blockchain::" << __func__);
2143  CRITICAL_REGION_LOCAL(m_blockchain_lock);
2144  uint64_t tx_index;
2145  if (!m_db->tx_exists(tx_id, tx_index))
2146  {
2147    LOG_PRINT_RED_L1("warning: get_tx_outputs_gindexs failed to find transaction with id = " << tx_id);
2148    return false;
2149  }
2150
2151  // get amount output indexes, currently referred to in parts as "output global indices", but they are actually specific to amounts
2152  indexs = m_db->get_tx_amount_output_indices(tx_index);
2153  if (indexs.empty())
2154  {
2155    // empty indexs is only valid if the vout is empty, which is legal but rare
2156    cryptonote::transaction tx = m_db->get_tx(tx_id);
2157    CHECK_AND_ASSERT_MES(tx.vout.empty(), false, "internal error: global indexes for transaction " << tx_id << " is empty, and tx vout is not");
2158  }
2159
2160  return true;
2161}
2162//------------------------------------------------------------------
2163//FIXME: it seems this function is meant to be merely a wrapper around
2164//       another function of the same name, this one adding one bit of
2165//       functionality.  Should probably move anything more than that
2166//       (getting the hash of the block at height max_used_block_id)
2167//       to the other function to keep everything in one place.
2168// This function overloads its sister function with
2169// an extra value (hash of highest block that holds an output used as input)
2170// as a return-by-reference.
2171bool Blockchain::check_tx_inputs(transaction& tx, uint64_t& max_used_block_height, crypto::hash& max_used_block_id, tx_verification_context &tvc, bool kept_by_block)
2172{
2173  LOG_PRINT_L3("Blockchain::" << __func__);
2174  CRITICAL_REGION_LOCAL(m_blockchain_lock);
2175
2176#if defined(PER_BLOCK_CHECKPOINT)
2177  // check if we're doing per-block checkpointing
2178  // FIXME: investigate why this block returns
2179  if (m_db->height() < m_blocks_hash_check.size() && kept_by_block)
2180  {
2181    TIME_MEASURE_START(a);
2182    m_blocks_txs_check.push_back(get_transaction_hash(tx));
2183    max_used_block_id = null_hash;
2184    max_used_block_height = 0;
2185    TIME_MEASURE_FINISH(a);
2186    if(m_show_time_stats)
2187      LOG_PRINT_L0("HASH: " << "-" << " VIN/VOUT: " << tx.vin.size() << "/" << tx.vout.size() << " H: " << 0 << " chcktx: " << a);
2188    return true;
2189  }
2190#endif
2191
2192  TIME_MEASURE_START(a);
2193  bool res = check_tx_inputs(tx, tvc, &max_used_block_height);
2194  TIME_MEASURE_FINISH(a);
2195  if(m_show_time_stats)
2196    LOG_PRINT_L0("HASH: " << "+" << " VIN/VOUT: " << tx.vin.size() << "/" << tx.vout.size() << " H: " << max_used_block_height << " chcktx: " << a + m_fake_scan_time);
2197
2198  if (!res)
2199    return false;
2200
2201  CHECK_AND_ASSERT_MES(max_used_block_height < m_db->height(), false,  "internal error: max used block index=" << max_used_block_height << " is not less then blockchain size = " << m_db->height());
2202  max_used_block_id = m_db->get_block_hash_from_height(max_used_block_height);
2203  return true;
2204}
2205//------------------------------------------------------------------
2206bool Blockchain::check_tx_outputs(const transaction& tx, tx_verification_context &tvc)
2207{
2208  LOG_PRINT_L3("Blockchain::" << __func__);
2209  CRITICAL_REGION_LOCAL(m_blockchain_lock);
2210
2211  // in a v2 tx, all outputs must have 0 amount
2212  for (auto &o: tx.vout) {
2213    if (o.amount != 0) {
2214      tvc.m_invalid_output = true;
2215      return false;
2216    }
2217  }
2218  
2219  // from v4, forbid invalid pubkeys
2220  for (const auto &o: tx.vout) {
2221    if (o.target.type() == typeid(txout_to_key)) {
2222      const txout_to_key& out_to_key = boost::get<txout_to_key>(o.target);
2223      if (!crypto::check_key(out_to_key.key)) {
2224        tvc.m_invalid_output = true;
2225        return false;
2226      }
2227    }
2228  }
2229  
2230  return true;
2231}
2232//------------------------------------------------------------------
2233bool Blockchain::have_tx_keyimges_as_spent(const transaction &tx) const
2234{
2235  LOG_PRINT_L3("Blockchain::" << __func__);
2236  for (const txin_v& in: tx.vin)
2237  {
2238    CHECKED_GET_SPECIFIC_VARIANT(in, const txin_to_key, in_to_key, true);
2239    if(have_tx_keyimg_as_spent(in_to_key.k_image))
2240      return true;
2241  }
2242  return false;
2243}
2244bool Blockchain::expand_transaction_2(transaction &tx, const crypto::hash &tx_prefix_hash, const std::vector<std::vector<rct::ctkey>> &pubkeys)
2245{
2246  PERF_TIMER(expand_transaction_2);
2247  CHECK_AND_ASSERT_MES(tx.version == 2, false, "Transaction version is not 2");
2248
2249  rct::rctSig &rv = tx.rct_signatures;
2250
2251  // message - hash of the transaction prefix
2252  rv.message = rct::hash2rct(tx_prefix_hash);
2253
2254  // mixRing - full and simple store it in opposite ways
2255  if (rv.type == rct::RCTTypeFull)
2256  {
2257    rv.mixRing.resize(pubkeys[0].size());
2258    for (size_t m = 0; m < pubkeys[0].size(); ++m)
2259      rv.mixRing[m].clear();
2260    for (size_t n = 0; n < pubkeys.size(); ++n)
2261    {
2262      CHECK_AND_ASSERT_MES(pubkeys[n].size() <= pubkeys[0].size(), false, "More inputs that first ring");
2263      for (size_t m = 0; m < pubkeys[n].size(); ++m)
2264      {
2265        rv.mixRing[m].push_back(pubkeys[n][m]);
2266      }
2267    }
2268  }
2269  else if (rv.type == rct::RCTTypeSimple)
2270  {
2271    rv.mixRing.resize(pubkeys.size());
2272    for (size_t n = 0; n < pubkeys.size(); ++n)
2273    {
2274      rv.mixRing[n].clear();
2275      for (size_t m = 0; m < pubkeys[n].size(); ++m)
2276      {
2277        rv.mixRing[n].push_back(pubkeys[n][m]);
2278      }
2279    }
2280  }
2281  else
2282  {
2283    CHECK_AND_ASSERT_MES(false, false, "Unsupported rct tx type: " + boost::lexical_cast<std::string>(rv.type));
2284  }
2285
2286  // II
2287  if (rv.type == rct::RCTTypeFull)
2288  {
2289    rv.p.MGs.resize(1);
2290    rv.p.MGs[0].II.resize(tx.vin.size());
2291    for (size_t n = 0; n < tx.vin.size(); ++n)
2292      rv.p.MGs[0].II[n] = rct::ki2rct(boost::get<txin_to_key>(tx.vin[n]).k_image);
2293  }
2294  else if (rv.type == rct::RCTTypeSimple)
2295  {
2296    CHECK_AND_ASSERT_MES(rv.p.MGs.size() == tx.vin.size(), false, "Bad MGs size");
2297    for (size_t n = 0; n < tx.vin.size(); ++n)
2298    {
2299      rv.p.MGs[n].II.resize(1);
2300      rv.p.MGs[n].II[0] = rct::ki2rct(boost::get<txin_to_key>(tx.vin[n]).k_image);
2301    }
2302  }
2303  else
2304  {
2305    CHECK_AND_ASSERT_MES(false, false, "Unsupported rct tx type: " + boost::lexical_cast<std::string>(rv.type));
2306  }
2307
2308  // outPk
2309  CHECK_AND_ASSERT_MES(rv.outPk.size() == tx.vout.size(), false, "Bad outPk size");
2310  for (size_t n = 0; n < tx.rct_signatures.outPk.size(); ++n)
2311    rv.outPk[n].dest = rct::pk2rct(boost::get<txout_to_key>(tx.vout[n].target).key);
2312
2313  return true;
2314}
2315//------------------------------------------------------------------
2316// This function validates transaction inputs and their keys.
2317// FIXME: consider moving functionality specific to one input into
2318//        check_tx_input() rather than here, and use this function simply
2319//        to iterate the inputs as necessary (splitting the task
2320//        using threads, etc.)
2321bool Blockchain::check_tx_inputs(transaction& tx, tx_verification_context &tvc, uint64_t* pmax_used_block_height)
2322{
2323  PERF_TIMER(check_tx_inputs);
2324  LOG_PRINT_L3("Blockchain::" << __func__);
2325  size_t sig_index = 0;
2326  if(pmax_used_block_height)
2327    *pmax_used_block_height = 0;
2328
2329  crypto::hash tx_prefix_hash = get_transaction_prefix_hash(tx);
2330
2331  // from hard fork 2, we require mixin at least 2 unless one output cannot mix with 2 others
2332  // if one output cannot mix with 2 others, we accept at most 1 output that can mix
2333  size_t n_unmixable = 0, n_mixable = 0;
2334  size_t mixin = std::numeric_limits<size_t>::max();
2335  //const size_t min_mixin = hf_version >= 5 ? 4 : 2;
2336  const size_t min_mixin = DEFAULT_MIXIN;
2337  const size_t max_mixin = MAX_MIXIN;
2338  for (const auto& txin : tx.vin)
2339  {
2340    // non txin_to_key inputs will be rejected below
2341    if (txin.type() == typeid(txin_to_key))
2342    {
2343      const txin_to_key& in_to_key = boost::get<txin_to_key>(txin);
2344      if (in_to_key.amount == 0)
2345      {
2346        // always consider rct inputs mixable. Even if there's not enough rct
2347        // inputs on the chain to mix with, this is going to be the case for
2348        // just a few blocks right after the fork at most
2349        ++n_mixable;
2350      }
2351      else
2352      {
2353        uint64_t n_outputs = m_db->get_num_outputs(in_to_key.amount);
2354        LOG_PRINT_L2("output size " << print_money(in_to_key.amount) << ": " << n_outputs << " available");
2355        // n_outputs includes the output we're considering
2356        if (n_outputs <= min_mixin)
2357          ++n_unmixable;
2358        else
2359          ++n_mixable;
2360      }
2361      if (in_to_key.key_offsets.size() - 1 < mixin)
2362        mixin = in_to_key.key_offsets.size() - 1;
2363    }
2364  }
2365
2366  if (mixin < min_mixin)
2367  {
2368    if (n_unmixable == 0)
2369    {
2370      LOG_PRINT_L1("Tx " << get_transaction_hash(tx) << " has too low mixin (" << mixin << "), and no unmixable inputs");
2371      tvc.m_low_mixin = true;
2372      return false;
2373    }
2374    if (n_mixable > 1)
2375    {
2376      LOG_PRINT_L1("Tx " << get_transaction_hash(tx) << " has too low mixin (" << mixin << "), and more than one mixable input with unmixable inputs");
2377      tvc.m_low_mixin = true;
2378      return false;
2379    }
2380  }
2381
2382  if (mixin > max_mixin)
2383  {
2384    LOG_PRINT_L1("Tx " << get_transaction_hash(tx) << " has too high mixin (" << mixin << "), max mixin = " << max_mixin);
2385    tvc.m_high_mixin = true;
2386    return false;
2387  }
2388  
2389
2390  // min/max tx version based on HF, and we accept v1 txes if having a non mixable
2391  const size_t max_tx_version = CURRENT_TRANSACTION_VERSION;
2392  if (tx.version > max_tx_version)
2393  {
2394    LOG_PRINT_L1("transaction version " << (unsigned)tx.version << " is higher than max accepted version " << max_tx_version);
2395    tvc.m_verifivation_failed = true;
2396    return false;
2397  }
2398  const size_t min_tx_version = MIN_TRANSACTION_VERSION;
2399  if (tx.version < min_tx_version)
2400  {
2401    LOG_PRINT_L1("transaction version " << (unsigned)tx.version << " is lower than min accepted version " << min_tx_version);
2402    tvc.m_verifivation_failed = true;
2403    return false;
2404  }
2405  
2406
2407  auto it = m_check_txin_table.find(tx_prefix_hash);
2408  if(it == m_check_txin_table.end())
2409  {
2410    m_check_txin_table.emplace(tx_prefix_hash, std::unordered_map<crypto::key_image, bool>());
2411    it = m_check_txin_table.find(tx_prefix_hash);
2412    assert(it != m_check_txin_table.end());
2413  }
2414
2415  uint64_t t_t1 = 0;
2416  std::vector<std::vector<rct::ctkey>> pubkeys(tx.vin.size());
2417  std::vector < uint64_t > results;
2418  results.resize(tx.vin.size(), 0);
2419
2420  int threads = tools::get_max_concurrency();
2421
2422  boost::asio::io_service ioservice;
2423  boost::thread_group threadpool;
2424  bool ioservice_active = false;
2425
2426  std::unique_ptr < boost::asio::io_service::work > work(new boost::asio::io_service::work(ioservice));
2427  if(threads > 1)
2428  {
2429    for (int i = 0; i < threads; i++)
2430    {
2431      threadpool.create_thread(boost::bind(&boost::asio::io_service::run, &ioservice));
2432    }
2433    ioservice_active = true;
2434  }
2435
2436#define KILL_IOSERVICE()  \
2437        if(ioservice_active) \
2438        { \
2439            work.reset(); \
2440            while (!ioservice.stopped()) ioservice.poll(); \
2441            threadpool.join_all(); \
2442            ioservice.stop(); \
2443            ioservice_active = false; \
2444        }
2445
2446  epee::misc_utils::auto_scope_leave_caller ioservice_killer = epee::misc_utils::create_scope_leave_handler([&]() { KILL_IOSERVICE(); });
2447
2448  for (const auto& txin : tx.vin)
2449  {
2450    // make sure output being spent is of type txin_to_key, rather than
2451    // e.g. txin_gen, which is only used for miner transactions
2452    CHECK_AND_ASSERT_MES(txin.type() == typeid(txin_to_key), false, "wrong type id in tx input at Blockchain::check_tx_inputs");
2453    const txin_to_key& in_to_key = boost::get<txin_to_key>(txin);
2454
2455    // make sure tx output has key offset(s) (is signed to be used)
2456    CHECK_AND_ASSERT_MES(in_to_key.key_offsets.size(), false, "empty in_to_key.key_offsets in transaction with id " << get_transaction_hash(tx));
2457
2458    if(have_tx_keyimg_as_spent(in_to_key.k_image))
2459    {
2460      LOG_PRINT_L1("Key image already spent in blockchain: " << epee::string_tools::pod_to_hex(in_to_key.k_image));
2461      tvc.m_double_spend = true;
2462      return false;
2463    }
2464
2465    if (tx.version == 1)
2466    {
2467      // basically, make sure number of inputs == number of signatures
2468      CHECK_AND_ASSERT_MES(sig_index < tx.signatures.size(), false, "wrong transaction: not signature entry for input with index= " << sig_index);
2469
2470#if defined(CACHE_VIN_RESULTS)
2471      auto itk = it->second.find(in_to_key.k_image);
2472      if(itk != it->second.end())
2473      {
2474        if(!itk->second)
2475        {
2476          LOG_PRINT_L1("Failed ring signature for tx " << get_transaction_hash(tx) << "  vin key with k_image: " << in_to_key.k_image << "  sig_index: " << sig_index);
2477          return false;
2478        }
2479
2480        // txin has been verified already, skip
2481        sig_index++;
2482        continue;
2483      }
2484#endif
2485    }
2486
2487    // make sure that output being spent matches up correctly with the
2488    // signature spending it.
2489    if (!check_tx_input(tx.version, in_to_key, tx_prefix_hash, std::vector<crypto::signature>(), tx.rct_signatures, pubkeys[sig_index], pmax_used_block_height))
2490    {
2491      it->second[in_to_key.k_image] = false;
2492      LOG_PRINT_L1("Failed to check ring signature for tx " << get_transaction_hash(tx) << "  vin key with k_image: " << in_to_key.k_image << "  sig_index: " << sig_index);
2493      if (pmax_used_block_height) // a default value of NULL is used when called from Blockchain::handle_block_to_main_chain()
2494      {
2495        LOG_PRINT_L1("  *pmax_used_block_height: " << *pmax_used_block_height);
2496      }
2497
2498      return false;
2499    }
2500
2501    sig_index++;
2502  }
2503
2504  KILL_IOSERVICE();
2505
2506  if (!expand_transaction_2(tx, tx_prefix_hash, pubkeys))
2507  {
2508    LOG_PRINT_L1("Failed to expand rct signatures!");
2509    return false;
2510  }
2511
2512  // from version 2, check ringct signatures
2513  // obviously, the original and simple rct APIs use a mixRing that's indexes
2514  // in opposite orders, because it'd be too simple otherwise...
2515  const rct::rctSig &rv = tx.rct_signatures;
2516  switch (rv.type)
2517  {
2518  case rct::RCTTypeNull: {
2519    // we only accept no signatures for coinbase txes
2520    LOG_PRINT_L1("Null rct signature on non-coinbase tx");
2521    return false;
2522  }
2523  case rct::RCTTypeSimple: {
2524    // check all this, either recontructed (so should really pass), or not
2525    {
2526      if (pubkeys.size() != rv.mixRing.size())
2527      {
2528        LOG_PRINT_L1("Failed to check ringct signatures: mismatched pubkeys/mixRing size");
2529        return false;
2530      }
2531      for (size_t i = 0; i < pubkeys.size(); ++i)
2532      {
2533        if (pubkeys[i].size() != rv.mixRing[i].size())
2534        {
2535          LOG_PRINT_L1("Failed to check ringct signatures: mismatched pubkeys/mixRing size");
2536          return false;
2537        }
2538      }
2539
2540      for (size_t n = 0; n < pubkeys.size(); ++n)
2541      {
2542        for (size_t m = 0; m < pubkeys[n].size(); ++m)
2543        {
2544          if (pubkeys[n][m].dest != rct::rct2pk(rv.mixRing[n][m].dest))
2545          {
2546            LOG_PRINT_L1("Failed to check ringct signatures: mismatched pubkey at vin " << n << ", index " << m);
2547            return false;
2548          }
2549          if (pubkeys[n][m].mask != rct::rct2pk(rv.mixRing[n][m].mask))
2550          {
2551            LOG_PRINT_L1("Failed to check ringct signatures: mismatched commitment at vin " << n << ", index " << m);
2552            return false;
2553          }
2554        }
2555      }
2556    }
2557
2558    if (rv.p.MGs.size() != tx.vin.size())
2559    {
2560      LOG_PRINT_L1("Failed to check ringct signatures: mismatched MGs/vin sizes");
2561      return false;
2562    }
2563    for (size_t n = 0; n < tx.vin.size(); ++n)
2564    {
2565      if (memcmp(&boost::get<txin_to_key>(tx.vin[n]).k_image, &rv.p.MGs[n].II[0], 32))
2566      {
2567        LOG_PRINT_L1("Failed to check ringct signatures: mismatched key image");
2568        return false;
2569      }
2570    }
2571
2572    if (!rct::verRctSimple(rv))
2573    {
2574      LOG_PRINT_L1("Failed to check ringct signatures!");
2575      return false;
2576    }
2577    break;
2578  }
2579  case rct::RCTTypeFull: {
2580    // check all this, either recontructed (so should really pass), or not
2581    {
2582      bool size_matches = true;
2583      for (size_t i = 0; i < pubkeys.size(); ++i)
2584        size_matches &= pubkeys[i].size() == rv.mixRing.size();
2585      for (size_t i = 0; i < rv.mixRing.size(); ++i)
2586        size_matches &= pubkeys.size() == rv.mixRing[i].size();
2587      if (!size_matches)
2588      {
2589        LOG_PRINT_L1("Failed to check ringct signatures: mismatched pubkeys/mixRing size");
2590        return false;
2591      }
2592
2593      for (size_t n = 0; n < pubkeys.size(); ++n)
2594      {
2595        for (size_t m = 0; m < pubkeys[n].size(); ++m)
2596        {
2597          if (pubkeys[n][m].dest != rct::rct2pk(rv.mixRing[m][n].dest))
2598          {
2599            LOG_PRINT_L1("Failed to check ringct signatures: mismatched pubkey at vin " << n << ", index " << m);
2600            return false;
2601          }
2602          if (pubkeys[n][m].mask != rct::rct2pk(rv.mixRing[m][n].mask))
2603          {
2604            LOG_PRINT_L1("Failed to check ringct signatures: mismatched commitment at vin " << n << ", index " << m);
2605            return false;
2606          }
2607        }
2608      }
2609    }
2610
2611    if (rv.p.MGs.size() != 1)
2612    {
2613      LOG_PRINT_L1("Failed to check ringct signatures: Bad MGs size");
2614      return false;
2615    }
2616    if (rv.p.MGs[0].II.size() != tx.vin.size())
2617    {
2618      LOG_PRINT_L1("Failed to check ringct signatures: mismatched II/vin sizes");
2619      return false;
2620    }
2621    for (size_t n = 0; n < tx.vin.size(); ++n)
2622    {
2623      if (memcmp(&boost::get<txin_to_key>(tx.vin[n]).k_image, &rv.p.MGs[0].II[n], 32))
2624      {
2625        LOG_PRINT_L1("Failed to check ringct signatures: mismatched II/vin sizes");
2626        return false;
2627      }
2628    }
2629
2630    if (!rct::verRct(rv))
2631    {
2632      LOG_PRINT_L1("Failed to check ringct signatures!");
2633      return false;
2634    }
2635    break;
2636  }
2637  default:
2638    LOG_PRINT_L1("Unsupported rct type: " << rv.type);
2639    return false;
2640  }
2641  
2642  return true;
2643}
2644
2645//------------------------------------------------------------------
2646void Blockchain::check_ring_signature(const crypto::hash &tx_prefix_hash, const crypto::key_image &key_image, const std::vector<rct::ctkey> &pubkeys, const std::vector<crypto::signature>& sig, uint64_t &result)
2647{
2648  if (m_is_in_checkpoint_zone)
2649  {
2650    result = true;
2651    return;
2652  }
2653
2654  std::vector<const crypto::public_key *> p_output_keys;
2655  for (auto &key : pubkeys)
2656  {
2657    // rct::key and crypto::public_key have the same structure, avoid object ctor/memcpy
2658    p_output_keys.push_back(&(const crypto::public_key&)key.dest);
2659  }
2660
2661  result = crypto::check_ring_signature(tx_prefix_hash, key_image, p_output_keys, sig.data()) ? 1 : 0;
2662}
2663
2664//------------------------------------------------------------------
2665uint64_t Blockchain::get_dynamic_per_kb_fee(uint64_t block_reward, size_t median_block_size)
2666{
2667  if (median_block_size < BLOCK_SIZE_GROWTH_FAVORED_ZONE)
2668    median_block_size = BLOCK_SIZE_GROWTH_FAVORED_ZONE;
2669  
2670  // this to avoid full block fee getting too low when block reward decline, i.e. easier for "block filler" attack
2671  if (block_reward < DYNAMIC_FEE_PER_KB_BASE_BLOCK_REWARD)
2672    block_reward = DYNAMIC_FEE_PER_KB_BASE_BLOCK_REWARD;
2673
2674  uint64_t unscaled_fee_per_kb = (DYNAMIC_FEE_PER_KB_BASE_FEE * BLOCK_SIZE_GROWTH_FAVORED_ZONE / median_block_size);
2675  uint64_t hi, lo = mul128(unscaled_fee_per_kb, block_reward, &hi);
2676  static_assert(DYNAMIC_FEE_PER_KB_BASE_BLOCK_REWARD % 1000000 == 0, "DYNAMIC_FEE_PER_KB_BASE_BLOCK_REWARD must be divisible by 1000000");
2677  static_assert(DYNAMIC_FEE_PER_KB_BASE_BLOCK_REWARD / 1000000 <= std::numeric_limits<uint32_t>::max(), "DYNAMIC_FEE_PER_KB_BASE_BLOCK_REWARD is too large");
2678  // divide in two steps, since the divisor must be 32 bits, but DYNAMIC_FEE_PER_KB_BASE_BLOCK_REWARD isn't
2679  div128_32(hi, lo, DYNAMIC_FEE_PER_KB_BASE_BLOCK_REWARD / 1000000, &hi, &lo);
2680  div128_32(hi, lo, 1000000, &hi, &lo);
2681  assert(hi == 0);
2682
2683  return lo;
2684}
2685
2686//------------------------------------------------------------------
2687bool Blockchain::check_fee(size_t blob_size, uint64_t fee) const
2688{
2689  uint64_t fee_per_kb;
2690  uint64_t median = m_current_block_cumul_sz_limit / 2;
2691  uint64_t height = m_db->height();
2692  uint64_t cal_height = height - height % COIN_EMISSION_HEIGHT_INTERVAL;
2693  uint64_t cal_generated_coins = cal_height ? m_db->get_block_already_generated_coins(cal_height - 1) : 0;
2694  uint64_t base_reward;
2695  if (!get_block_reward(median, 1, cal_generated_coins, base_reward, height))
2696    return false;
2697  fee_per_kb = get_dynamic_per_kb_fee(base_reward, median);
2698  
2699  LOG_PRINT_L2("Using " << print_money(fee) << "/kB fee");
2700
2701  float kB = (blob_size - CRYPTONOTE_COINBASE_BLOB_RESERVED_SIZE) * 1.0f / 1024;
2702  uint64_t needed_fee = ((uint64_t)(kB * fee_per_kb)) / 100 * 100;
2703
2704  if (fee < needed_fee)
2705  {
2706    LOG_PRINT_L1("transaction fee is not enough: " << print_money(fee) << ", minimum fee: " << print_money(needed_fee));
2707    return false;
2708  }
2709  return true;
2710}
2711
2712//------------------------------------------------------------------
2713uint64_t Blockchain::get_dynamic_per_kb_fee_estimate(uint64_t grace_blocks) const
2714{
2715  if (grace_blocks >= CRYPTONOTE_REWARD_BLOCKS_WINDOW)
2716    grace_blocks = CRYPTONOTE_REWARD_BLOCKS_WINDOW - 1;
2717
2718  std::vector<size_t> sz;
2719  get_last_n_blocks_sizes(sz, CRYPTONOTE_REWARD_BLOCKS_WINDOW - grace_blocks);
2720  for (size_t i = 0; i < grace_blocks; ++i)
2721    sz.push_back(CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE);
2722
2723  uint64_t median = epee::misc_utils::median(sz);
2724  if(median <= CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE)
2725    median = CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE;
2726
2727  //uint64_t already_generated_coins = m_db->height() ? m_db->get_block_already_generated_coins(m_db->height() - 1) : 0;
2728  uint64_t height = m_db->height();
2729  uint64_t cal_height = height - height % COIN_EMISSION_HEIGHT_INTERVAL;
2730  uint64_t cal_generated_coins = cal_height ? m_db->get_block_already_generated_coins(cal_height - 1) : 0;
2731  uint64_t base_reward;
2732  if (!get_block_reward(median, 1, cal_generated_coins, base_reward, height))
2733  {
2734    LOG_PRINT_L1("Failed to determine block reward, using placeholder " << print_money(BLOCK_REWARD_OVERESTIMATE) << " as a high bound");
2735    base_reward = BLOCK_REWARD_OVERESTIMATE;
2736  }
2737
2738  uint64_t fee = get_dynamic_per_kb_fee(base_reward, median);
2739  LOG_PRINT_L2("Estimating " << grace_blocks << "-block fee at " << print_money(fee) << "/kB");
2740  return fee;
2741}
2742
2743//------------------------------------------------------------------
2744// This function checks to see if a tx is unlocked.  unlock_time is either
2745// a block index or a unix time.
2746bool Blockchain::is_tx_spendtime_unlocked(uint64_t unlock_time) const
2747{
2748  LOG_PRINT_L3("Blockchain::" << __func__);
2749  if(unlock_time < CRYPTONOTE_MAX_BLOCK_NUMBER)
2750  {
2751    // ND: Instead of calling get_current_blockchain_height(), call m_db->height()
2752    //    directly as get_current_blockchain_height() locks the recursive mutex.
2753    if(m_db->height()-1 + CRYPTONOTE_LOCKED_TX_ALLOWED_DELTA_BLOCKS >= unlock_time)
2754      return true;
2755    else
2756      return false;
2757  }
2758  else
2759  {
2760    //interpret as time
2761    uint64_t current_time = static_cast<uint64_t>(time(NULL));
2762    if(current_time + (CRYPTONOTE_LOCKED_TX_ALLOWED_DELTA_SECONDS) >= unlock_time)
2763      return true;
2764    else
2765      return false;
2766  }
2767  return false;
2768}
2769//------------------------------------------------------------------
2770// This function locates all outputs associated with a given input (mixins)
2771// and validates that they exist and are usable.  It also checks the ring
2772// signature for each input.
2773bool Blockchain::check_tx_input(size_t tx_version, const txin_to_key& txin, const crypto::hash& tx_prefix_hash, const std::vector<crypto::signature>& sig, const rct::rctSig &rct_signatures, std::vector<rct::ctkey> &output_keys, uint64_t* pmax_related_block_height)
2774{
2775  LOG_PRINT_L3("Blockchain::" << __func__);
2776
2777  // ND:
2778  // 1. Disable locking and make method private.
2779  //CRITICAL_REGION_LOCAL(m_blockchain_lock);
2780
2781  struct outputs_visitor
2782  {
2783    std::vector<rct::ctkey >& m_output_keys;
2784    const Blockchain& m_bch;
2785    outputs_visitor(std::vector<rct::ctkey>& output_keys, const Blockchain& bch) :
2786      m_output_keys(output_keys), m_bch(bch)
2787    {
2788    }
2789    bool handle_output(uint64_t unlock_time, const crypto::public_key &pubkey, const rct::key &commitment)
2790    {
2791      //check tx unlock time
2792      if (!m_bch.is_tx_spendtime_unlocked(unlock_time))
2793      {
2794        LOG_PRINT_L1("One of outputs for one of inputs has wrong tx.unlock_time = " << unlock_time);
2795        return false;
2796      }
2797
2798      // The original code includes a check for the output corresponding to this input
2799      // to be a txout_to_key. This is removed, as the database does not store this info,
2800      // but only txout_to_key outputs are stored in the DB in the first place, done in
2801      // Blockchain*::add_output
2802
2803      m_output_keys.push_back(rct::ctkey({rct::pk2rct(pubkey), commitment}));
2804      return true;
2805    }
2806  };
2807
2808  output_keys.clear();
2809
2810  // collect output keys
2811  outputs_visitor vi(output_keys, *this);
2812  if (!scan_outputkeys_for_indexes(tx_version, txin, vi, tx_prefix_hash, pmax_related_block_height))
2813  {
2814    LOG_PRINT_L1("Failed to get output keys for tx with amount = " << print_money(txin.amount) << " and count indexes " << txin.key_offsets.size());
2815    return false;
2816  }
2817
2818  if(txin.key_offsets.size() != output_keys.size())
2819  {
2820    LOG_PRINT_L1("Output keys for tx with amount = " << txin.amount << " and count indexes " << txin.key_offsets.size() << " returned wrong keys count " << output_keys.size());
2821    return false;
2822  }
2823  if (tx_version == 1) {
2824    CHECK_AND_ASSERT_MES(sig.size() == output_keys.size(), false, "internal error: tx signatures count=" << sig.size() << " mismatch with outputs keys count for inputs=" << output_keys.size());
2825  }
2826  // rct_signatures will be expanded after this
2827  return true;
2828}
2829//------------------------------------------------------------------
2830//TODO: Is this intended to do something else?  Need to look into the todo there.
2831uint64_t Blockchain::get_adjusted_time() const
2832{
2833  LOG_PRINT_L3("Blockchain::" << __func__);
2834  //TODO: add collecting median time
2835  return time(NULL);
2836}
2837//------------------------------------------------------------------
2838//TODO: revisit, has changed a bit on upstream
2839bool Blockchain::check_block_timestamp(std::vector<uint64_t>& timestamps, const block& b) const
2840{
2841  LOG_PRINT_L3("Blockchain::" << __func__);
2842  uint64_t median_ts = epee::misc_utils::median(timestamps);
2843  size_t blockchain_timestamp_check_window = get_current_hard_fork_version() < 2 ?
2844                                                BLOCKCHAIN_TIMESTAMP_CHECK_WINDOW :
2845                                                BLOCKCHAIN_TIMESTAMP_CHECK_WINDOW_V2;
2846  if(b.timestamp < median_ts)
2847  {
2848    LOG_PRINT_L1("Timestamp of block with id: " << get_block_hash(b) << ", " << b.timestamp << ", less than median of last " << blockchain_timestamp_check_window << " blocks, " << median_ts);
2849    return false;
2850  }
2851
2852  return true;
2853}
2854//------------------------------------------------------------------
2855// This function grabs the timestamps from the most recent <n> blocks,
2856// where n = BLOCKCHAIN_TIMESTAMP_CHECK_WINDOW.  If there are not those many
2857// blocks in the blockchain, the timestap is assumed to be valid.  If there
2858// are, this function returns:
2859//   true if the block's timestamp is not less than the timestamp of the
2860//       median of the selected blocks
2861//   false otherwise
2862bool Blockchain::check_block_timestamp(const block& b) const
2863{
2864  LOG_PRINT_L3("Blockchain::" << __func__);
2865  uint64_t block_future_time_limit = get_current_hard_fork_version() < 2 ? 
2866                                          CRYPTONOTE_BLOCK_FUTURE_TIME_LIMIT : 
2867                                          CRYPTONOTE_BLOCK_FUTURE_TIME_LIMIT_V2;
2868  size_t blockchain_timestamp_check_window = get_current_hard_fork_version() < 2 ?
2869                                                  BLOCKCHAIN_TIMESTAMP_CHECK_WINDOW :
2870                                                  BLOCKCHAIN_TIMESTAMP_CHECK_WINDOW_V2;
2871  if (b.timestamp > get_adjusted_time() + block_future_time_limit)
2872  {
2873    LOG_PRINT_L1("Timestamp of block with id: " << get_block_hash(b) << ", " << b.timestamp << 
2874      ", bigger than adjusted time + " << (get_current_hard_fork_version() < 2 ? "2 hours" : "30 minutes"));
2875    return false;
2876  }
2877
2878  // if not enough blocks, no proper median yet, return true
2879  if (m_db->height() < blockchain_timestamp_check_window)
2880  {
2881    return true;
2882  }
2883
2884  std::vector<uint64_t> timestamps;
2885  auto h = m_db->height();
2886
2887  // need most recent 60 blocks, get index of first of those
2888  size_t offset = h - blockchain_timestamp_check_window;
2889  for(;offset < h; ++offset)
2890  {
2891    timestamps.push_back(m_db->get_block_timestamp(offset));
2892  }
2893
2894  return check_block_timestamp(timestamps, b);
2895}
2896//------------------------------------------------------------------
2897void Blockchain::return_tx_to_pool(const std::vector<transaction> &txs)
2898{
2899  uint8_t version = get_current_hard_fork_version();
2900  for (auto& tx : txs)
2901  {
2902    cryptonote::tx_verification_context tvc = AUTO_VAL_INIT(tvc);
2903    // We assume that if they were in a block, the transactions are already
2904    // known to the network as a whole. However, if we had mined that block,
2905    // that might not be always true. Unlikely though, and always relaying
2906    // these again might cause a spike of traffic as many nodes re-relay
2907    // all the transactions in a popped block when a reorg happens.
2908    if (!m_tx_pool.add_tx(tx, tvc, true, true, version))
2909    {
2910      LOG_PRINT_L0("Failed to return taken transaction with hash: " << get_transaction_hash(tx) << " to tx_pool");
2911    }
2912  }
2913}
2914//------------------------------------------------------------------
2915bool Blockchain::flush_txes_from_pool(const std::list<crypto::hash> &txids)
2916{
2917  CRITICAL_REGION_LOCAL(m_tx_pool);
2918
2919  bool res = true;
2920  for (const auto &txid: txids)
2921  {
2922    cryptonote::transaction tx;
2923    size_t blob_size;
2924    uint64_t fee;
2925    bool relayed;
2926    LOG_PRINT_L1("Removing txid " << txid << " from the pool");
2927    if(m_tx_pool.have_tx(txid) && !m_tx_pool.take_tx(txid, tx, blob_size, fee, relayed))
2928    {
2929      LOG_PRINT_L0("Failed to remove txid " << txid << " from the pool");
2930      res = false;
2931    }
2932  }
2933  return res;
2934}
2935//------------------------------------------------------------------
2936//      Needs to validate the block and acquire each transaction from the
2937//      transaction mem_pool, then pass the block and transactions to
2938//      m_db->add_block()
2939bool Blockchain::handle_block_to_main_chain(const block& bl, const crypto::hash& id, block_verification_context& bvc)
2940{
2941  LOG_PRINT_L3("Blockchain::" << __func__);
2942
2943  TIME_MEASURE_START(block_processing_time);
2944  CRITICAL_REGION_LOCAL(m_blockchain_lock);
2945  TIME_MEASURE_START(t1);
2946
2947  m_db->block_txn_start(true);
2948  if(bl.prev_id != get_tail_id())
2949  {
2950    LOG_PRINT_L1("Block with id: " << id << std::endl << "has wrong prev_id: " << bl.prev_id << std::endl << "expected: " << get_tail_id());
2951leave:
2952    m_db->block_txn_stop();
2953    return false;
2954  }
2955
2956  // this is a cheap test
2957  if (!m_hardfork->check(bl))
2958  {
2959    LOG_PRINT_L1("Block with id: " << id << std::endl << "has old version: " << (unsigned)bl.major_version << std::endl << "current: " << (unsigned)m_hardfork->get_current_version());
2960    bvc.m_verifivation_failed = true;
2961    goto leave;
2962  }
2963
2964  TIME_MEASURE_FINISH(t1);
2965  TIME_MEASURE_START(t2);
2966
2967  // make sure block timestamp is not less than the median timestamp
2968  // of a set number of the most recent blocks.
2969  if(!check_block_timestamp(bl))
2970  {
2971    LOG_PRINT_L1("Block with id: " << id << std::endl << "has invalid timestamp: " << bl.timestamp);
2972    bvc.m_verifivation_failed = true;
2973    goto leave;
2974  }
2975
2976  TIME_MEASURE_FINISH(t2);
2977  //check proof of work
2978  TIME_MEASURE_START(target_calculating_time);
2979
2980  // get the target difficulty for the block.
2981  // the calculation can overflow, among other failure cases,
2982  // so we need to check the return type.
2983  // FIXME: get_difficulty_for_next_block can also assert, look into
2984  // changing this to throwing exceptions instead so we can clean up.
2985  difficulty_type current_diffic = get_difficulty_for_next_block();
2986  CHECK_AND_ASSERT_MES(current_diffic, false, "!!!!!!!!! difficulty overhead !!!!!!!!!");
2987
2988  TIME_MEASURE_FINISH(target_calculating_time);
2989
2990  TIME_MEASURE_START(longhash_calculating_time);
2991
2992  crypto::hash proof_of_work = null_hash;
2993
2994  // Formerly the code below contained an if loop with the following condition
2995  // !m_checkpoints.is_in_checkpoint_zone(get_current_blockchain_height())
2996  // however, this caused the daemon to not bother checking PoW for blocks
2997  // before checkpoints, which is very dangerous behaviour. We moved the PoW
2998  // validation out of the next chunk of code to make sure that we correctly
2999  // check PoW now.
3000  // FIXME: height parameter is not used...should it be used or should it not
3001  // be a parameter?
3002  // validate proof_of_work versus difficulty target
3003  bool precomputed = false;
3004  bool fast_check = false;
3005#if defined(PER_BLOCK_CHECKPOINT)
3006  if (m_db->height() < m_blocks_hash_check.size())
3007  {
3008    auto hash = get_block_hash(bl);
3009    if (memcmp(&hash, &m_blocks_hash_check[m_db->height()], sizeof(hash)) != 0)
3010    {
3011      LOG_PRINT_L1("Block with id is INVALID: " << id);
3012      bvc.m_verifivation_failed = true;
3013      goto leave;
3014    }
3015    fast_check = true;
3016  }
3017  else
3018#endif
3019  {
3020    auto it = m_blocks_longhash_table.find(id);
3021    if (it != m_blocks_longhash_table.end())
3022    {
3023      precomputed = true;
3024      proof_of_work = it->second;
3025    }
3026    else
3027	{
3028		get_block_longhash(bl, m_pow_ctx, proof_of_work);
3029	}
3030
3031    // validate proof_of_work versus difficulty target
3032    if(!check_hash(proof_of_work, current_diffic))
3033    {
3034      LOG_PRINT_L1("Block with id: " << id << std::endl << "does not have enough proof of work: " << proof_of_work << std::endl << "unexpected difficulty: " << current_diffic);
3035      bvc.m_verifivation_failed = true;
3036      goto leave;
3037    }
3038  }
3039
3040  // If we're at a checkpoint, ensure that our hardcoded checkpoint hash
3041  // is correct.
3042  if(m_checkpoints.is_in_checkpoint_zone(get_current_blockchain_height()))
3043  {
3044    if(!m_checkpoints.check_block(get_current_blockchain_height(), id))
3045    {
3046      LOG_ERROR("CHECKPOINT VALIDATION FAILED");
3047      bvc.m_verifivation_failed = true;
3048      goto leave;
3049    }
3050  }
3051
3052  TIME_MEASURE_FINISH(longhash_calculating_time);
3053  if (precomputed)
3054    longhash_calculating_time += m_fake_pow_calc_time;
3055
3056  TIME_MEASURE_START(t3);
3057
3058  // sanity check basic miner tx properties;
3059  if(!prevalidate_miner_transaction(bl, m_db->height()))
3060  {
3061    LOG_PRINT_L1("Block with id: " << id << " failed to pass prevalidation");
3062    bvc.m_verifivation_failed = true;
3063    goto leave;
3064  }
3065
3066  size_t coinbase_blob_size = get_object_blobsize(bl.miner_tx);
3067  size_t cumulative_block_size = coinbase_blob_size;
3068
3069  std::vector<transaction> txs;
3070  key_images_container keys;
3071
3072  uint64_t fee_summary = 0;
3073  uint64_t t_checktx = 0;
3074  uint64_t t_exists = 0;
3075  uint64_t t_pool = 0;
3076  uint64_t t_dblspnd = 0;
3077  TIME_MEASURE_FINISH(t3);
3078
3079// XXX old code adds miner tx here
3080
3081  int tx_index = 0;
3082  // Iterate over the block's transaction hashes, grabbing each
3083  // from the tx_pool and validating them.  Each is then added
3084  // to txs.  Keys spent in each are added to <keys> by the double spend check.
3085  for (const crypto::hash& tx_id : bl.tx_hashes)
3086  {
3087    transaction tx;
3088    size_t blob_size = 0;
3089    uint64_t fee = 0;
3090    bool relayed = false;
3091    TIME_MEASURE_START(aa);
3092
3093// XXX old code does not check whether tx exists
3094    if (m_db->tx_exists(tx_id))
3095    {
3096      LOG_PRINT_L1("Block with id: " << id << " attempting to add transaction already in blockchain with id: " << tx_id);
3097      bvc.m_verifivation_failed = true;
3098      return_tx_to_pool(txs);
3099      goto leave;
3100    }
3101
3102    TIME_MEASURE_FINISH(aa);
3103    t_exists += aa;
3104    TIME_MEASURE_START(bb);
3105
3106    // get transaction with hash <tx_id> from tx_pool
3107    if(!m_tx_pool.take_tx(tx_id, tx, blob_size, fee, relayed))
3108    {
3109      LOG_PRINT_L1("Block with id: " << id  << " has at least one unknown transaction with id: " << tx_id);
3110      bvc.m_verifivation_failed = true;
3111      return_tx_to_pool(txs);
3112      goto leave;
3113    }
3114
3115    TIME_MEASURE_FINISH(bb);
3116    t_pool += bb;
3117    // add the transaction to the temp list of transactions, so we can either
3118    // store the list of transactions all at once or return the ones we've
3119    // taken from the tx_pool back to it if the block fails verification.
3120    txs.push_back(tx);
3121    TIME_MEASURE_START(dd);
3122
3123    // FIXME: the storage should not be responsible for validation.
3124    //        If it does any, it is merely a sanity check.
3125    //        Validation is the purview of the Blockchain class
3126    //        - TW
3127    //
3128    // ND: this is not needed, db->add_block() checks for duplicate k_images and fails accordingly.
3129    // if (!check_for_double_spend(tx, keys))
3130    // {
3131    //     LOG_PRINT_L0("Double spend detected in transaction (id: " << tx_id);
3132    //     bvc.m_verifivation_failed = true;
3133    //     break;
3134    // }
3135
3136    TIME_MEASURE_FINISH(dd);
3137    t_dblspnd += dd;
3138    TIME_MEASURE_START(cc);
3139
3140#if defined(PER_BLOCK_CHECKPOINT)
3141    if (!fast_check)
3142#endif
3143    {
3144      // validate that transaction inputs and the keys spending them are correct.
3145      tx_verification_context tvc;
3146      if(!check_tx_inputs(tx, tvc))
3147      {
3148        LOG_PRINT_L1("Block with id: " << id  << " has at least one transaction (id: " << tx_id << ") with wrong inputs.");
3149
3150        //TODO: why is this done?  make sure that keeping invalid blocks makes sense.
3151        add_block_as_invalid(bl, id);
3152        LOG_PRINT_L1("Block with id " << id << " added as invalid because of wrong inputs in transactions");
3153        bvc.m_verifivation_failed = true;
3154        return_tx_to_pool(txs);
3155        goto leave;
3156      }
3157    }
3158#if defined(PER_BLOCK_CHECKPOINT)
3159    else
3160    {
3161      // ND: if fast_check is enabled for blocks, there is no need to check
3162      // the transaction inputs, but do some sanity checks anyway.
3163      if (memcmp(&m_blocks_txs_check[tx_index++], &tx_id, sizeof(tx_id)) != 0)
3164      {
3165        LOG_PRINT_L1("Block with id: " << id << " has at least one transaction (id: " << tx_id << ") with wrong inputs.");
3166        //TODO: why is this done?  make sure that keeping invalid blocks makes sense.
3167        add_block_as_invalid(bl, id);
3168        LOG_PRINT_L1("Block with id " << id << " added as invalid because of wrong inputs in transactions");
3169        bvc.m_verifivation_failed = true;
3170        return_tx_to_pool(txs);
3171        goto leave;
3172      }
3173    }
3174#endif
3175    TIME_MEASURE_FINISH(cc);
3176    t_checktx += cc;
3177    fee_summary += fee;
3178    cumulative_block_size += blob_size;
3179  }
3180
3181  m_blocks_txs_check.clear();
3182
3183  TIME_MEASURE_START(vmt);
3184  uint64_t base_reward = 0;
3185  uint64_t height = m_db->height();
3186  uint64_t cal_height = height - height % COIN_EMISSION_HEIGHT_INTERVAL;
3187  uint64_t cal_generated_coins = cal_height ? m_db->get_block_already_generated_coins(cal_height - 1) : 0;
3188  if (!validate_miner_transaction(bl, cumulative_block_size, fee_summary, base_reward, cal_generated_coins, bvc.m_partial_block_reward, m_hardfork->get_current_version()))
3189  {
3190    LOG_PRINT_L1("Block with id: " << id << " has incorrect miner transaction");
3191    bvc.m_verifivation_failed = true;
3192    return_tx_to_pool(txs);
3193    goto leave;
3194  }
3195
3196  TIME_MEASURE_FINISH(vmt);
3197  size_t block_size;
3198  difficulty_type cumulative_difficulty;
3199
3200  // populate various metadata about the block to be stored alongside it.
3201  block_size = cumulative_block_size;
3202  cumulative_difficulty = current_diffic;
3203  // In the "tail" state when the minimum subsidy (implemented in get_block_reward) is in effect, the number of
3204  // coins will eventually exceed MONEY_SUPPLY and overflow a uint64. To prevent overflow, cap already_generated_coins
3205  // at MONEY_SUPPLY. already_generated_coins is only used to compute the block subsidy and MONEY_SUPPLY yields a
3206  // subsidy of 0 under the base formula and therefore the minimum subsidy >0 in the tail state.
3207  uint64_t already_generated_coins = height ? m_db->get_block_already_generated_coins(height - 1) : 0;
3208  already_generated_coins = base_reward < (MONEY_SUPPLY-already_generated_coins) ? already_generated_coins + base_reward : MONEY_SUPPLY;
3209  if (height)
3210    cumulative_difficulty += m_db->get_block_cumulative_difficulty(height - 1);
3211
3212  TIME_MEASURE_FINISH(block_processing_time);
3213  if(precomputed)
3214    block_processing_time += m_fake_pow_calc_time;
3215
3216  m_db->block_txn_stop();
3217  TIME_MEASURE_START(addblock);
3218  uint64_t new_height = 0;
3219  if (!bvc.m_verifivation_failed)
3220  {
3221    try
3222    {
3223      new_height = m_db->add_block(bl, block_size, cumulative_difficulty, already_generated_coins, txs);
3224    }
3225    catch (const KEY_IMAGE_EXISTS& e)
3226    {
3227      LOG_ERROR("Error adding block with hash: " << id << " to blockchain, what = " << e.what());
3228      bvc.m_verifivation_failed = true;
3229      return_tx_to_pool(txs);
3230      return false;
3231    }
3232    catch (const std::exception& e)
3233    {
3234      //TODO: figure out the best way to deal with this failure
3235      LOG_ERROR("Error adding block with hash: " << id << " to blockchain, what = " << e.what());
3236      return_tx_to_pool(txs);
3237      return false;
3238    }
3239  }
3240  else
3241  {
3242    LOG_ERROR("Blocks that failed verification should not reach here");
3243  }
3244
3245  TIME_MEASURE_FINISH(addblock);
3246
3247  // do this after updating the hard fork state since the size limit may change due to fork
3248  update_next_cumulative_size_limit();
3249
3250  LOG_PRINT_L1("+++++ BLOCK SUCCESSFULLY ADDED" << std::endl << "id:\t" << id << std::endl << "PoW:\t" << proof_of_work << std::endl << "HEIGHT " << new_height-1 << ", difficulty:\t" << current_diffic << std::endl << "block reward: " << print_money(fee_summary + base_reward) << "(" << print_money(base_reward) << " + " << print_money(fee_summary) << "), coinbase_blob_size: " << coinbase_blob_size << ", cumulative size: " << cumulative_block_size << ", " << block_processing_time << "(" << target_calculating_time << "/" << longhash_calculating_time << ")ms");
3251  if(m_show_time_stats)
3252  {
3253    LOG_PRINT_L0("Height: " << new_height << " blob: " << coinbase_blob_size << " cumm: "
3254        << cumulative_block_size << " p/t: " << block_processing_time << " ("
3255        << target_calculating_time << "/" << longhash_calculating_time << "/"
3256        << t1 << "/" << t2 << "/" << t3 << "/" << t_exists << "/" << t_pool
3257        << "/" << t_checktx << "/" << t_dblspnd << "/" << vmt << "/" << addblock << ")ms");
3258  }
3259
3260  bvc.m_added_to_main_chain = true;
3261  ++m_sync_counter;
3262
3263  // appears to be a NOP *and* is called elsewhere.  wat?
3264  m_tx_pool.on_blockchain_inc(new_height, id);
3265
3266  return true;
3267}
3268//------------------------------------------------------------------
3269bool Blockchain::update_next_cumulative_size_limit()
3270{
3271  uint64_t full_reward_zone = CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE;
3272
3273  LOG_PRINT_L3("Blockchain::" << __func__);
3274  std::vector<size_t> sz;
3275  get_last_n_blocks_sizes(sz, CRYPTONOTE_REWARD_BLOCKS_WINDOW);
3276
3277  uint64_t median = epee::misc_utils::median(sz);
3278  if(median <= full_reward_zone)
3279    median = full_reward_zone;
3280
3281  m_current_block_cumul_sz_limit = median*2;
3282  return true;
3283}
3284//------------------------------------------------------------------
3285bool Blockchain::add_new_block(const block& bl_, block_verification_context& bvc)
3286{
3287  LOG_PRINT_L3("Blockchain::" << __func__);
3288  //copy block here to let modify block.target
3289  block bl = bl_;
3290  crypto::hash id = get_block_hash(bl);
3291  CRITICAL_REGION_LOCAL(m_tx_pool);//to avoid deadlock lets lock tx_pool for whole add/reorganize process
3292  CRITICAL_REGION_LOCAL1(m_blockchain_lock);
3293  m_db->block_txn_start(true);
3294  if(have_block(id))
3295  {
3296    LOG_PRINT_L3("block with id = " << id << " already exists");
3297    bvc.m_already_exists = true;
3298    m_db->block_txn_stop();
3299    return false;
3300  }
3301
3302  //check that block refers to chain tail
3303  if(!(bl.prev_id == get_tail_id()))
3304  {
3305    //chain switching or wrong block
3306    bvc.m_added_to_main_chain = false;
3307    m_db->block_txn_stop();
3308    return handle_alternative_block(bl, id, bvc);
3309    //never relay alternative blocks
3310  }
3311
3312  m_db->block_txn_stop();
3313  return handle_block_to_main_chain(bl, id, bvc);
3314}
3315//------------------------------------------------------------------
3316//TODO: Refactor, consider returning a failure height and letting
3317//      caller decide course of action.
3318void Blockchain::check_against_checkpoints(const checkpoints& points, bool enforce)
3319{
3320  const auto& pts = points.get_points();
3321
3322  CRITICAL_REGION_LOCAL(m_blockchain_lock);
3323  m_db->batch_start();
3324  for (const auto& pt : pts)
3325  {
3326    // if the checkpoint is for a block we don't have yet, move on
3327    if (pt.first >= m_db->height())
3328    {
3329      continue;
3330    }
3331
3332    if (!points.check_block(pt.first, m_db->get_block_hash_from_height(pt.first)))
3333    {
3334      // if asked to enforce checkpoints, roll back to a couple of blocks before the checkpoint
3335      if (enforce)
3336      {
3337        LOG_ERROR("Local blockchain failed to pass a checkpoint, rolling back!");
3338        std::list<block> empty;
3339        rollback_blockchain_switching(empty, pt.first - 2);
3340      }
3341      else
3342      {
3343        LOG_ERROR("WARNING: local blockchain failed to pass a SumoPulse checkpoint, and you could be on a fork. You should either sync up from scratch, OR download a fresh blockchain bootstrap, OR enable checkpoint enforcing with the --enforce-dns-checkpointing command-line option");
3344      }
3345    }
3346  }
3347  m_db->batch_stop();
3348}
3349//------------------------------------------------------------------
3350// returns false if any of the checkpoints loading returns false.
3351// That should happen only if a checkpoint is added that conflicts
3352// with an existing checkpoint.
3353bool Blockchain::update_checkpoints(const std::string& file_path, bool check_dns)
3354{
3355  if (!m_checkpoints.load_checkpoints_from_json(file_path))
3356  {
3357      return false;
3358  }
3359
3360  // if we're checking both dns and json, load checkpoints from dns.
3361  // if we're not hard-enforcing dns checkpoints, handle accordingly
3362  if (m_enforce_dns_checkpoints && check_dns)
3363  {
3364    if (!m_checkpoints.load_checkpoints_from_dns())
3365    {
3366      return false;
3367    }
3368  }
3369  else if (check_dns)
3370  {
3371    checkpoints dns_points;
3372    dns_points.load_checkpoints_from_dns();
3373    if (m_checkpoints.check_for_conflicts(dns_points))
3374    {
3375      check_against_checkpoints(dns_points, false);
3376    }
3377    else
3378    {
3379      LOG_PRINT_L0("One or more checkpoints fetched from DNS conflicted with existing checkpoints!");
3380    }
3381  }
3382
3383  check_against_checkpoints(m_checkpoints, true);
3384
3385  return true;
3386}
3387//------------------------------------------------------------------
3388void Blockchain::set_enforce_dns_checkpoints(bool enforce_checkpoints)
3389{
3390  m_enforce_dns_checkpoints = enforce_checkpoints;
3391}
3392
3393//------------------------------------------------------------------
3394void Blockchain::block_longhash_worker(cn_pow_hash_v2& hash_ctx, const std::vector<block> &blocks, std::unordered_map<crypto::hash, crypto::hash> &map)
3395{
3396  TIME_MEASURE_START(t);
3397
3398  //FIXME: height should be changing here, as get_block_longhash expects
3399  //       the height of the block passed to it
3400  for (const auto & block : blocks)
3401  {
3402    if (m_cancel)
3403       return;
3404    crypto::hash id = get_block_hash(block);
3405    crypto::hash pow;
3406	get_block_longhash(block, hash_ctx, pow);
3407    map.emplace(id, pow);
3408  }
3409
3410  TIME_MEASURE_FINISH(t);
3411}
3412
3413//------------------------------------------------------------------
3414bool Blockchain::cleanup_handle_incoming_blocks(bool force_sync)
3415{
3416  LOG_PRINT_YELLOW("Blockchain::" << __func__, LOG_LEVEL_3);
3417  CRITICAL_REGION_LOCAL(m_blockchain_lock);
3418  TIME_MEASURE_START(t1);
3419
3420  if (m_sync_counter > 0)
3421  {
3422    if (force_sync)
3423    {
3424      if(m_db_sync_mode != db_nosync)
3425        store_blockchain();
3426      m_sync_counter = 0;
3427    }
3428    else if (m_db_blocks_per_sync && m_sync_counter >= m_db_blocks_per_sync)
3429    {
3430      if(m_db_sync_mode == db_async)
3431      {
3432        m_sync_counter = 0;
3433        m_async_service.dispatch(boost::bind(&Blockchain::store_blockchain, this));
3434      }
3435      else if(m_db_sync_mode == db_sync)
3436      {
3437        store_blockchain();
3438      }
3439      else // db_nosync
3440      {
3441        // DO NOTHING, not required to call sync.
3442      }
3443    }
3444  }
3445
3446  TIME_MEASURE_FINISH(t1);
3447  m_blocks_longhash_table.clear();
3448  m_scan_table.clear();
3449  m_blocks_txs_check.clear();
3450  m_check_txin_table.clear();
3451
3452  return true;
3453}
3454
3455//------------------------------------------------------------------
3456//FIXME: unused parameter txs
3457void Blockchain::output_scan_worker(const uint64_t amount, const std::vector<uint64_t> &offsets, std::vector<output_data_t> &outputs, std::unordered_map<crypto::hash, cryptonote::transaction> &txs) const
3458{
3459  try
3460  {
3461    m_db->get_output_key(amount, offsets, outputs);
3462  }
3463  catch (const std::exception& e)
3464  {
3465    LOG_PRINT_L1("EXCEPTION: " << e.what());
3466  }
3467  catch (...)
3468  {
3469
3470  }
3471}
3472
3473//------------------------------------------------------------------
3474// ND: Speedups:
3475// 1. Thread long_hash computations if possible (m_max_prepare_blocks_threads = nthreads, default = 4)
3476// 2. Group all amounts (from txs) and related absolute offsets and form a table of tx_prefix_hash
3477//    vs [k_image, output_keys] (m_scan_table). This is faster because it takes advantage of bulk queries
3478//    and is threaded if possible. The table (m_scan_table) will be used later when querying output
3479//    keys.
3480bool Blockchain::prepare_handle_incoming_blocks(const std::list<block_complete_entry> &blocks_entry)
3481{
3482  LOG_PRINT_YELLOW("Blockchain::" << __func__, LOG_LEVEL_3);
3483  TIME_MEASURE_START(prepare);
3484  CRITICAL_REGION_LOCAL(m_blockchain_lock);
3485
3486  if(blocks_entry.size() == 0)
3487    return false;
3488
3489  if ((m_db->height() + blocks_entry.size()) < m_blocks_hash_check.size())
3490    return true;
3491
3492  bool blocks_exist = false;
3493  uint64_t threads = tools::get_max_concurrency();
3494
3495  if (blocks_entry.size() > 1 && threads > 1 && m_max_prepare_blocks_threads > 1)
3496  {
3497    // limit threads, default limit = 4
3498    if(threads > m_max_prepare_blocks_threads)
3499      threads = m_max_prepare_blocks_threads;
3500
3501    uint64_t height = m_db->height();
3502    std::vector<boost::thread *> thread_list;
3503    int batches = blocks_entry.size() / threads;
3504    int extra = blocks_entry.size() % threads;
3505    LOG_PRINT_L1("block_batches: " << batches);
3506    std::vector<std::unordered_map<crypto::hash, crypto::hash>> maps(threads);
3507    std::vector < std::vector < block >> blocks(threads);
3508    auto it = blocks_entry.begin();
3509
3510    for (uint64_t i = 0; i < threads; i++)
3511    {
3512      for (int j = 0; j < batches; j++)
3513      {
3514        block block;
3515
3516        if (!parse_and_validate_block_from_blob(it->block, block))
3517        {
3518          std::advance(it, 1);
3519          continue;
3520        }
3521
3522        // check first block and skip all blocks if its not chained properly
3523        if (i == 0 && j == 0)
3524        {
3525          crypto::hash tophash = m_db->top_block_hash();
3526          if (block.prev_id != tophash)
3527          {
3528            LOG_PRINT_L1("Skipping prepare blocks. New blocks don't belong to chain.");
3529            return true;
3530          }
3531        }
3532        if (have_block(get_block_hash(block)))
3533        {
3534          blocks_exist = true;
3535          break;
3536        }
3537
3538        blocks[i].push_back(block);
3539        std::advance(it, 1);
3540      }
3541    }
3542
3543    for (int i = 0; i < extra && !blocks_exist; i++)
3544    {
3545      block block;
3546
3547      if (!parse_and_validate_block_from_blob(it->block, block))
3548      {
3549        std::advance(it, 1);
3550        continue;
3551      }
3552
3553      if (have_block(get_block_hash(block)))
3554      {
3555        blocks_exist = true;
3556        break;
3557      }
3558
3559      blocks[i].push_back(block);
3560      std::advance(it, 1);
3561    }
3562
3563    if (!blocks_exist)
3564    {
3565      m_blocks_longhash_table.clear();
3566
3567	  if(m_hash_ctxes_multi.size() < threads)
3568		m_hash_ctxes_multi.resize(threads);
3569
3570      for (uint64_t i = 0; i < threads; i++)
3571      {
3572        thread_list.push_back(new boost::thread(&Blockchain::block_longhash_worker, this, std::ref(m_hash_ctxes_multi[i]), std::cref(blocks[i]), std::ref(maps[i])));
3573      }
3574
3575      for (size_t j = 0; j < thread_list.size(); j++)
3576      {
3577        thread_list[j]->join();
3578        delete thread_list[j];
3579      }
3580
3581      thread_list.clear();
3582
3583      if (m_cancel)
3584         return false;
3585
3586      for (const auto & map : maps)
3587      {
3588        m_blocks_longhash_table.insert(map.begin(), map.end());
3589      }
3590    }
3591  }
3592
3593  if (m_cancel)
3594    return false;
3595
3596  if (blocks_exist)
3597  {
3598    LOG_PRINT_L0("Skipping prepare blocks. Blocks exist.");
3599    return true;
3600  }
3601
3602  m_fake_scan_time = 0;
3603  m_fake_pow_calc_time = 0;
3604
3605  m_scan_table.clear();
3606  m_check_txin_table.clear();
3607
3608  TIME_MEASURE_FINISH(prepare);
3609  m_fake_pow_calc_time = prepare / blocks_entry.size();
3610
3611  if (blocks_entry.size() > 1 && threads > 1 && m_show_time_stats)
3612    LOG_PRINT_L0("Prepare blocks took: " << prepare << " ms");
3613
3614  TIME_MEASURE_START(scantable);
3615
3616  // [input] stores all unique amounts found
3617  std::vector < uint64_t > amounts;
3618  // [input] stores all absolute_offsets for each amount
3619  std::map<uint64_t, std::vector<uint64_t>> offset_map;
3620  // [output] stores all output_data_t for each absolute_offset
3621  std::map<uint64_t, std::vector<output_data_t>> tx_map;
3622
3623#define SCAN_TABLE_QUIT(m) \
3624        do { \
3625            LOG_PRINT_L0(m) ;\
3626            m_scan_table.clear(); \
3627            return false; \
3628        } while(0); \
3629
3630  // generate sorted tables for all amounts and absolute offsets
3631  for (const auto &entry : blocks_entry)
3632  {
3633    if (m_cancel)
3634      return false;
3635
3636    for (const auto &tx_blob : entry.txs)
3637    {
3638      crypto::hash tx_hash = null_hash;
3639      crypto::hash tx_prefix_hash = null_hash;
3640      transaction tx;
3641
3642      if (!parse_and_validate_tx_from_blob(tx_blob, tx, tx_hash, tx_prefix_hash))
3643        SCAN_TABLE_QUIT("Could not parse tx from incoming blocks.");
3644
3645      auto its = m_scan_table.find(tx_prefix_hash);
3646      if (its != m_scan_table.end())
3647        SCAN_TABLE_QUIT("Duplicate tx found from incoming blocks.");
3648
3649      m_scan_table.emplace(tx_prefix_hash, std::unordered_map<crypto::key_image, std::vector<output_data_t>>());
3650      its = m_scan_table.find(tx_prefix_hash);
3651      assert(its != m_scan_table.end());
3652
3653      // get all amounts from tx.vin(s)
3654      for (const auto &txin : tx.vin)
3655      {
3656        const txin_to_key &in_to_key = boost::get < txin_to_key > (txin);
3657
3658        // check for duplicate
3659        auto it = its->second.find(in_to_key.k_image);
3660        if (it != its->second.end())
3661          SCAN_TABLE_QUIT("Duplicate key_image found from incoming blocks.");
3662
3663        amounts.push_back(in_to_key.amount);
3664      }
3665
3666      // sort and remove duplicate amounts from amounts list
3667      std::sort(amounts.begin(), amounts.end());
3668      auto last = std::unique(amounts.begin(), amounts.end());
3669      amounts.erase(last, amounts.end());
3670
3671      // add amount to the offset_map and tx_map
3672      for (const uint64_t &amount : amounts)
3673      {
3674        if (offset_map.find(amount) == offset_map.end())
3675          offset_map.emplace(amount, std::vector<uint64_t>());
3676
3677        if (tx_map.find(amount) == tx_map.end())
3678          tx_map.emplace(amount, std::vector<output_data_t>());
3679      }
3680
3681      // add new absolute_offsets to offset_map
3682      for (const auto &txin : tx.vin)
3683      {
3684        const txin_to_key &in_to_key = boost::get < txin_to_key > (txin);
3685        // no need to check for duplicate here.
3686        auto absolute_offsets = relative_output_offsets_to_absolute(in_to_key.key_offsets);
3687        for (const auto & offset : absolute_offsets)
3688          offset_map[in_to_key.amount].push_back(offset);
3689
3690      }
3691
3692      // sort and remove duplicate absolute_offsets in offset_map
3693      for (auto &offsets : offset_map)
3694      {
3695        std::sort(offsets.second.begin(), offsets.second.end());
3696        auto last = std::unique(offsets.second.begin(), offsets.second.end());
3697        offsets.second.erase(last, offsets.second.end());
3698      }
3699    }
3700  }
3701
3702  // [output] stores all transactions for each tx_out_index::hash found
3703  std::vector<std::unordered_map<crypto::hash, cryptonote::transaction>> transactions(amounts.size());
3704
3705  threads = tools::get_max_concurrency();
3706  if (!m_db->can_thread_bulk_indices())
3707    threads = 1;
3708
3709  if (threads > 1)
3710  {
3711    boost::asio::io_service ioservice;
3712    boost::thread_group threadpool;
3713    std::unique_ptr < boost::asio::io_service::work > work(new boost::asio::io_service::work(ioservice));
3714
3715    for (uint64_t i = 0; i < threads; i++)
3716    {
3717      threadpool.create_thread(boost::bind(&boost::asio::io_service::run, &ioservice));
3718    }
3719
3720    for (size_t i = 0; i < amounts.size(); i++)
3721    {
3722      uint64_t amount = amounts[i];
3723      ioservice.dispatch(boost::bind(&Blockchain::output_scan_worker, this, amount, std::cref(offset_map[amount]), std::ref(tx_map[amount]), std::ref(transactions[i])));
3724    }
3725
3726    work.reset();
3727    threadpool.join_all();
3728    ioservice.stop();
3729  }
3730  else
3731  {
3732    for (size_t i = 0; i < amounts.size(); i++)
3733    {
3734      uint64_t amount = amounts[i];
3735      output_scan_worker(amount, offset_map[amount], tx_map[amount], transactions[i]);
3736    }
3737  }
3738
3739  int total_txs = 0;
3740
3741  // now generate a table for each tx_prefix and k_image hashes
3742  for (const auto &entry : blocks_entry)
3743  {
3744    if (m_cancel)
3745      return false;
3746
3747    for (const auto &tx_blob : entry.txs)
3748    {
3749      crypto::hash tx_hash = null_hash;
3750      crypto::hash tx_prefix_hash = null_hash;
3751      transaction tx;
3752
3753      if (!parse_and_validate_tx_from_blob(tx_blob, tx, tx_hash, tx_prefix_hash))
3754        SCAN_TABLE_QUIT("Could not parse tx from incoming blocks.");
3755
3756      ++total_txs;
3757      auto its = m_scan_table.find(tx_prefix_hash);
3758      if (its == m_scan_table.end())
3759        SCAN_TABLE_QUIT("Tx not found on scan table from incoming blocks.");
3760
3761      for (const auto &txin : tx.vin)
3762      {
3763        const txin_to_key &in_to_key = boost::get < txin_to_key > (txin);
3764        auto needed_offsets = relative_output_offsets_to_absolute(in_to_key.key_offsets);
3765
3766        std::vector<output_data_t> outputs;
3767        for (const uint64_t & offset_needed : needed_offsets)
3768        {
3769          size_t pos = 0;
3770          bool found = false;
3771
3772          for (const uint64_t &offset_found : offset_map[in_to_key.amount])
3773          {
3774            if (offset_needed == offset_found)
3775            {
3776              found = true;
3777              break;
3778            }
3779
3780            ++pos;
3781          }
3782
3783          if (found && pos < tx_map[in_to_key.amount].size())
3784            outputs.push_back(tx_map[in_to_key.amount].at(pos));
3785          else
3786            break;
3787        }
3788
3789        its->second.emplace(in_to_key.k_image, outputs);
3790      }
3791    }
3792  }
3793
3794  TIME_MEASURE_FINISH(scantable);
3795  if (total_txs > 0)
3796  {
3797    m_fake_scan_time = scantable / total_txs;
3798    if(m_show_time_stats)
3799      LOG_PRINT_L0("Prepare scantable took: " << scantable << " ms");
3800  }
3801
3802  return true;
3803}
3804
3805void Blockchain::set_user_options(uint64_t maxthreads, uint64_t blocks_per_sync, blockchain_db_sync_mode sync_mode, bool fast_sync)
3806{
3807  m_db_sync_mode = sync_mode;
3808  m_fast_sync = fast_sync;
3809  m_db_blocks_per_sync = blocks_per_sync;
3810  m_max_prepare_blocks_threads = maxthreads;
3811}
3812
3813HardFork::State Blockchain::get_hard_fork_state() const
3814{
3815  return m_hardfork->get_state();
3816}
3817
3818bool Blockchain::get_hard_fork_voting_info(uint8_t version, uint32_t &window, uint32_t &votes, uint32_t &threshold, uint64_t &earliest_height, uint8_t &voting) const
3819{
3820  return m_hardfork->get_voting_info(version, window, votes, threshold, earliest_height, voting);
3821}
3822
3823std::map<uint64_t, std::tuple<uint64_t, uint64_t, uint64_t>> Blockchain:: get_output_histogram(const std::vector<uint64_t> &amounts, bool unlocked, uint64_t recent_cutoff) const
3824{
3825  return m_db->get_output_histogram(amounts, unlocked, recent_cutoff);
3826}
3827
3828void Blockchain::cancel()
3829{
3830  m_cancel = true;
3831}
3832
3833#if defined(PER_BLOCK_CHECKPOINT)
3834void Blockchain::load_compiled_in_block_hashes()
3835{
3836  if (m_fast_sync && get_blocks_dat_start(m_testnet) != nullptr)
3837  {
3838    if (get_blocks_dat_size(m_testnet) > 4)
3839    {
3840      const unsigned char *p = get_blocks_dat_start(m_testnet);
3841      const uint32_t nblocks = *p | ((*(p+1))<<8) | ((*(p+2))<<16) | ((*(p+3))<<24);
3842      const size_t size_needed = 4 + nblocks * sizeof(crypto::hash);
3843      if(nblocks > 0 && nblocks > m_db->height() && get_blocks_dat_size(m_testnet) >= size_needed)
3844      {
3845        LOG_PRINT_L0("Loading precomputed blocks: " << nblocks);
3846        p += sizeof(uint32_t);
3847        for (uint32_t i = 0; i < nblocks; i++)
3848        {
3849          crypto::hash hash;
3850          memcpy(hash.data, p, sizeof(hash.data));
3851          p += sizeof(hash.data);
3852          m_blocks_hash_check.push_back(hash);
3853        }
3854
3855        // FIXME: clear tx_pool because the process might have been
3856        // terminated and caused it to store txs kept by blocks.
3857        // The core will not call check_tx_inputs(..) for these
3858        // transactions in this case. Consequently, the sanity check
3859        // for tx hashes will fail in handle_block_to_main_chain(..)
3860        std::list<transaction> txs;
3861        m_tx_pool.get_transactions(txs);
3862
3863        size_t blob_size;
3864        uint64_t fee;
3865        bool relayed;
3866        transaction pool_tx;
3867        for(const transaction &tx : txs)
3868        {
3869          crypto::hash tx_hash = get_transaction_hash(tx);
3870          m_tx_pool.take_tx(tx_hash, pool_tx, blob_size, fee, relayed);
3871        }
3872      }
3873    }
3874  }
3875}
3876#endif
3877
3878bool Blockchain::for_all_key_images(std::function<bool(const crypto::key_image&)> f) const
3879{
3880  return m_db->for_all_key_images(f);
3881}
3882
3883bool Blockchain::for_all_blocks(std::function<bool(uint64_t, const crypto::hash&, const block&)> f) const
3884{
3885  return m_db->for_all_blocks(f);
3886}
3887
3888bool Blockchain::for_all_transactions(std::function<bool(const crypto::hash&, const cryptonote::transaction&)> f) const
3889{
3890  return m_db->for_all_transactions(f);
3891}
3892
3893bool Blockchain::for_all_outputs(std::function<bool(uint64_t amount, const crypto::hash &tx_hash, size_t tx_idx)> f) const
3894{
3895  return m_db->for_all_outputs(f);;
3896}