diff -uNr a/gbw-node/README b/gbw-node/README --- a/gbw-node/README 0b76b48c6dc8143232742bf405a824a863298c9cd20967b4a20e6ba5721f56a1d3ba1b1baaa72a4f10b66fa8861fcf6de2fc6eecea0cb756f55478cc732345ee +++ b/gbw-node/README 18a9decf5dd9c389fb43f328721f8faee515ed899444667bc5604c3e3022c4ebd669488659675be7b31017b7612987a9305d0d3a0fc14c78f2fb68316e478e5c @@ -27,11 +27,11 @@ 1. Create the top-level /package directory if necessary and place the tree at its fully version-qualified path: mkdir -p /package - cp -r /YOUR/PATH/TO/gbw-node /package/gbw-node-2 + cp -r /YOUR/PATH/TO/gbw-node /package/gbw-node-2.1 2. Run the install script from the above directory: - cd /package/gbw-node-2 + cd /package/gbw-node-2.1 sh package/install To revert to this version after installing a different one, simply repeat step 2. @@ -54,7 +54,3 @@ The first enables enforcement of the declared referential constraints, which SQLite disables by default for compatibility reasons. The rest make the output more readable; especially quote mode is essential for viewing address and txid fields which are packed as blobs. If your SQLite version doesn't support it, ".mode insert" is a more verbose alternative. RPC username and password for sending commands to the local bitcoind are read from ~/.bitcoin/bitcoin.conf. - -It's possible for bitcoind to get blocked on writing to a pipe if a scan process is interrupted during the dumpblock call. To recover, allow the write to complete: - - cat ~/.gbw/blockpipe >/dev/null diff -uNr a/gbw-node/command/gbw-node b/gbw-node/command/gbw-node --- a/gbw-node/command/gbw-node 47a402e270f590661d65b8300c3529294504f5535801969a15e25110aedccd9767749eaf5792ef479e77b12b1707358ce560b2e633e8a585a4505c8ad5aaa4f3 +++ b/gbw-node/command/gbw-node 0f7a8331c2f2f41ac79b6838465b66d398e6392afcadae7e82df057b7dcfb4498e60edc813bcd905e35dd7da2084d34efe5b8c6472a75124e1189d20f793f498 @@ -1,11 +1,10 @@ #!/usr/bin/python2 # J. Welsh, December 2019 -from os import getenv, open as os_open, O_CREAT, O_EXCL, O_RDONLY, O_WRONLY, mkdir, mkfifo, read, write, close, stat, rename -from stat import S_ISDIR, S_ISFIFO +from os import getenv, open as os_open, O_CREAT, O_EXCL, O_WRONLY, mkdir, read, write, close, stat, rename, unlink, getpid +from stat import S_ISDIR from sys import argv, stdin, stdout, stderr, exit from socket import socket -from threading import Thread, Event from binascii import a2b_hex, b2a_hex from base64 import b64encode from struct import Struct @@ -13,7 +12,6 @@ from decimal import Decimal from inspect import getdoc from getpass import getpass -from cStringIO import StringIO import errno import signal import string @@ -33,10 +31,11 @@ # $ gbw-node reset # $ gbw-node scan -INSTALL_PATH = '/package/gbw-node-2' +INSTALL_PATH = '/package/gbw-node-2.1' gbw_home = getenv('HOME') + '/.gbw' -bitcoin_conf_path = getenv('HOME') + '/.bitcoin/bitcoin.conf' +bitcoin_datadir = getenv('HOME') + '/.bitcoin' +bitcoin_conf_path = bitcoin_datadir + '/bitcoin.conf' db = None @@ -53,6 +52,9 @@ raise # Database tuning knobs here db = sqlite3.connect(db_path, timeout=600) # in seconds + # Use Write Ahead Log mode for much improved commit latency. This generally persists with the database, but doesn't survive a dump and restore to new file, so set it here rather than in the schema. Requires sqlite >= 3.7.0. + db.execute('PRAGMA journal_mode=WAL') + # SQLite's foreign key enforcement is per connection and default off for compatibility reasons. db.execute('PRAGMA foreign_keys=ON') db.execute('PRAGMA cache_size=-8000') # negative means in KiB db.execute('PRAGMA wal_autocheckpoint=10000') # in pages (4k) @@ -131,13 +133,6 @@ if not (e.errno == errno.EEXIST and S_ISDIR(stat(path).st_mode)): raise -def require_fifo(path): - try: - mkfifo(path) - except OSError, e: - if not (e.errno == errno.EEXIST and S_ISFIFO(stat(path).st_mode)): - raise - ################################################## # RPC client @@ -264,31 +259,6 @@ return s[3:23] return None -getblock_thread = None -getblock_done = Event() -getblock_result = None -def getblock_reader(pipe): - global getblock_result - while True: - fd = os_open(pipe, O_RDONLY) - getblock_result = read_all(fd) - getblock_done.set() - close(fd) - -def getblock(height): - global getblock_thread - pipe = gbw_home + '/blockpipe' - if getblock_thread is None: - require_fifo(pipe) - getblock_thread = Thread(target=getblock_reader, args=(pipe,)) - getblock_thread.daemon = True - getblock_thread.start() - if not rpc('dumpblock', height, pipe): - raise ValueError('dumpblock returned false') - getblock_done.wait() - getblock_done.clear() - return getblock_result - ################################################## # Base58 @@ -448,9 +418,9 @@ ################################################## # Command implementations -def scan_block(height, block_data): +def scan_block(height, stream): stdout.write('block %s' % height) - blkhash, prev, time, target, txs = load_block(StringIO(block_data)) + blkhash, prev, time, target, txs = load_block(stream) count_out = 0 n_tx = 0 @@ -504,11 +474,16 @@ scan Iterate blocks from bitcoind, indexing transaction inputs and outputs affecting watched addresses. May be safely interrupted and resumed. - - NOT PRESENTLY SAFE TO RUN CONCURRENT INSTANCES due to the dumpblock to named pipe kludge. ''' db.execute('PRAGMA synchronous=NORMAL') - height = db.execute('SELECT scan_height FROM state').fetchone()[0] + height = db.execute('SELECT scan_height FROM state').fetchone()[0] + 1 + # dumpblock is ugly and expensive (O(n) search); we'll still use it once to get an initial hash, as the only available way for now to look up an arbitrary block by height, but then we can walk up the best chain using the block index and read each block directly from its file on disk. + dump_filename = '%s/dump_%d_%d.blk' % (gbw_home, height, getpid()) + if not rpc('dumpblock', height, dump_filename): + raise ValueError('dumpblock returned false') + with open(dump_filename) as f: + blkhash = b2lx(load_block(f)[0]) + unlink(dump_filename) # RPC calls can have high latency, so refresh the goal height upon reaching the previous rather than for each block scanned. last_blockcount = -1 while True: @@ -516,11 +491,19 @@ if blockcount == last_blockcount: break last_blockcount = blockcount - while height < blockcount: - height += 1 - scan_block(height, getblock(height)) + while height <= blockcount: + index = rpc('getblockindex', blkhash) + assert index['hash'] == blkhash + assert index['height'] == height + # main.cpp OpenBlockFile + with open('%s/blk%04d.dat' % (bitcoin_datadir, index['file'])) as f: + f.seek(index['blockpos']) + scan_block(height, f) db.execute('UPDATE state SET scan_height = ?', (height,)) db.commit() + height += 1 + # nextblockhash refers to the block on best chain whose prev is the current block. + blkhash = index['nextblockhash'] def cmd_reset(): ''' diff -uNr a/gbw-node/library/schema-node.sql b/gbw-node/library/schema-node.sql --- a/gbw-node/library/schema-node.sql 6085183c2be7f218a59ea58aca4aa67fce70da9b6b1f3d2b255f23ac801cd65458f9d7543f33911ca5fafbff87cda0ee095a523372caab7cf0cbc62abd37b9e0 +++ b/gbw-node/library/schema-node.sql e2552194c6a9fc0e4a594585a17d3a03277a9818b9ddd0c1b534ebb740e432b0fe1498f41bc8f7ff3bf23048269c6b76baee350da8e6c9f1c1e6128e6cf4fb9d @@ -1,8 +1,7 @@ --- Gales Bitcoin Wallet: node (online component) schema --- J. Welsh, December 2019 ---- Dialect: SQLite (3.7.0 for WAL) +--- Dialect: SQLite -PRAGMA journal_mode=WAL; BEGIN; CREATE TABLE tx ( diff -uNr a/gbw-node/manifest b/gbw-node/manifest --- a/gbw-node/manifest e772e11336101afba7c6a85b4656d229e59d1fbdb2a3b484d902d0e74ae9721bd5a6e82375e332f7c046371486ba94034f6002d8ae4d1712fba92366edeba447 +++ b/gbw-node/manifest af606fac4d39cefc00560375ba4c509cd713d7744159992689fba7a8955397dc1bcffb5f7c066bbc494346407dce7252dc70f96c3a7aecf77393cc01a8f23624 @@ -4,3 +4,4 @@ 783269 gbw-node_error_tolerance jfw Don't die immediately on bad input lines. 783270 gbw-node_db_auto_init jfw Automate database initialization from schema script; update and expand related documentation. 783272 gbw-node_memoryview_replacement jfw Replace memoryview with cStringIO for loading block data, eliminating the main cause of Python 2.7 dependence, simplifying the code and even somewhat improving performance. +783273 gbw-node_direct_block_read jfw Use the newer getblockindex RPC to walk the chain in linear time and read blocks directly from files on disk. This bypasses the dumpblock hack, except for finding the initial hash; in turn, it makes sense to remove the named pipe and threading complications and just dump to a regular file. Also enable WAL mode more reliably by setting it on connect. Bump version by way of adding a minor number (for instance because there are no schema changes or significant interface or behavior changes). diff -uNr a/gbw-node/package/install b/gbw-node/package/install --- a/gbw-node/package/install 4d48bc47d532c169ade4e8757048318fd5b013b1b94533b4088d1dd8bb1855224b1b4f64a083a5a3dc1559948c163f8b01f63ba38bc0a1d29b82896f7a8c4568 +++ b/gbw-node/package/install 2bde97438c8760519881ed9327ed4caafd6d9384cfd85d9a818af6cd1e387d440e3f5513fc16774646958a0eb959d4ce59d0d7c83030fc17688bfa0601f8049a @@ -2,7 +2,7 @@ set -e P=gbw-node -V=2 +V=2.1 cd /package/$P-$V # Versioned path duplicated in: