From cd1e0885ad04cb1019a91a3b04db338baea3758a Mon Sep 17 00:00:00 2001 From: SomberNight Date: Thu, 22 Oct 2020 19:32:35 +0200 Subject: [PATCH 01/27] history db: change schema, and rm compaction --- electrumx_compact_history | 11 - pyproject.toml | 1 - .../cli/electrumx_compact_history.py | 83 ---- src/electrumx/server/block_processor.py | 17 +- src/electrumx/server/db.py | 33 +- src/electrumx/server/history.py | 387 +++++++----------- src/electrumx/server/session.py | 1 - tests/server/test_compaction.py | 133 ------ 8 files changed, 172 insertions(+), 494 deletions(-) delete mode 100755 electrumx_compact_history delete mode 100644 src/electrumx/cli/electrumx_compact_history.py delete mode 100644 tests/server/test_compaction.py diff --git a/electrumx_compact_history b/electrumx_compact_history deleted file mode 100755 index 8104be127..000000000 --- a/electrumx_compact_history +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 ; mode: python -*- -import os -import sys - - -if __name__ == '__main__': - src_dir = os.path.join(os.path.dirname(__file__), "src") - sys.path.insert(0, src_dir) - from electrumx.cli.electrumx_compact_history import main - sys.exit(main()) diff --git a/pyproject.toml b/pyproject.toml index 5a91cf79b..606bb6a2c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,7 +37,6 @@ Repository = "https://github.com/spesmilo/electrumx" [project.scripts] electrumx_server = "electrumx.cli.electrumx_server:main" electrumx_rpc = "electrumx.cli.electrumx_rpc:main" -electrumx_compact_history = "electrumx.cli.electrumx_compact_history:main" [tool.setuptools.dynamic] version = { attr = 'electrumx.__version__' } diff --git a/src/electrumx/cli/electrumx_compact_history.py b/src/electrumx/cli/electrumx_compact_history.py deleted file mode 100644 index 66cc81312..000000000 --- a/src/electrumx/cli/electrumx_compact_history.py +++ /dev/null @@ -1,83 +0,0 @@ -#!/usr/bin/env python3 -# -# Copyright (c) 2017, Neil Booth -# -# All rights reserved. -# -# See the file "LICENCE" for information about the copyright -# and warranty status of this software. - -'''Script to compact the history database. This should save space and -will reset the flush counter to a low number, avoiding overflow when -the flush count reaches 65,536. - -This needs to lock the database so ElectrumX must not be running - -shut it down cleanly first. - -It is recommended you run this script with the same environment as -ElectrumX. However, it is intended to be runnable with just -DB_DIRECTORY and COIN set (COIN defaults as for ElectrumX). - -If you use daemon tools, you might run this script like so: - - envdir /path/to/the/environment/directory ./compact_history.py - -Depending on your hardware, this script may take up to 6 hours to -complete; it logs progress regularly. - -Compaction can be interrupted and restarted harmlessly and will pick -up where it left off. However, if you restart ElectrumX without -running the compaction to completion, it will not benefit and -subsequent compactions will restart from the beginning. -''' - -import asyncio -import logging -import sys -import traceback -from os import environ - -from electrumx import Env -from electrumx.server.db import DB - - -async def compact_history(): - if sys.version_info < (3, 10): - raise RuntimeError('Python >= 3.10 is required to run ElectrumX') - - environ['DAEMON_URL'] = '' # Avoid Env erroring out - env = Env() - db = DB(env) - await db.open_for_compacting() - - assert not db.first_sync - history = db.history - # Continue where we left off, if interrupted - if history.comp_cursor == -1: - history.comp_cursor = 0 - - history.comp_flush_count = max(history.comp_flush_count, 1) - limit = 8 * 1000 * 1000 - - while history.comp_cursor != -1: - history._compact_history(limit) - - # When completed also update the UTXO flush count - db.set_flush_count(history.flush_count) - - -def main(): - logging.basicConfig(level=logging.INFO) - logging.info('Starting history compaction...') - loop = asyncio.get_event_loop() - try: - loop.run_until_complete(compact_history()) - except Exception: - traceback.print_exc() - logging.critical('History compaction terminated abnormally') - else: - logging.info('History compaction complete') - - -if __name__ == '__main__': - main() diff --git a/src/electrumx/server/block_processor.py b/src/electrumx/server/block_processor.py index 23ae54378..01dd95a5b 100644 --- a/src/electrumx/server/block_processor.py +++ b/src/electrumx/server/block_processor.py @@ -196,7 +196,7 @@ def __init__(self, env: 'Env', db: DB, daemon: Daemon, notifications: 'Notificat # Caches of unflushed items. self.headers = [] - self.tx_hashes = [] + self.tx_hashes = [] # type: List[bytes] self.undo_infos = [] # type: List[Tuple[Sequence[bytes], int]] # UTXO cache @@ -357,9 +357,16 @@ def estimate_txs_remaining(self): def flush_data(self): '''The data for a flush. The lock must be taken.''' assert self.state_lock.locked() - return FlushData(self.height, self.tx_count, self.headers, - self.tx_hashes, self.undo_infos, self.utxo_cache, - self.db_deletes, self.tip) + return FlushData( + height=self.height, + tx_count=self.tx_count, + headers=self.headers, + block_tx_hashes=self.tx_hashes, + undo_infos=self.undo_infos, + adds=self.utxo_cache, + deletes=self.db_deletes, + tip=self.tip, + ) async def flush(self, flush_utxos): def flush(): @@ -378,7 +385,7 @@ async def _maybe_flush(self): await self.flush(flush_arg) self.next_cache_check = time.monotonic() + 30 - def check_cache_size(self): + def check_cache_size(self) -> Optional[bool]: '''Flush a cache if it gets too big.''' # Good average estimates based on traversal of subobjects and # requesting size from Python (see deep_getsizeof). diff --git a/src/electrumx/server/db.py b/src/electrumx/server/db.py index b3d93c37e..9e4e23a00 100644 --- a/src/electrumx/server/db.py +++ b/src/electrumx/server/db.py @@ -106,13 +106,11 @@ def __init__(self, env: 'Env'): # "undo data: list of UTXOs spent at block height" self.utxo_db = None - self.utxo_flush_count = 0 self.fs_height = -1 self.fs_tx_count = 0 self.db_height = -1 self.db_tx_count = 0 self.db_tip = None # type: Optional[bytes] - self.tx_counts = None self.last_flush = time.time() self.last_flush_tx_count = 0 self.wall_time = 0 @@ -128,6 +126,7 @@ def __init__(self, env: 'Env'): # on-disk: raw block headers in chain order self.headers_file = util.LogicalFile('meta/headers', 2, 16000000) # on-disk: cumulative number of txs at the end of height N + self.tx_counts = None # type: Optional[array] self.tx_counts_file = util.LogicalFile('meta/txcounts', 2, 2000000) # on-disk: 32 byte txids in chain order, allows (tx_num -> txid) map self.hashes_file = util.LogicalFile('meta/hashes', 4, 16000000) @@ -149,7 +148,7 @@ async def _read_tx_counts(self): else: assert self.db_tx_count == 0 - async def _open_dbs(self, for_sync: bool, compacting: bool): + async def _open_dbs(self, *, for_sync: bool): assert self.utxo_db is None # First UTXO DB @@ -168,17 +167,16 @@ async def _open_dbs(self, for_sync: bool, compacting: bool): self.read_utxo_state() # Then history DB - self.utxo_flush_count = self.history.open_db(self.db_class, for_sync, - self.utxo_flush_count, - compacting) + self.history.open_db( + db_class=self.db_class, + for_sync=for_sync, + utxo_db_tx_count=self.db_tx_count, + ) self.clear_excess_undo_info() # Read TX counts (requires meta directory) await self._read_tx_counts() - async def open_for_compacting(self): - await self._open_dbs(True, True) - async def open_for_sync(self): '''Open the databases to sync to the daemon. @@ -186,7 +184,7 @@ async def open_for_sync(self): synchronization. When serving clients we want the open files for serving network connections. ''' - await self._open_dbs(True, False) + await self._open_dbs(for_sync=True) async def open_for_serving(self): '''Open the databases for serving. If they are already open they are @@ -197,7 +195,7 @@ async def open_for_serving(self): self.utxo_db.close() self.history.close_db() self.utxo_db = None - await self._open_dbs(False, False) + await self._open_dbs(for_sync=False) # Header merkle cache @@ -253,7 +251,7 @@ def flush_dbs(self, flush_data, flush_utxos, estimate_txs_remaining): self.flush_state(self.utxo_db) elapsed = self.last_flush - start_time - self.logger.info(f'flush #{self.history.flush_count:,d} took ' + self.logger.info(f'flush took ' f'{elapsed:.1f}s. Height {flush_data.height:,d} ' f'txs: {flush_data.tx_count:,d} ({tx_delta:+,d})') @@ -352,7 +350,6 @@ def flush_utxo_db(self, batch, flush_data: FlushData): f'{spend_count:,d} spends in ' f'{elapsed:.1f}s, committing...') - self.utxo_flush_count = self.history.flush_count self.db_height = flush_data.height self.db_tx_count = flush_data.tx_count self.db_tip = flush_data.tip @@ -383,7 +380,7 @@ def flush_backup(self, flush_data, touched): self.flush_state(batch) elapsed = self.last_flush - start_time - self.logger.info(f'backup flush #{self.history.flush_count:,d} took ' + self.logger.info(f'backup flush took ' f'{elapsed:.1f}s. Height {flush_data.height:,d} ' f'txs: {flush_data.tx_count:,d} ({tx_delta:+,d})') @@ -594,7 +591,6 @@ def read_utxo_state(self): self.db_tx_count = 0 self.db_tip = b'\0' * 32 self.db_version = max(self.DB_VERSIONS) - self.utxo_flush_count = 0 self.wall_time = 0 self.first_sync = True else: @@ -616,7 +612,6 @@ def read_utxo_state(self): self.db_height = state['height'] self.db_tx_count = state['tx_count'] self.db_tip = state['tip'] - self.utxo_flush_count = state['utxo_flush_count'] self.wall_time = state['wall_time'] self.first_sync = state['first_sync'] @@ -734,18 +729,12 @@ def write_utxo_state(self, batch): 'height': self.db_height, 'tx_count': self.db_tx_count, 'tip': self.db_tip, - 'utxo_flush_count': self.utxo_flush_count, 'wall_time': self.wall_time, 'first_sync': self.first_sync, 'db_version': self.db_version, } batch.put(b'state', repr(state).encode()) - def set_flush_count(self, count): - self.utxo_flush_count = count - with self.utxo_db.write_batch() as batch: - self.write_utxo_state(batch) - async def all_utxos(self, hashX): '''Return all UTXOs for an address sorted in no particular order.''' def read_utxos(): diff --git a/src/electrumx/server/history.py b/src/electrumx/server/history.py index dc7dd8d54..1cd2f0008 100644 --- a/src/electrumx/server/history.py +++ b/src/electrumx/server/history.py @@ -25,46 +25,37 @@ TXNUM_LEN = 5 -FLUSHID_LEN = 2 class History: - DB_VERSIONS = (0, 1) + DB_VERSIONS = (0, 1, 2) db: Optional['Storage'] def __init__(self): self.logger = util.class_logger(__name__, self.__class__.__name__) - # For history compaction - self.max_hist_row_entries = 12500 self.unflushed = defaultdict(bytearray) self.unflushed_count = 0 - self.flush_count = 0 - self.comp_flush_count = -1 - self.comp_cursor = -1 + self.hist_db_tx_count = 0 + self.hist_db_tx_count_next = 0 # after next flush, next value for self.hist_db_tx_count self.db_version = max(self.DB_VERSIONS) self.upgrade_cursor = -1 - # Key: address_hashX + flush_id - # Value: sorted "list" of tx_nums in history of hashX + # Key: address_hashX + tx_num + # Value: self.db = None def open_db( self, + *, db_class: Type['Storage'], for_sync: bool, - utxo_flush_count: int, - compacting: bool, - ): + utxo_db_tx_count: int, + ) -> None: self.db = db_class('hist', for_sync) self.read_state() - self.clear_excess(utxo_flush_count) - # An incomplete compaction needs to be cancelled otherwise - # restarting it will corrupt the history - if not compacting: - self._cancel_compaction() - return self.flush_count + self.clear_excess(utxo_db_tx_count) def close_db(self): if self.db: @@ -77,17 +68,10 @@ def read_state(self): state = ast.literal_eval(state.decode()) if not isinstance(state, dict): raise RuntimeError('failed reading state from history DB') - self.flush_count = state['flush_count'] - self.comp_flush_count = state.get('comp_flush_count', -1) - self.comp_cursor = state.get('comp_cursor', -1) self.db_version = state.get('db_version', 0) self.upgrade_cursor = state.get('upgrade_cursor', -1) - else: - self.flush_count = 0 - self.comp_flush_count = -1 - self.comp_cursor = -1 - self.db_version = max(self.DB_VERSIONS) - self.upgrade_cursor = -1 + self.hist_db_tx_count = state.get('hist_db_tx_count', 0) + self.hist_db_tx_count_next = self.hist_db_tx_count if self.db_version not in self.DB_VERSIONS: msg = (f'your history DB version is {self.db_version} but ' @@ -97,26 +81,37 @@ def read_state(self): if self.db_version != max(self.DB_VERSIONS): self.upgrade_db() self.logger.info(f'history DB version: {self.db_version}') - self.logger.info(f'flush count: {self.flush_count:,d}') - def clear_excess(self, utxo_flush_count): - # < might happen at end of compaction as both DBs cannot be - # updated atomically - if self.flush_count <= utxo_flush_count: + def clear_excess(self, utxo_db_tx_count: int) -> None: + # self.hist_db_tx_count != utxo_db_tx_count might happen as + # both DBs cannot be updated atomically + # FIXME when advancing blocks, hist_db is flushed first, so its count can be higher; + # but when backing up (e.g. reorg), hist_db is flushed first as well, + # so its count can be lower?! + # Shouldn't we flush utxo_db first when backing up? + if self.hist_db_tx_count <= utxo_db_tx_count: + assert self.hist_db_tx_count == utxo_db_tx_count return self.logger.info('DB shut down uncleanly. Scanning for ' 'excess history flushes...') + key_len = HASHX_LEN + TXNUM_LEN + txnum_padding = bytes(8-TXNUM_LEN) keys = [] - for key, _hist in self.db.iterator(prefix=b''): - flush_id, = unpack_be_uint16_from(key[-FLUSHID_LEN:]) - if flush_id > utxo_flush_count: - keys.append(key) + for db_key, db_val in self.db.iterator(prefix=b''): + # Ignore non-history entries + if len(db_key) != key_len: + continue + tx_numb = db_key[HASHX_LEN:] + tx_num, = unpack_le_uint64(tx_numb + txnum_padding) + if tx_num >= utxo_db_tx_count: + keys.append(db_key) self.logger.info(f'deleting {len(keys):,d} history entries') - self.flush_count = utxo_flush_count + self.hist_db_tx_count = utxo_db_tx_count + self.hist_db_tx_count_next = self.hist_db_tx_count with self.db.write_batch() as batch: for key in keys: batch.delete(key) @@ -127,19 +122,17 @@ def clear_excess(self, utxo_flush_count): def write_state(self, batch): '''Write state to the history DB.''' state = { - 'flush_count': self.flush_count, - 'comp_flush_count': self.comp_flush_count, - 'comp_cursor': self.comp_cursor, + 'hist_db_tx_count': self.hist_db_tx_count, 'db_version': self.db_version, 'upgrade_cursor': self.upgrade_cursor, } - # History entries are not prefixed; the suffix \0\0 ensures we - # look similar to other entries and aren't interfered with + # History entries are not prefixed; the suffix \0\0 is just for legacy reasons batch.put(b'state\0\0', repr(state).encode()) def add_unflushed(self, hashXs_by_tx, first_tx_num): unflushed = self.unflushed count = 0 + tx_num = None for tx_num, hashXs in enumerate(hashXs_by_tx, start=first_tx_num): tx_numb = pack_le_uint64(tx_num)[:TXNUM_LEN] hashXs = set(hashXs) @@ -147,6 +140,9 @@ def add_unflushed(self, hashXs_by_tx, first_tx_num): unflushed[hashX] += tx_numb count += len(hashXs) self.unflushed_count += count + if tx_num is not None: + assert self.hist_db_tx_count_next + len(hashXs_by_tx) == tx_num + 1 + self.hist_db_tx_count_next = tx_num + 1 def unflushed_memsize(self): return len(self.unflushed) * 180 + self.unflushed_count * TXNUM_LEN @@ -156,14 +152,15 @@ def assert_flushed(self): def flush(self): start_time = time.monotonic() - self.flush_count += 1 - flush_id = pack_be_uint16(self.flush_count) unflushed = self.unflushed + chunks = util.chunks with self.db.write_batch() as batch: for hashX in sorted(unflushed): - key = hashX + flush_id - batch.put(key, bytes(unflushed[hashX])) + for tx_num in chunks(unflushed[hashX], TXNUM_LEN): + db_key = hashX + tx_num + batch.put(db_key, b'') + self.hist_db_tx_count = self.hist_db_tx_count_next self.write_state(batch) count = len(unflushed) @@ -176,34 +173,24 @@ def flush(self): f'for {count:,d} addrs') def backup(self, hashXs, tx_count): - # Not certain this is needed, but it doesn't hurt - self.flush_count += 1 + self.assert_flushed() nremoves = 0 - bisect_left = bisect.bisect_left - chunks = util.chunks - txnum_padding = bytes(8-TXNUM_LEN) with self.db.write_batch() as batch: for hashX in sorted(hashXs): deletes = [] - puts = {} - for key, hist in self.db.iterator(prefix=hashX, reverse=True): - a = array( - 'Q', - b''.join(item + txnum_padding for item in chunks(hist, TXNUM_LEN)) - ) - # Remove all history entries >= tx_count - idx = bisect_left(a, tx_count) - nremoves += len(a) - idx - if idx > 0: - puts[key] = hist[:TXNUM_LEN * idx] + for db_key, db_val in self.db.iterator(prefix=hashX, reverse=True): + tx_numb = db_key[HASHX_LEN:] + tx_num, = unpack_le_uint64(tx_numb + txnum_padding) + if tx_num >= tx_count: + nremoves += 1 + deletes.append(db_key) + else: break - deletes.append(key) - for key in deletes: batch.delete(key) - for key, value in puts.items(): - batch.put(key, value) + self.hist_db_tx_count = tx_count + self.hist_db_tx_count_next = self.hist_db_tx_count self.write_state(batch) self.logger.info(f'backing up removed {nremoves:,d} history entries') @@ -214,191 +201,115 @@ def get_txnums(self, hashX, limit=1000): transactions. By default yields at most 1000 entries. Set limit to None to get them all. ''' limit = util.resolve_limit(limit) - chunks = util.chunks txnum_padding = bytes(8-TXNUM_LEN) - for _key, hist in self.db.iterator(prefix=hashX): - for tx_numb in chunks(hist, TXNUM_LEN): - if limit == 0: - return - tx_num, = unpack_le_uint64(tx_numb + txnum_padding) - yield tx_num - limit -= 1 - - # - # History compaction - # - - # comp_cursor is a cursor into compaction progress. - # -1: no compaction in progress - # 0-65535: Compaction in progress; all prefixes < comp_cursor have - # been compacted, and later ones have not. - # 65536: compaction complete in-memory but not flushed - # - # comp_flush_count applies during compaction, and is a flush count - # for history with prefix < comp_cursor. flush_count applies - # to still uncompacted history. It is -1 when no compaction is - # taking place. Key suffixes up to and including comp_flush_count - # are used, so a parallel history flush must first increment this - # - # When compaction is complete and the final flush takes place, - # flush_count is reset to comp_flush_count, and comp_flush_count to -1 - - def _flush_compaction(self, cursor, write_items, keys_to_delete): - '''Flush a single compaction pass as a batch.''' - # Update compaction state - if cursor == 65536: - self.flush_count = self.comp_flush_count - self.comp_cursor = -1 - self.comp_flush_count = -1 - else: - self.comp_cursor = cursor - - # History DB. Flush compacted history and updated state - with self.db.write_batch() as batch: - # Important: delete first! The keyspace may overlap. - for key in keys_to_delete: - batch.delete(key) - for key, value in write_items: - batch.put(key, value) - self.write_state(batch) - - def _compact_hashX(self, hashX, hist_map, hist_list, - write_items, keys_to_delete): - '''Compres history for a hashX. hist_list is an ordered list of - the histories to be compressed.''' - # History entries (tx numbers) are TXNUM_LEN bytes each. Distribute - # over rows of up to 50KB in size. A fixed row size means - # future compactions will not need to update the first N - 1 - # rows. - max_row_size = self.max_hist_row_entries * TXNUM_LEN - full_hist = b''.join(hist_list) - nrows = (len(full_hist) + max_row_size - 1) // max_row_size - if nrows > 4: - self.logger.info( - f'hashX {hash_to_hex_str(hashX)} is large: ' - f'{len(full_hist) // TXNUM_LEN:,d} entries across {nrows:,d} rows' - ) - - # Find what history needs to be written, and what keys need to - # be deleted. Start by assuming all keys are to be deleted, - # and then remove those that are the same on-disk as when - # compacted. - write_size = 0 - keys_to_delete.update(hist_map) - for n, chunk in enumerate(util.chunks(full_hist, max_row_size)): - key = hashX + pack_be_uint16(n) - if hist_map.get(key) == chunk: - keys_to_delete.remove(key) - else: - write_items.append((key, chunk)) - write_size += len(chunk) - - assert n + 1 == nrows - self.comp_flush_count = max(self.comp_flush_count, n) - - return write_size - - def _compact_prefix(self, prefix, write_items, keys_to_delete): - '''Compact all history entries for hashXs beginning with the - given prefix. Update keys_to_delete and write.''' - prior_hashX = None - hist_map = {} - hist_list = [] - - key_len = HASHX_LEN + FLUSHID_LEN - write_size = 0 - for key, hist in self.db.iterator(prefix=prefix): - # Ignore non-history entries - if len(key) != key_len: - continue - hashX = key[:-FLUSHID_LEN] - if hashX != prior_hashX and prior_hashX: - write_size += self._compact_hashX(prior_hashX, hist_map, - hist_list, write_items, - keys_to_delete) - hist_map.clear() - hist_list.clear() - prior_hashX = hashX - hist_map[key] = hist - hist_list.append(hist) - - if prior_hashX: - write_size += self._compact_hashX(prior_hashX, hist_map, hist_list, - write_items, keys_to_delete) - return write_size - - def _compact_history(self, limit): - '''Inner loop of history compaction. Loops until limit bytes have - been processed. - ''' - keys_to_delete = set() - write_items = [] # A list of (key, value) pairs - write_size = 0 - - # Loop over 2-byte prefixes - cursor = self.comp_cursor - while write_size < limit and cursor < 65536: - prefix = pack_be_uint16(cursor) - write_size += self._compact_prefix(prefix, write_items, - keys_to_delete) - cursor += 1 - - max_rows = self.comp_flush_count + 1 - self._flush_compaction(cursor, write_items, keys_to_delete) - - self.logger.info( - f'history compaction: wrote {len(write_items):,d} rows ' - f'({write_size / 1000000:.1f} MB), removed ' - f'{len(keys_to_delete):,d} rows, largest: {max_rows:,d}, ' - f'{100 * cursor / 65536:.1f}% complete' - ) - return write_size - - def _cancel_compaction(self): - if self.comp_cursor != -1: - self.logger.warning('cancelling in-progress history compaction') - self.comp_flush_count = -1 - self.comp_cursor = -1 + for db_key, db_val in self.db.iterator(prefix=hashX): + tx_numb = db_key[HASHX_LEN:] + if limit == 0: + return + tx_num, = unpack_le_uint64(tx_numb + txnum_padding) + yield tx_num + limit -= 1 # # DB upgrade # def upgrade_db(self): - self.logger.info(f'history DB version: {self.db_version}') + self.logger.info(f'history DB current version: {self.db_version}. ' + f'latest is: {max(self.DB_VERSIONS)}') self.logger.info('Upgrading your history DB; this can take some time...') - def upgrade_cursor(cursor): + def convert_version_1(): + def upgrade_cursor(cursor): + count = 0 + prefix = pack_be_uint16(cursor) + key_len = HASHX_LEN + 2 + chunks = util.chunks + with self.db.write_batch() as batch: + batch_put = batch.put + for key, hist in self.db.iterator(prefix=prefix): + # Ignore non-history entries + if len(key) != key_len: + continue + count += 1 + hist = b''.join(item + b'\0' for item in chunks(hist, 4)) + batch_put(key, hist) + self.upgrade_cursor = cursor + self.write_state(batch) + return count + + last = time.monotonic() count = 0 - prefix = pack_be_uint16(cursor) - key_len = HASHX_LEN + 2 - chunks = util.chunks + + for cursor in range(self.upgrade_cursor + 1, 65536): + count += upgrade_cursor(cursor) + now = time.monotonic() + if now > last + 10: + last = now + self.logger.info(f'history DB v0->v1: {count:,d} entries updated, ' + f'{cursor * 100 / 65536:.1f}% complete') + + self.db_version = 1 + self.upgrade_cursor = -1 with self.db.write_batch() as batch: - batch_put = batch.put - for key, hist in self.db.iterator(prefix=prefix): - # Ignore non-history entries - if len(key) != key_len: - continue - count += 1 - hist = b''.join(item + b'\0' for item in chunks(hist, 4)) - batch_put(key, hist) - self.upgrade_cursor = cursor self.write_state(batch) - return count + self.logger.info('history DB upgraded to v1 successfully') + + def convert_version_2(): + # old schema: + # Key: address_hashX + flush_id + # Value: sorted "list" of tx_nums in history of hashX + # ----- + # new schema: + # Key: address_hashX + tx_num + # Value: + + def upgrade_cursor(cursor): + count = 0 + prefix = pack_be_uint16(cursor) + key_len = HASHX_LEN + 2 + chunks = util.chunks + txnum_padding = bytes(8-TXNUM_LEN) + with self.db.write_batch() as batch: + batch_put = batch.put + batch_delete = batch.delete + max_tx_num = 0 + for db_key, db_val in self.db.iterator(prefix=prefix): + # Ignore non-history entries + if len(db_key) != key_len: + continue + count += 1 + batch_delete(db_key) + hashX = db_key[:HASHX_LEN] + for tx_numb in chunks(db_val, 5): + batch_put(hashX + tx_numb, b'') + tx_num, = unpack_le_uint64(tx_numb + txnum_padding) + max_tx_num = max(max_tx_num, tx_num) + self.upgrade_cursor = cursor + self.hist_db_tx_count = max(self.hist_db_tx_count, max_tx_num + 1) + self.hist_db_tx_count_next = self.hist_db_tx_count + self.write_state(batch) + return count + + last = time.monotonic() + count = 0 - last = time.monotonic() - count = 0 + for cursor in range(self.upgrade_cursor + 1, 65536): + count += upgrade_cursor(cursor) + now = time.monotonic() + if now > last + 10: + last = now + self.logger.info(f'history DB v1->v2: {count:,d} entries updated, ' + f'{cursor * 100 / 65536:.1f}% complete') - for cursor in range(self.upgrade_cursor + 1, 65536): - count += upgrade_cursor(cursor) - now = time.monotonic() - if now > last + 10: - last = now - self.logger.info(f'DB 3 of 3: {count:,d} entries updated, ' - f'{cursor * 100 / 65536:.1f}% complete') + self.db_version = 2 + self.upgrade_cursor = -1 + with self.db.write_batch() as batch: + self.write_state(batch) + self.logger.info('history DB upgraded to v2 successfully') + if self.db_version == 0: + convert_version_1() + if self.db_version == 1: + convert_version_2() self.db_version = max(self.DB_VERSIONS) - self.upgrade_cursor = -1 - with self.db.write_batch() as batch: - self.write_state(batch) - self.logger.info('DB 3 of 3 upgraded successfully') diff --git a/src/electrumx/server/session.py b/src/electrumx/server/session.py index cd76b33e0..7a4ad9107 100644 --- a/src/electrumx/server/session.py +++ b/src/electrumx/server/session.py @@ -337,7 +337,6 @@ def cache_fmt(cache: LRUCache): 'daemon': self.daemon.logged_url(), 'daemon height': self.daemon.cached_height(), 'db height': self.db.db_height, - 'db_flush_count': self.db.history.flush_count, 'groups': len(self.session_groups), 'history cache': cache_fmt(self._history_cache), 'merkle txid cache': cache_fmt(self._merkle_txid_cache), diff --git a/tests/server/test_compaction.py b/tests/server/test_compaction.py deleted file mode 100644 index ad6c96a43..000000000 --- a/tests/server/test_compaction.py +++ /dev/null @@ -1,133 +0,0 @@ -'''Test of compaction code in server/history.py''' -import array -import random -from os import environ, urandom - -import pytest - -from electrumx.lib.hash import HASHX_LEN -from electrumx.lib.util import pack_be_uint16, pack_le_uint64 -from electrumx.server.env import Env -from electrumx.server.db import DB - - -def create_histories(history, hashX_count=100): - '''Creates a bunch of random transaction histories, and write them - to disk in a series of small flushes.''' - hashXs = [urandom(HASHX_LEN) for n in range(hashX_count)] - mk_array = lambda : array.array('Q') - histories = {hashX : mk_array() for hashX in hashXs} - unflushed = history.unflushed - tx_num = 0 - while hashXs: - tx_numb = pack_le_uint64(tx_num)[:5] - hash_indexes = set(random.randrange(len(hashXs)) - for n in range(1 + random.randrange(4))) - for index in hash_indexes: - histories[hashXs[index]].append(tx_num) - unflushed[hashXs[index]].extend(tx_numb) - - tx_num += 1 - # Occasionally flush and drop a random hashX if non-empty - if random.random() < 0.1: - history.flush() - index = random.randrange(0, len(hashXs)) - if histories[hashXs[index]]: - del hashXs[index] - - return histories - - -def check_hashX_compaction(history): - history.max_hist_row_entries = 40 - row_size = history.max_hist_row_entries * 5 - full_hist = b''.join(pack_le_uint64(tx_num)[:5] for tx_num in range(100)) - hashX = urandom(HASHX_LEN) - pairs = ((1, 20), (26, 50), (56, 30)) - - cum = 0 - hist_list = [] - hist_map = {} - for flush_count, count in pairs: - key = hashX + pack_be_uint16(flush_count) - hist = full_hist[cum * 5: (cum+count) * 5] - hist_map[key] = hist - hist_list.append(hist) - cum += count - - write_items = [] - keys_to_delete = set() - write_size = history._compact_hashX(hashX, hist_map, hist_list, - write_items, keys_to_delete) - # Check results for sanity - assert write_size == len(full_hist) - assert len(write_items) == 3 - assert len(keys_to_delete) == 3 - assert len(hist_map) == len(pairs) - for n, item in enumerate(write_items): - assert item == (hashX + pack_be_uint16(n), - full_hist[n * row_size: (n + 1) * row_size]) - for flush_count, count in pairs: - assert hashX + pack_be_uint16(flush_count) in keys_to_delete - - # Check re-compaction is null - hist_map = {key: value for key, value in write_items} - hist_list = [value for key, value in write_items] - write_items.clear() - keys_to_delete.clear() - write_size = history._compact_hashX(hashX, hist_map, hist_list, - write_items, keys_to_delete) - assert write_size == 0 - assert len(write_items) == 0 - assert len(keys_to_delete) == 0 - assert len(hist_map) == len(pairs) - - # Check re-compaction adding a single tx writes the one row - hist_list[-1] += array.array('I', [100]).tobytes() - write_size = history._compact_hashX(hashX, hist_map, hist_list, - write_items, keys_to_delete) - assert write_size == len(hist_list[-1]) - assert write_items == [(hashX + pack_be_uint16(2), hist_list[-1])] - assert len(keys_to_delete) == 1 - assert write_items[0][0] in keys_to_delete - assert len(hist_map) == len(pairs) - - -def check_written(history, histories): - for hashX, hist in histories.items(): - db_hist = array.array('I', history.get_txnums(hashX, limit=None)) - assert hist == db_hist - - -def compact_history(history): - '''Synchronously compact the DB history.''' - history.comp_cursor = 0 - - history.comp_flush_count = max(history.comp_flush_count, 1) - limit = 5 * 1000 - - write_size = 0 - while history.comp_cursor != -1: - write_size += history._compact_history(limit) - assert write_size != 0 - - -@pytest.mark.asyncio -async def test_compaction(tmpdir): - db_dir = str(tmpdir) - print(f'Temp dir: {db_dir}') - environ.clear() - environ['DB_DIRECTORY'] = db_dir - environ['DAEMON_URL'] = '' - environ['COIN'] = 'BitcoinSV' - db = DB(Env()) - await db.open_for_serving() - history = db.history - - # Test abstract compaction - check_hashX_compaction(history) - # Now test in with random data - histories = create_histories(history) - check_written(history, histories) - compact_history(history) - check_written(history, histories) From 24c6ea1a5880d9620cb3bf1fed2d63352c3b670d Mon Sep 17 00:00:00 2001 From: SomberNight Date: Wed, 28 Oct 2020 16:11:20 +0100 Subject: [PATCH 02/27] history db: change schema: prefix entries with b'H' --- src/electrumx/server/history.py | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/src/electrumx/server/history.py b/src/electrumx/server/history.py index 1cd2f0008..596114bfe 100644 --- a/src/electrumx/server/history.py +++ b/src/electrumx/server/history.py @@ -29,7 +29,7 @@ class History: - DB_VERSIONS = (0, 1, 2) + DB_VERSIONS = (3, ) db: Optional['Storage'] @@ -42,7 +42,7 @@ def __init__(self): self.db_version = max(self.DB_VERSIONS) self.upgrade_cursor = -1 - # Key: address_hashX + tx_num + # Key: b'H' + address_hashX + tx_num # Value: self.db = None @@ -96,14 +96,10 @@ def clear_excess(self, utxo_db_tx_count: int) -> None: self.logger.info('DB shut down uncleanly. Scanning for ' 'excess history flushes...') - key_len = HASHX_LEN + TXNUM_LEN txnum_padding = bytes(8-TXNUM_LEN) keys = [] - for db_key, db_val in self.db.iterator(prefix=b''): - # Ignore non-history entries - if len(db_key) != key_len: - continue - tx_numb = db_key[HASHX_LEN:] + for db_key, db_val in self.db.iterator(prefix=b'H'): + tx_numb = db_key[-TXNUM_LEN:] tx_num, = unpack_le_uint64(tx_numb + txnum_padding) if tx_num >= utxo_db_tx_count: keys.append(db_key) @@ -158,7 +154,7 @@ def flush(self): with self.db.write_batch() as batch: for hashX in sorted(unflushed): for tx_num in chunks(unflushed[hashX], TXNUM_LEN): - db_key = hashX + tx_num + db_key = b'H' + hashX + tx_num batch.put(db_key, b'') self.hist_db_tx_count = self.hist_db_tx_count_next self.write_state(batch) @@ -179,8 +175,9 @@ def backup(self, hashXs, tx_count): with self.db.write_batch() as batch: for hashX in sorted(hashXs): deletes = [] - for db_key, db_val in self.db.iterator(prefix=hashX, reverse=True): - tx_numb = db_key[HASHX_LEN:] + prefix = b'H' + hashX + for db_key, db_val in self.db.iterator(prefix=prefix, reverse=True): + tx_numb = db_key[-TXNUM_LEN:] tx_num, = unpack_le_uint64(tx_numb + txnum_padding) if tx_num >= tx_count: nremoves += 1 @@ -202,8 +199,9 @@ def get_txnums(self, hashX, limit=1000): limit to None to get them all. ''' limit = util.resolve_limit(limit) txnum_padding = bytes(8-TXNUM_LEN) - for db_key, db_val in self.db.iterator(prefix=hashX): - tx_numb = db_key[HASHX_LEN:] + prefix = b'H' + hashX + for db_key, db_val in self.db.iterator(prefix=prefix): + tx_numb = db_key[-TXNUM_LEN:] if limit == 0: return tx_num, = unpack_le_uint64(tx_numb + txnum_padding) From 0861c15fa184f1574dd033aae032f6f9c778645b Mon Sep 17 00:00:00 2001 From: SomberNight Date: Wed, 28 Oct 2020 19:11:23 +0100 Subject: [PATCH 03/27] db: rm upgrade logic with the pending db changes, an upgrade is ~as fast as a resync from genesis --- src/electrumx/server/db.py | 90 +------------------------- src/electrumx/server/history.py | 111 +------------------------------- 2 files changed, 6 insertions(+), 195 deletions(-) diff --git a/src/electrumx/server/db.py b/src/electrumx/server/db.py index 9e4e23a00..14bf22860 100644 --- a/src/electrumx/server/db.py +++ b/src/electrumx/server/db.py @@ -585,7 +585,7 @@ def clear_excess_undo_info(self): # -- UTXO database def read_utxo_state(self): - state = self.utxo_db.get(b'state') + state = self.utxo_db.get(b'\0state') if not state: self.db_height = -1 self.db_tx_count = 0 @@ -622,7 +622,7 @@ def read_utxo_state(self): # Upgrade DB if self.db_version != max(self.DB_VERSIONS): - self.upgrade_db() + pass # call future upgrade logic here # Log some stats self.logger.info(f'UTXO DB version: {self.db_version:d}') @@ -638,90 +638,6 @@ def read_utxo_state(self): f'sync time so far: {util.formatted_time(self.wall_time)}' ) - def upgrade_db(self): - self.logger.info(f'UTXO DB version: {self.db_version}') - self.logger.info('Upgrading your DB; this can take some time...') - - def upgrade_u_prefix(prefix): - count = 0 - with self.utxo_db.write_batch() as batch: - batch_delete = batch.delete - batch_put = batch.put - # Key: b'u' + address_hashX + tx_idx + tx_num - for db_key, db_value in self.utxo_db.iterator(prefix=prefix): - if len(db_key) == 21: - return - break - if self.db_version == 6: - for db_key, db_value in self.utxo_db.iterator(prefix=prefix): - count += 1 - batch_delete(db_key) - batch_put(db_key[:14] + b'\0\0' + db_key[14:] + b'\0', db_value) - else: - for db_key, db_value in self.utxo_db.iterator(prefix=prefix): - count += 1 - batch_delete(db_key) - batch_put(db_key + b'\0', db_value) - return count - - last = time.monotonic() - count = 0 - for cursor in range(65536): - prefix = b'u' + pack_be_uint16(cursor) - count += upgrade_u_prefix(prefix) - now = time.monotonic() - if now > last + 10: - last = now - self.logger.info(f'DB 1 of 3: {count:,d} entries updated, ' - f'{cursor * 100 / 65536:.1f}% complete') - self.logger.info('DB 1 of 3 upgraded successfully') - - def upgrade_h_prefix(prefix): - count = 0 - with self.utxo_db.write_batch() as batch: - batch_delete = batch.delete - batch_put = batch.put - # Key: b'h' + compressed_tx_hash + tx_idx + tx_num - for db_key, db_value in self.utxo_db.iterator(prefix=prefix): - if len(db_key) == 14: - return - break - if self.db_version == 6: - for db_key, db_value in self.utxo_db.iterator(prefix=prefix): - count += 1 - batch_delete(db_key) - batch_put(db_key[:7] + b'\0\0' + db_key[7:] + b'\0', db_value) - else: - for db_key, db_value in self.utxo_db.iterator(prefix=prefix): - count += 1 - batch_delete(db_key) - batch_put(db_key + b'\0', db_value) - return count - - last = time.monotonic() - count = 0 - for cursor in range(65536): - prefix = b'h' + pack_be_uint16(cursor) - count += upgrade_h_prefix(prefix) - now = time.monotonic() - if now > last + 10: - last = now - self.logger.info(f'DB 2 of 3: {count:,d} entries updated, ' - f'{cursor * 100 / 65536:.1f}% complete') - - # Upgrade tx_counts file - size = (self.db_height + 1) * 8 - tx_counts = self.tx_counts_file.read(0, size) - if len(tx_counts) == (self.db_height + 1) * 4: - tx_counts = array('I', tx_counts) - tx_counts = array('Q', tx_counts) - self.tx_counts_file.write(0, tx_counts.tobytes()) - - self.db_version = max(self.DB_VERSIONS) - with self.utxo_db.write_batch() as batch: - self.write_utxo_state(batch) - self.logger.info('DB 2 of 3 upgraded successfully') - def write_utxo_state(self, batch): '''Write (UTXO) state to the batch.''' state = { @@ -733,7 +649,7 @@ def write_utxo_state(self, batch): 'first_sync': self.first_sync, 'db_version': self.db_version, } - batch.put(b'state', repr(state).encode()) + batch.put(b'\0state', repr(state).encode()) async def all_utxos(self, hashX): '''Return all UTXOs for an address sorted in no particular order.''' diff --git a/src/electrumx/server/history.py b/src/electrumx/server/history.py index 596114bfe..bbf19eb95 100644 --- a/src/electrumx/server/history.py +++ b/src/electrumx/server/history.py @@ -63,7 +63,7 @@ def close_db(self): self.db = None def read_state(self): - state = self.db.get(b'state\0\0') + state = self.db.get(b'\0state') if state: state = ast.literal_eval(state.decode()) if not isinstance(state, dict): @@ -79,7 +79,7 @@ def read_state(self): self.logger.error(msg) raise RuntimeError(msg) if self.db_version != max(self.DB_VERSIONS): - self.upgrade_db() + pass # call future upgrade logic here self.logger.info(f'history DB version: {self.db_version}') def clear_excess(self, utxo_db_tx_count: int) -> None: @@ -122,8 +122,7 @@ def write_state(self, batch): 'db_version': self.db_version, 'upgrade_cursor': self.upgrade_cursor, } - # History entries are not prefixed; the suffix \0\0 is just for legacy reasons - batch.put(b'state\0\0', repr(state).encode()) + batch.put(b'\0state', repr(state).encode()) def add_unflushed(self, hashXs_by_tx, first_tx_num): unflushed = self.unflushed @@ -207,107 +206,3 @@ def get_txnums(self, hashX, limit=1000): tx_num, = unpack_le_uint64(tx_numb + txnum_padding) yield tx_num limit -= 1 - - # - # DB upgrade - # - - def upgrade_db(self): - self.logger.info(f'history DB current version: {self.db_version}. ' - f'latest is: {max(self.DB_VERSIONS)}') - self.logger.info('Upgrading your history DB; this can take some time...') - - def convert_version_1(): - def upgrade_cursor(cursor): - count = 0 - prefix = pack_be_uint16(cursor) - key_len = HASHX_LEN + 2 - chunks = util.chunks - with self.db.write_batch() as batch: - batch_put = batch.put - for key, hist in self.db.iterator(prefix=prefix): - # Ignore non-history entries - if len(key) != key_len: - continue - count += 1 - hist = b''.join(item + b'\0' for item in chunks(hist, 4)) - batch_put(key, hist) - self.upgrade_cursor = cursor - self.write_state(batch) - return count - - last = time.monotonic() - count = 0 - - for cursor in range(self.upgrade_cursor + 1, 65536): - count += upgrade_cursor(cursor) - now = time.monotonic() - if now > last + 10: - last = now - self.logger.info(f'history DB v0->v1: {count:,d} entries updated, ' - f'{cursor * 100 / 65536:.1f}% complete') - - self.db_version = 1 - self.upgrade_cursor = -1 - with self.db.write_batch() as batch: - self.write_state(batch) - self.logger.info('history DB upgraded to v1 successfully') - - def convert_version_2(): - # old schema: - # Key: address_hashX + flush_id - # Value: sorted "list" of tx_nums in history of hashX - # ----- - # new schema: - # Key: address_hashX + tx_num - # Value: - - def upgrade_cursor(cursor): - count = 0 - prefix = pack_be_uint16(cursor) - key_len = HASHX_LEN + 2 - chunks = util.chunks - txnum_padding = bytes(8-TXNUM_LEN) - with self.db.write_batch() as batch: - batch_put = batch.put - batch_delete = batch.delete - max_tx_num = 0 - for db_key, db_val in self.db.iterator(prefix=prefix): - # Ignore non-history entries - if len(db_key) != key_len: - continue - count += 1 - batch_delete(db_key) - hashX = db_key[:HASHX_LEN] - for tx_numb in chunks(db_val, 5): - batch_put(hashX + tx_numb, b'') - tx_num, = unpack_le_uint64(tx_numb + txnum_padding) - max_tx_num = max(max_tx_num, tx_num) - self.upgrade_cursor = cursor - self.hist_db_tx_count = max(self.hist_db_tx_count, max_tx_num + 1) - self.hist_db_tx_count_next = self.hist_db_tx_count - self.write_state(batch) - return count - - last = time.monotonic() - count = 0 - - for cursor in range(self.upgrade_cursor + 1, 65536): - count += upgrade_cursor(cursor) - now = time.monotonic() - if now > last + 10: - last = now - self.logger.info(f'history DB v1->v2: {count:,d} entries updated, ' - f'{cursor * 100 / 65536:.1f}% complete') - - self.db_version = 2 - self.upgrade_cursor = -1 - with self.db.write_batch() as batch: - self.write_state(batch) - self.logger.info('history DB upgraded to v2 successfully') - - if self.db_version == 0: - convert_version_1() - if self.db_version == 1: - convert_version_2() - self.db_version = max(self.DB_VERSIONS) From c63a0bf00eee2b723cb87fda669458a0e827e3a3 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Fri, 30 Oct 2020 23:31:33 +0100 Subject: [PATCH 04/27] (trivial) make TXNUM_PADDING a global --- src/electrumx/server/block_processor.py | 5 ++--- src/electrumx/server/db.py | 8 +++----- src/electrumx/server/history.py | 10 ++++------ 3 files changed, 9 insertions(+), 14 deletions(-) diff --git a/src/electrumx/server/block_processor.py b/src/electrumx/server/block_processor.py index 01dd95a5b..04f684185 100644 --- a/src/electrumx/server/block_processor.py +++ b/src/electrumx/server/block_processor.py @@ -24,7 +24,7 @@ ) from electrumx.lib.tx import Tx from electrumx.server.db import FlushData, COMP_TXID_LEN, DB -from electrumx.server.history import TXNUM_LEN +from electrumx.server.history import TXNUM_LEN, TXNUM_PADDING if TYPE_CHECKING: from electrumx.lib.coins import Coin, Block @@ -634,7 +634,6 @@ def spend_utxo(self, tx_hash: bytes, tx_idx: int) -> bytes: return cache_value # Spend it from the DB. - txnum_padding = bytes(8-TXNUM_LEN) # Key: b'h' + compressed_tx_hash + tx_idx + tx_num # Value: hashX @@ -646,7 +645,7 @@ def spend_utxo(self, tx_hash: bytes, tx_idx: int) -> bytes: tx_num_packed = hdb_key[-TXNUM_LEN:] if len(candidates) > 1: - tx_num, = unpack_le_uint64(tx_num_packed + txnum_padding) + tx_num, = unpack_le_uint64(tx_num_packed + TXNUM_PADDING) hash, _height = self.db.fs_tx_hash(tx_num) if hash != tx_hash: assert hash is not None # Should always be found diff --git a/src/electrumx/server/db.py b/src/electrumx/server/db.py index 14bf22860..230d418b1 100644 --- a/src/electrumx/server/db.py +++ b/src/electrumx/server/db.py @@ -29,7 +29,7 @@ unpack_le_uint32, unpack_be_uint32, unpack_le_uint64 ) from electrumx.server.storage import db_class, Storage -from electrumx.server.history import History, TXNUM_LEN +from electrumx.server.history import History, TXNUM_LEN, TXNUM_PADDING if TYPE_CHECKING: from electrumx.server.env import Env @@ -656,13 +656,12 @@ async def all_utxos(self, hashX): def read_utxos(): utxos = [] utxos_append = utxos.append - txnum_padding = bytes(8-TXNUM_LEN) # Key: b'u' + address_hashX + txout_idx + tx_num # Value: the UTXO value as a 64-bit unsigned integer prefix = b'u' + hashX for db_key, db_value in self.utxo_db.iterator(prefix=prefix): txout_idx, = unpack_le_uint32(db_key[-TXNUM_LEN-4:-TXNUM_LEN]) - tx_num, = unpack_le_uint64(db_key[-TXNUM_LEN:] + txnum_padding) + tx_num, = unpack_le_uint64(db_key[-TXNUM_LEN:] + TXNUM_PADDING) value, = unpack_le_uint64(db_value) tx_hash, height = self.fs_tx_hash(tx_num) utxos_append(UTXO(tx_num, txout_idx, tx_hash, height, value)) @@ -688,7 +687,6 @@ def lookup_hashXs(): ''' def lookup_hashX(tx_hash, tx_idx): idx_packed = pack_le_uint32(tx_idx) - txnum_padding = bytes(8-TXNUM_LEN) # Key: b'h' + compressed_tx_hash + tx_idx + tx_num # Value: hashX @@ -697,7 +695,7 @@ def lookup_hashX(tx_hash, tx_idx): # Find which entry, if any, the TX_HASH matches. for db_key, hashX in self.utxo_db.iterator(prefix=prefix): tx_num_packed = db_key[-TXNUM_LEN:] - tx_num, = unpack_le_uint64(tx_num_packed + txnum_padding) + tx_num, = unpack_le_uint64(tx_num_packed + TXNUM_PADDING) hash, _height = self.fs_tx_hash(tx_num) if hash == tx_hash: return hashX, idx_packed + tx_num_packed diff --git a/src/electrumx/server/history.py b/src/electrumx/server/history.py index bbf19eb95..08812bd2e 100644 --- a/src/electrumx/server/history.py +++ b/src/electrumx/server/history.py @@ -25,6 +25,7 @@ TXNUM_LEN = 5 +TXNUM_PADDING = bytes(8 - TXNUM_LEN) class History: @@ -96,11 +97,10 @@ def clear_excess(self, utxo_db_tx_count: int) -> None: self.logger.info('DB shut down uncleanly. Scanning for ' 'excess history flushes...') - txnum_padding = bytes(8-TXNUM_LEN) keys = [] for db_key, db_val in self.db.iterator(prefix=b'H'): tx_numb = db_key[-TXNUM_LEN:] - tx_num, = unpack_le_uint64(tx_numb + txnum_padding) + tx_num, = unpack_le_uint64(tx_numb + TXNUM_PADDING) if tx_num >= utxo_db_tx_count: keys.append(db_key) @@ -170,14 +170,13 @@ def flush(self): def backup(self, hashXs, tx_count): self.assert_flushed() nremoves = 0 - txnum_padding = bytes(8-TXNUM_LEN) with self.db.write_batch() as batch: for hashX in sorted(hashXs): deletes = [] prefix = b'H' + hashX for db_key, db_val in self.db.iterator(prefix=prefix, reverse=True): tx_numb = db_key[-TXNUM_LEN:] - tx_num, = unpack_le_uint64(tx_numb + txnum_padding) + tx_num, = unpack_le_uint64(tx_numb + TXNUM_PADDING) if tx_num >= tx_count: nremoves += 1 deletes.append(db_key) @@ -197,12 +196,11 @@ def get_txnums(self, hashX, limit=1000): transactions. By default yields at most 1000 entries. Set limit to None to get them all. ''' limit = util.resolve_limit(limit) - txnum_padding = bytes(8-TXNUM_LEN) prefix = b'H' + hashX for db_key, db_val in self.db.iterator(prefix=prefix): tx_numb = db_key[-TXNUM_LEN:] if limit == 0: return - tx_num, = unpack_le_uint64(tx_numb + txnum_padding) + tx_num, = unpack_le_uint64(tx_numb + TXNUM_PADDING) yield tx_num limit -= 1 From be540c959a73db9acb75d475247a58c1e302f61e Mon Sep 17 00:00:00 2001 From: SomberNight Date: Fri, 30 Oct 2020 23:52:40 +0100 Subject: [PATCH 05/27] db: add parameter for TXOUTIDX_LEN --- src/electrumx/server/block_processor.py | 16 +++++++++------- src/electrumx/server/db.py | 10 ++++++---- src/electrumx/server/history.py | 2 ++ 3 files changed, 17 insertions(+), 11 deletions(-) diff --git a/src/electrumx/server/block_processor.py b/src/electrumx/server/block_processor.py index 04f684185..1e35d8607 100644 --- a/src/electrumx/server/block_processor.py +++ b/src/electrumx/server/block_processor.py @@ -24,7 +24,7 @@ ) from electrumx.lib.tx import Tx from electrumx.server.db import FlushData, COMP_TXID_LEN, DB -from electrumx.server.history import TXNUM_LEN, TXNUM_PADDING +from electrumx.server.history import TXNUM_LEN, TXNUM_PADDING, TXOUTIDX_LEN, TXOUTIDX_PADDING if TYPE_CHECKING: from electrumx.lib.coins import Coin, Block @@ -478,7 +478,7 @@ def advance_txs( # Get the hashX hashX = script_hashX(txout.pk_script) append_hashX(hashX) - put_utxo(tx_hash + to_le_uint32(idx), + put_utxo(tx_hash + to_le_uint32(idx)[:TXOUTIDX_LEN], hashX + tx_numb + to_le_uint64(txout.value)) append_hashXs(hashXs) @@ -559,7 +559,8 @@ def backup_txs( continue n -= undo_entry_len undo_item = undo_info[n:n + undo_entry_len] - put_utxo(txin.prev_hash + pack_le_uint32(txin.prev_idx), undo_item) + prevout = txin.prev_hash + pack_le_uint32(txin.prev_idx)[:TXOUTIDX_LEN] + put_utxo(prevout, undo_item) hashX = undo_item[:HASHX_LEN] touched.add(hashX) @@ -628,7 +629,7 @@ def spend_utxo(self, tx_hash: bytes, tx_idx: int) -> bytes: corruption. ''' # Fast track is it being in the cache - idx_packed = pack_le_uint32(tx_idx) + idx_packed = pack_le_uint32(tx_idx)[:TXOUTIDX_LEN] cache_value = self.utxo_cache.pop(tx_hash + idx_packed, None) if cache_value: return cache_value @@ -653,7 +654,7 @@ def spend_utxo(self, tx_hash: bytes, tx_idx: int) -> bytes: # Key: b'u' + address_hashX + tx_idx + tx_num # Value: the UTXO value as a 64-bit unsigned integer - udb_key = b'u' + hashX + hdb_key[-4-TXNUM_LEN:] + udb_key = b'u' + hashX + hdb_key[-TXOUTIDX_LEN-TXNUM_LEN:] utxo_value_packed = self.db.utxo_db.get(udb_key) if utxo_value_packed: # Remove both entries for this UTXO @@ -809,7 +810,7 @@ def advance_txs(self, txs, is_unspendable): # Get the hashX hashX = script_hashX(txout.pk_script) add_hashXs(hashX) - put_utxo(tx_hash + to_le_uint32(idx), + put_utxo(tx_hash + to_le_uint32(idx)[:TXOUTIDX_LEN], hashX + tx_numb + to_le_uint64(txout.value)) tx_num += 1 @@ -856,7 +857,8 @@ def backup_txs(self, txs, is_unspendable): if txin.is_generation(): continue undo_item = undo_info[n:n + undo_entry_len] - put_utxo(txin.prev_hash + pack_le_uint32(txin.prev_idx), undo_item) + prevout = txin.prev_hash + pack_le_uint32(txin.prev_idx)[:TXOUTIDX_LEN] + put_utxo(prevout, undo_item) add_touched(undo_item[:HASHX_LEN]) n += undo_entry_len diff --git a/src/electrumx/server/db.py b/src/electrumx/server/db.py index 230d418b1..cfe1b2b84 100644 --- a/src/electrumx/server/db.py +++ b/src/electrumx/server/db.py @@ -29,7 +29,9 @@ unpack_le_uint32, unpack_be_uint32, unpack_le_uint64 ) from electrumx.server.storage import db_class, Storage -from electrumx.server.history import History, TXNUM_LEN, TXNUM_PADDING +from electrumx.server.history import ( + History, TXNUM_LEN, TXNUM_PADDING, TXOUTIDX_LEN, TXOUTIDX_PADDING, +) if TYPE_CHECKING: from electrumx.server.env import Env @@ -329,7 +331,7 @@ def flush_utxo_db(self, batch, flush_data: FlushData): for key, value in flush_data.adds.items(): # key: txid+out_idx, value: hashX+tx_num+value_sats hashX = value[:HASHX_LEN] - txout_idx = key[-4:] + txout_idx = key[-TXOUTIDX_LEN:] tx_num = value[HASHX_LEN: HASHX_LEN+TXNUM_LEN] value_sats = value[-8:] suffix = txout_idx + tx_num @@ -660,7 +662,7 @@ def read_utxos(): # Value: the UTXO value as a 64-bit unsigned integer prefix = b'u' + hashX for db_key, db_value in self.utxo_db.iterator(prefix=prefix): - txout_idx, = unpack_le_uint32(db_key[-TXNUM_LEN-4:-TXNUM_LEN]) + txout_idx, = unpack_le_uint32(db_key[-TXNUM_LEN-TXOUTIDX_LEN:-TXNUM_LEN] + TXOUTIDX_PADDING) tx_num, = unpack_le_uint64(db_key[-TXNUM_LEN:] + TXNUM_PADDING) value, = unpack_le_uint64(db_value) tx_hash, height = self.fs_tx_hash(tx_num) @@ -686,7 +688,7 @@ def lookup_hashXs(): for each prevout. ''' def lookup_hashX(tx_hash, tx_idx): - idx_packed = pack_le_uint32(tx_idx) + idx_packed = pack_le_uint32(tx_idx)[:TXOUTIDX_LEN] # Key: b'h' + compressed_tx_hash + tx_idx + tx_num # Value: hashX diff --git a/src/electrumx/server/history.py b/src/electrumx/server/history.py index 08812bd2e..f8b350962 100644 --- a/src/electrumx/server/history.py +++ b/src/electrumx/server/history.py @@ -26,6 +26,8 @@ TXNUM_LEN = 5 TXNUM_PADDING = bytes(8 - TXNUM_LEN) +TXOUTIDX_LEN = 4 +TXOUTIDX_PADDING = bytes(4 - TXOUTIDX_LEN) class History: From 773a6c1d9afd93f9482fc6323effd306b6529487 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Fri, 30 Oct 2020 23:59:59 +0100 Subject: [PATCH 06/27] db: change TXOUTIDX_LEN from 4 to 3, to save db storage size In Bitcoin consensus, a txout index is stored as a uint32_t. However, in practice, an output in a tx uses at least 10 bytes (for an OP_TRUE output), so - to exhaust a 2 byte namespace, a tx would need to have a size of at least 2 ** 16 * 10 = 655 KB, - to exhaust a 3 byte namespace, a tx would need to have a size of at least 2 ** 24 * 10 = 167 MB. --- src/electrumx/server/history.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/electrumx/server/history.py b/src/electrumx/server/history.py index f8b350962..210d0a03d 100644 --- a/src/electrumx/server/history.py +++ b/src/electrumx/server/history.py @@ -26,7 +26,7 @@ TXNUM_LEN = 5 TXNUM_PADDING = bytes(8 - TXNUM_LEN) -TXOUTIDX_LEN = 4 +TXOUTIDX_LEN = 3 TXOUTIDX_PADDING = bytes(4 - TXOUTIDX_LEN) From 95db64a693bfe34b519a8ebd85a2fdd838c92781 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Mon, 16 Nov 2020 18:58:43 +0100 Subject: [PATCH 07/27] (bugfix) db: change tx_num endianness (LE->BE) to match db comparator History.get_txnums and History.backup depend on ordering of tx_nums, so we want the lexicographical order (used by leveldb comparator) to match the numerical order. --- src/electrumx/lib/util.py | 3 +++ src/electrumx/server/block_processor.py | 10 ++++++---- src/electrumx/server/db.py | 6 +++--- src/electrumx/server/history.py | 23 +++++++++++++++++------ 4 files changed, 29 insertions(+), 13 deletions(-) diff --git a/src/electrumx/lib/util.py b/src/electrumx/lib/util.py index 11236cd0e..4038cb640 100644 --- a/src/electrumx/lib/util.py +++ b/src/electrumx/lib/util.py @@ -333,6 +333,7 @@ def is_hex_str(text: Any) -> bool: struct_le_Q = Struct('H') struct_be_I = Struct('>I') +struct_be_Q = Struct('>Q') structB = Struct('B') unpack_le_int32_from = struct_le_i.unpack_from @@ -346,6 +347,7 @@ def is_hex_str(text: Any) -> bool: unpack_le_uint32 = struct_le_I.unpack unpack_le_uint64 = struct_le_Q.unpack unpack_be_uint32 = struct_be_I.unpack +unpack_be_uint64 = struct_be_Q.unpack pack_le_int32 = struct_le_i.pack pack_le_int64 = struct_le_q.pack @@ -354,6 +356,7 @@ def is_hex_str(text: Any) -> bool: pack_le_uint64 = struct_le_Q.pack pack_be_uint16 = struct_be_H.pack pack_be_uint32 = struct_be_I.pack +pack_be_uint64 = struct_be_Q.pack pack_byte = structB.pack hex_to_bytes = bytes.fromhex diff --git a/src/electrumx/server/block_processor.py b/src/electrumx/server/block_processor.py index 1e35d8607..64df627ae 100644 --- a/src/electrumx/server/block_processor.py +++ b/src/electrumx/server/block_processor.py @@ -24,7 +24,7 @@ ) from electrumx.lib.tx import Tx from electrumx.server.db import FlushData, COMP_TXID_LEN, DB -from electrumx.server.history import TXNUM_LEN, TXNUM_PADDING, TXOUTIDX_LEN, TXOUTIDX_PADDING +from electrumx.server.history import TXNUM_LEN, TXNUM_PADDING, TXOUTIDX_LEN, TXOUTIDX_PADDING, pack_txnum, unpack_txnum if TYPE_CHECKING: from electrumx.lib.coins import Coin, Block @@ -454,12 +454,13 @@ def advance_txs( append_hashXs = hashXs_by_tx.append to_le_uint32 = pack_le_uint32 to_le_uint64 = pack_le_uint64 + _pack_txnum = pack_txnum for tx in txs: tx_hash = tx.txid hashXs = [] append_hashX = hashXs.append - tx_numb = to_le_uint64(tx_num)[:TXNUM_LEN] + tx_numb = _pack_txnum(tx_num) # Spend the inputs for txin in tx.inputs: @@ -646,7 +647,7 @@ def spend_utxo(self, tx_hash: bytes, tx_idx: int) -> bytes: tx_num_packed = hdb_key[-TXNUM_LEN:] if len(candidates) > 1: - tx_num, = unpack_le_uint64(tx_num_packed + TXNUM_PADDING) + tx_num = unpack_txnum(tx_num_packed) hash, _height = self.db.fs_tx_hash(tx_num) if hash != tx_hash: assert hash is not None # Should always be found @@ -793,6 +794,7 @@ def advance_txs(self, txs, is_unspendable): update_touched = self.touched.update to_le_uint32 = pack_le_uint32 to_le_uint64 = pack_le_uint64 + _pack_txnum = pack_txnum hashXs_by_tx = [set() for _ in txs] @@ -800,7 +802,7 @@ def advance_txs(self, txs, is_unspendable): for tx, hashXs in zip(txs, hashXs_by_tx): tx_hash = tx.txid add_hashXs = hashXs.add - tx_numb = to_le_uint64(tx_num)[:TXNUM_LEN] + tx_numb = _pack_txnum(tx_num) for idx, txout in enumerate(tx.outputs): # Ignore unspendable outputs diff --git a/src/electrumx/server/db.py b/src/electrumx/server/db.py index cfe1b2b84..539d9aae3 100644 --- a/src/electrumx/server/db.py +++ b/src/electrumx/server/db.py @@ -30,7 +30,7 @@ ) from electrumx.server.storage import db_class, Storage from electrumx.server.history import ( - History, TXNUM_LEN, TXNUM_PADDING, TXOUTIDX_LEN, TXOUTIDX_PADDING, + History, TXNUM_LEN, TXNUM_PADDING, TXOUTIDX_LEN, TXOUTIDX_PADDING, pack_txnum, unpack_txnum, ) if TYPE_CHECKING: @@ -663,7 +663,7 @@ def read_utxos(): prefix = b'u' + hashX for db_key, db_value in self.utxo_db.iterator(prefix=prefix): txout_idx, = unpack_le_uint32(db_key[-TXNUM_LEN-TXOUTIDX_LEN:-TXNUM_LEN] + TXOUTIDX_PADDING) - tx_num, = unpack_le_uint64(db_key[-TXNUM_LEN:] + TXNUM_PADDING) + tx_num = unpack_txnum(db_key[-TXNUM_LEN:]) value, = unpack_le_uint64(db_value) tx_hash, height = self.fs_tx_hash(tx_num) utxos_append(UTXO(tx_num, txout_idx, tx_hash, height, value)) @@ -697,7 +697,7 @@ def lookup_hashX(tx_hash, tx_idx): # Find which entry, if any, the TX_HASH matches. for db_key, hashX in self.utxo_db.iterator(prefix=prefix): tx_num_packed = db_key[-TXNUM_LEN:] - tx_num, = unpack_le_uint64(tx_num_packed + TXNUM_PADDING) + tx_num = unpack_txnum(tx_num_packed) hash, _height = self.fs_tx_hash(tx_num) if hash == tx_hash: return hashX, idx_packed + tx_num_packed diff --git a/src/electrumx/server/history.py b/src/electrumx/server/history.py index 210d0a03d..db43f48d0 100644 --- a/src/electrumx/server/history.py +++ b/src/electrumx/server/history.py @@ -17,8 +17,10 @@ import electrumx.lib.util as util from electrumx.lib.hash import HASHX_LEN, hash_to_hex_str -from electrumx.lib.util import (pack_be_uint16, pack_le_uint64, - unpack_be_uint16_from, unpack_le_uint64) +from electrumx.lib.util import ( + pack_le_uint32, unpack_le_uint32, + pack_be_uint64, unpack_be_uint64, +) if TYPE_CHECKING: from electrumx.server.storage import Storage @@ -30,6 +32,14 @@ TXOUTIDX_PADDING = bytes(4 - TXOUTIDX_LEN) +def unpack_txnum(tx_numb: bytes) -> int: + return unpack_be_uint64(TXNUM_PADDING + tx_numb)[0] + + +def pack_txnum(tx_num: int) -> bytes: + return pack_be_uint64(tx_num)[-TXNUM_LEN:] + + class History: DB_VERSIONS = (3, ) @@ -102,7 +112,7 @@ def clear_excess(self, utxo_db_tx_count: int) -> None: keys = [] for db_key, db_val in self.db.iterator(prefix=b'H'): tx_numb = db_key[-TXNUM_LEN:] - tx_num, = unpack_le_uint64(tx_numb + TXNUM_PADDING) + tx_num = unpack_txnum(tx_numb) if tx_num >= utxo_db_tx_count: keys.append(db_key) @@ -131,7 +141,7 @@ def add_unflushed(self, hashXs_by_tx, first_tx_num): count = 0 tx_num = None for tx_num, hashXs in enumerate(hashXs_by_tx, start=first_tx_num): - tx_numb = pack_le_uint64(tx_num)[:TXNUM_LEN] + tx_numb = pack_txnum(tx_num) hashXs = set(hashXs) for hashX in hashXs: unflushed[hashX] += tx_numb @@ -178,11 +188,12 @@ def backup(self, hashXs, tx_count): prefix = b'H' + hashX for db_key, db_val in self.db.iterator(prefix=prefix, reverse=True): tx_numb = db_key[-TXNUM_LEN:] - tx_num, = unpack_le_uint64(tx_numb + TXNUM_PADDING) + tx_num = unpack_txnum(tx_numb) if tx_num >= tx_count: nremoves += 1 deletes.append(db_key) else: + # note: we can break now, due to 'reverse=True' and txnums being big endian break for key in deletes: batch.delete(key) @@ -203,6 +214,6 @@ def get_txnums(self, hashX, limit=1000): tx_numb = db_key[-TXNUM_LEN:] if limit == 0: return - tx_num, = unpack_le_uint64(tx_numb + TXNUM_PADDING) + tx_num = unpack_txnum(tx_numb) yield tx_num limit -= 1 From d460f3b86f6025c95fb6ff7916bdf0450d1bae34 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Thu, 7 May 2026 09:57:49 +0000 Subject: [PATCH 08/27] db: add (block_hash -> block_height) in-memory map For 1 million blocks, in a python dict, I guess this might take around 100 MB of RAM. // - block_hash is 32 bytes, block_height is a varsize int // - dict size seems to increase by 93 bytes for every new item // - however there is considerable additional overhead, I guess amortized over many adds, // for the hashmap. Looks like it takes around ~145 bytes per item overall. --- src/electrumx/server/block_processor.py | 5 ++++ src/electrumx/server/db.py | 31 ++++++++++++++++++++++--- 2 files changed, 33 insertions(+), 3 deletions(-) diff --git a/src/electrumx/server/block_processor.py b/src/electrumx/server/block_processor.py index 64df627ae..663c4fba7 100644 --- a/src/electrumx/server/block_processor.py +++ b/src/electrumx/server/block_processor.py @@ -369,6 +369,7 @@ def flush_data(self): ) async def flush(self, flush_utxos): + # side-effect: fields of FlushData will be cleared (passed by reference) def flush(): self.db.flush_dbs(self.flush_data(), flush_utxos, self.estimate_txs_remaining) @@ -418,12 +419,15 @@ def advance_blocks(self, blocks: Sequence['Block']): min_height = self.db.min_undo_height(self.daemon.cached_height()) height = self.height genesis_activation = self.coin.GENESIS_ACTIVATION + coin = self.coin for block in blocks: height += 1 + header_hash = coin.header_hash(block.header) is_unspendable = (is_unspendable_genesis if height >= genesis_activation else is_unspendable_legacy) undo_info = self.advance_txs(block.transactions, is_unspendable) + self.db.bhash_to_bheight[header_hash] = height if height >= min_height: self.undo_infos.append((undo_info, height)) self.db.write_raw_block(block.raw, height) @@ -517,6 +521,7 @@ def backup_blocks(self, raw_blocks: Sequence[bytes]): is_unspendable = (is_unspendable_genesis if self.height >= genesis_activation else is_unspendable_legacy) self.backup_txs(block.transactions, is_unspendable) + assert self.height == self.db.bhash_to_bheight.pop(header_hash) self.height -= 1 self.db.tx_counts.pop() diff --git a/src/electrumx/server/db.py b/src/electrumx/server/db.py index 539d9aae3..bd86424b5 100644 --- a/src/electrumx/server/db.py +++ b/src/electrumx/server/db.py @@ -22,7 +22,7 @@ from aiorpcx import run_in_thread, sleep import electrumx.lib.util as util -from electrumx.lib.hash import hash_to_hex_str, HASHX_LEN +from electrumx.lib.hash import hash_to_hex_str, HASHX_LEN, hex_str_to_hash from electrumx.lib.merkle import Merkle, MerkleCache from electrumx.lib.util import ( formatted_time, pack_be_uint16, pack_be_uint32, pack_le_uint64, pack_le_uint32, @@ -128,7 +128,7 @@ def __init__(self, env: 'Env'): # on-disk: raw block headers in chain order self.headers_file = util.LogicalFile('meta/headers', 2, 16000000) # on-disk: cumulative number of txs at the end of height N - self.tx_counts = None # type: Optional[array] + self.tx_counts = None # type: Optional[array] # in-memory self.tx_counts_file = util.LogicalFile('meta/txcounts', 2, 2000000) # on-disk: 32 byte txids in chain order, allows (tx_num -> txid) map self.hashes_file = util.LogicalFile('meta/hashes', 4, 16000000) @@ -136,6 +136,9 @@ def __init__(self, env: 'Env'): self.headers_offsets_file = util.LogicalFile( 'meta/headers_offsets', 2, 16000000) + # in-memory: (block_hash -> block_height) map + self.bhash_to_bheight = None # type: Optional[dict[bytes, int]] + async def _read_tx_counts(self): if self.tx_counts is not None: return @@ -176,8 +179,11 @@ async def _open_dbs(self, *, for_sync: bool): ) self.clear_excess_undo_info() - # Read TX counts (requires meta directory) + # Now prepare in-memory structures. + # - Read TX counts (requires meta directory) await self._read_tx_counts() + # - (block_hash -> block_height) map + await self._prep_bhash_to_bheight_map() async def open_for_sync(self): '''Open the databases to sync to the daemon. @@ -212,6 +218,25 @@ async def populate_header_merkle_cache(self): async def header_branch_and_root(self, length, height): return await self.header_mc.branch_and_root(length, height) + # -- (block_hash -> block_height) map + async def _prep_bhash_to_bheight_map(self) -> None: + if self.bhash_to_bheight is not None: + return + self.bhash_to_bheight = {} + count = self.db_height + 1 + block_hashes = await self.fs_block_hashes(0, count) + if len(block_hashes) != count: + raise Exception( + f"failed to prep bhash_to_bheight. " + f"wanted {count} bhashes, only got {len(block_hashes)}") + for bheight, bhash in enumerate(block_hashes): + self.bhash_to_bheight[bhash] = bheight + # note: for new blocks, the block_processor will keep the map up-to-date + + def get_blockheight_from_blockhash(self, block_hash: str) -> Optional[int]: + bhash = hex_str_to_hash(block_hash) + return self.bhash_to_bheight.get(bhash, None) + # Flushing def assert_flushed(self, flush_data): '''Asserts state is fully flushed.''' From 934b97440f904555822600e344295d74ac5c1314 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Tue, 3 Nov 2020 19:32:38 +0100 Subject: [PATCH 09/27] session: implement "blockchain.outpoint.subscribe" RPC squashed with: - "blockchain.outpoint.subscribe" RPC: add optional "spk_hint" argument - "blockchain.outpoint.subscribe" RPC: implement notifications - "blockchain.outpoint.subscribe" RPC: distinguish heights "-1" and "0" Similar to scripthash statuses, the height of an unconfirmed tx is: - `-1` if it has any unconfirmed parents, - `0` otherwise. - session: implement "blockchain.outpoint.get_status" RPC - "blockchain.outpoint.subscribe": use bitcoind 31 gettxspendingprevout --- src/electrumx/lib/tx.py | 7 + src/electrumx/server/block_processor.py | 58 +++-- src/electrumx/server/controller.py | 85 +++++-- src/electrumx/server/daemon.py | 9 +- src/electrumx/server/db.py | 12 +- src/electrumx/server/mempool.py | 173 +++++++++++--- src/electrumx/server/session.py | 306 +++++++++++++++++++++--- tests/server/test_mempool.py | 18 +- tests/server/test_notifications.py | 36 +-- 9 files changed, 553 insertions(+), 151 deletions(-) diff --git a/src/electrumx/lib/tx.py b/src/electrumx/lib/tx.py index 7f5605508..d0ff1758d 100644 --- a/src/electrumx/lib/tx.py +++ b/src/electrumx/lib/tx.py @@ -113,6 +113,13 @@ def serialize(self): )) +@dataclass(kw_only=True, slots=True) +class TXOSpendStatus: + prev_height: Optional[int] # block height TXO is mined at. None if the outpoint never existed + spender_txhash: bytes = None + spender_height: int = None + + class Deserializer: '''Deserializes blocks into transactions. diff --git a/src/electrumx/server/block_processor.py b/src/electrumx/server/block_processor.py index 663c4fba7..d49df6ac7 100644 --- a/src/electrumx/server/block_processor.py +++ b/src/electrumx/server/block_processor.py @@ -11,7 +11,7 @@ import asyncio import time -from typing import Sequence, Tuple, List, Callable, Optional, TYPE_CHECKING, Type +from typing import Sequence, Tuple, List, Callable, Optional, TYPE_CHECKING, Type, Set from aiorpcx import run_in_thread, CancelledError @@ -186,7 +186,8 @@ def __init__(self, env: 'Env', db: DB, daemon: Daemon, notifications: 'Notificat # Meta self.next_cache_check = 0 - self.touched = set() + self.touched_hashxs = set() # type: Set[bytes] + self.touched_outpoints = set() # type: Set[Tuple[bytes, int]] self.reorg_count = 0 self.height = -1 self.tip = None # type: Optional[bytes] @@ -244,8 +245,13 @@ async def check_and_advance_blocks(self, raw_blocks: Sequence[bytes]) -> None: self.logger.info(f'processed {len(blocks):,d} block{s} size {blocks_size:.2f} MB ' f'in {time.monotonic() - start:.1f}s') if self._caught_up_event.is_set(): - await self.notifications.on_block(self.touched, self.height) - self.touched = set() + await self.notifications.on_block( + touched_hashxs=self.touched_hashxs, + touched_outpoints=self.touched_outpoints, + height=self.height, + ) + self.touched_hashxs = set() + self.touched_outpoints = set() elif hprevs[0] != chain[0]: await self.reorg_chain() else: @@ -279,10 +285,10 @@ async def get_raw_blocks(last_height, hex_hashes) -> Sequence[bytes]: return await self.daemon.raw_blocks(hex_hashes) def flush_backup(): - # self.touched can include other addresses which is + # self.touched_hashxs can include other addresses which is # harmless, but remove None. - self.touched.discard(None) - self.db.flush_backup(self.flush_data(), self.touched) + self.touched_hashxs.discard(None) + self.db.flush_backup(self.flush_data(), self.touched_hashxs) _start, last, hashes = await self.reorg_hashes(count) # Reverse and convert to hex strings. @@ -453,7 +459,8 @@ def advance_txs( put_utxo = self.utxo_cache.__setitem__ spend_utxo = self.spend_utxo undo_info_append = undo_info.append - update_touched = self.touched.update + update_touched_hashxs = self.touched_hashxs.update + add_touched_outpoint = self.touched_outpoints.add hashXs_by_tx = [] append_hashXs = hashXs_by_tx.append to_le_uint32 = pack_le_uint32 @@ -473,6 +480,8 @@ def advance_txs( cache_value = spend_utxo(txin.prev_hash, txin.prev_idx) undo_info_append(cache_value) append_hashX(cache_value[:HASHX_LEN]) + prevout_tuple = (txin.prev_hash, txin.prev_idx) + add_touched_outpoint(prevout_tuple) # Add the new UTXOs for idx, txout in enumerate(tx.outputs): @@ -485,9 +494,10 @@ def advance_txs( append_hashX(hashX) put_utxo(tx_hash + to_le_uint32(idx)[:TXOUTIDX_LEN], hashX + tx_numb + to_le_uint64(txout.value)) + add_touched_outpoint((tx_hash, idx)) append_hashXs(hashXs) - update_touched(hashXs) + update_touched_hashxs(hashXs) tx_num += 1 self.db.history.add_unflushed(hashXs_by_tx, self.tx_count) @@ -543,7 +553,8 @@ def backup_txs( # Use local vars for speed in the loops put_utxo = self.utxo_cache.__setitem__ spend_utxo = self.spend_utxo - touched = self.touched + add_touched_hashx = self.touched_hashxs.add + add_touched_outpoint = self.touched_outpoints.add undo_entry_len = HASHX_LEN + TXNUM_LEN + 8 for tx in reversed(txs): @@ -557,7 +568,8 @@ def backup_txs( # Get the hashX cache_value = spend_utxo(tx_hash, idx) hashX = cache_value[:HASHX_LEN] - touched.add(hashX) + add_touched_hashx(hashX) + add_touched_outpoint((tx_hash, idx)) # Restore the inputs for txin in reversed(tx.inputs): @@ -568,7 +580,8 @@ def backup_txs( prevout = txin.prev_hash + pack_le_uint32(txin.prev_idx)[:TXOUTIDX_LEN] put_utxo(prevout, undo_item) hashX = undo_item[:HASHX_LEN] - touched.add(hashX) + add_touched_hashx(hashX) + add_touched_outpoint((txin.prev_hash, txin.prev_idx)) assert n == 0 self.tx_count -= len(txs) @@ -760,7 +773,7 @@ def advance_txs(self, txs, is_unspendable): tx_num = self.tx_count - len(txs) script_name_hashX = self.coin.name_hashX_from_script - update_touched = self.touched.update + update_touched_hashxs = self.touched_hashxs.update hashXs_by_tx = [] append_hashXs = hashXs_by_tx.append @@ -776,7 +789,7 @@ def advance_txs(self, txs, is_unspendable): append_hashX(hashX) append_hashXs(hashXs) - update_touched(hashXs) + update_touched_hashxs(hashXs) tx_num += 1 self.db.history.add_unflushed(hashXs_by_tx, self.tx_count - len(txs)) @@ -796,7 +809,8 @@ def advance_txs(self, txs, is_unspendable): put_utxo = self.utxo_cache.__setitem__ spend_utxo = self.spend_utxo undo_info_append = undo_info.append - update_touched = self.touched.update + update_touched_hashxs = self.touched_hashxs.update + add_touched_outpoint = self.touched_outpoints.add to_le_uint32 = pack_le_uint32 to_le_uint64 = pack_le_uint64 _pack_txnum = pack_txnum @@ -819,6 +833,7 @@ def advance_txs(self, txs, is_unspendable): add_hashXs(hashX) put_utxo(tx_hash + to_le_uint32(idx)[:TXOUTIDX_LEN], hashX + tx_numb + to_le_uint64(txout.value)) + add_touched_outpoint((tx_hash, idx)) tx_num += 1 # Spend the inputs @@ -831,10 +846,12 @@ def advance_txs(self, txs, is_unspendable): cache_value = spend_utxo(txin.prev_hash, txin.prev_idx) undo_info_append(cache_value) add_hashXs(cache_value[:HASHX_LEN]) + prevout_tuple = (txin.prev_hash, txin.prev_idx) + add_touched_outpoint(prevout_tuple) # Update touched set for notifications for hashXs in hashXs_by_tx: - update_touched(hashXs) + update_touched_hashxs(hashXs) self.db.history.add_unflushed(hashXs_by_tx, self.tx_count) @@ -853,7 +870,8 @@ def backup_txs(self, txs, is_unspendable): # Use local vars for speed in the loops put_utxo = self.utxo_cache.__setitem__ spend_utxo = self.spend_utxo - add_touched = self.touched.add + add_touched_hashx = self.touched_hashxs.add + add_touched_outpoint = self.touched_outpoints.add undo_entry_len = HASHX_LEN + TXNUM_LEN + 8 # Restore coins that had been spent @@ -866,7 +884,8 @@ def backup_txs(self, txs, is_unspendable): undo_item = undo_info[n:n + undo_entry_len] prevout = txin.prev_hash + pack_le_uint32(txin.prev_idx)[:TXOUTIDX_LEN] put_utxo(prevout, undo_item) - add_touched(undo_item[:HASHX_LEN]) + add_touched_hashx(undo_item[:HASHX_LEN]) + add_touched_outpoint((txin.prev_hash, txin.prev_idx)) n += undo_entry_len assert n == len(undo_info) @@ -883,6 +902,7 @@ def backup_txs(self, txs, is_unspendable): # Get the hashX cache_value = spend_utxo(tx_hash, idx) hashX = cache_value[:HASHX_LEN] - add_touched(hashX) + add_touched_hashx(hashX) + add_touched_outpoint((tx_hash, idx)) self.tx_count -= len(txs) diff --git a/src/electrumx/server/controller.py b/src/electrumx/server/controller.py index 60b8046b9..36408edc0 100644 --- a/src/electrumx/server/controller.py +++ b/src/electrumx/server/controller.py @@ -6,6 +6,7 @@ # and warranty status of this software. from asyncio import Event +from typing import Set, Dict, Tuple from aiorpcx import _version as aiorpcx_version @@ -31,43 +32,83 @@ class Notifications: # notifications appropriately. def __init__(self): - self._touched_mp = {} - self._touched_bp = {} + self._touched_hashxs_mp = {} # type: Dict[int, Set[bytes]] + self._touched_hashxs_bp = {} # type: Dict[int, Set[bytes]] + self._touched_outpoints_mp = {} # type: Dict[int, Set[Tuple[bytes, int]]] + self._touched_outpoints_bp = {} # type: Dict[int, Set[Tuple[bytes, int]]] self._highest_block = -1 async def _maybe_notify(self): - tmp, tbp = self._touched_mp, self._touched_bp - common = set(tmp).intersection(tbp) - if common: - height = max(common) - elif tmp and max(tmp) == self._highest_block: + th_mp, th_bp = self._touched_hashxs_mp, self._touched_hashxs_bp + # figure out block height + common_heights = set(th_mp).intersection(th_bp) + if common_heights: + height = max(common_heights) + elif th_mp and max(th_mp) == self._highest_block: height = self._highest_block else: # Either we are processing a block and waiting for it to # come in, or we have not yet had a mempool update for the # new block height return - touched = tmp.pop(height) - for old in [h for h in tmp if h <= height]: - del tmp[old] - for old in [h for h in tbp if h <= height]: - touched.update(tbp.pop(old)) - await self.notify(height, touched) - - async def notify(self, height, touched): + # hashXs + touched_hashxs = th_mp.pop(height) + for old in [h for h in th_mp if h <= height]: + del th_mp[old] + for old in [h for h in th_bp if h <= height]: + touched_hashxs.update(th_bp.pop(old)) + # outpoints + to_mp, to_bp = self._touched_outpoints_mp, self._touched_outpoints_bp + touched_outpoints = to_mp.pop(height) + for old in [h for h in to_mp if h <= height]: + del to_mp[old] + for old in [h for h in to_bp if h <= height]: + touched_outpoints.update(to_bp.pop(old)) + + await self.notify( + height=height, + touched_hashxs=touched_hashxs, + touched_outpoints=touched_outpoints, + ) + + async def notify( + self, + *, + touched_hashxs: Set[bytes], + touched_outpoints: Set[Tuple[bytes, int]], + height: int, + ): pass - async def start(self, height, notify_func): + async def start(self, height: int, notify_func): self._highest_block = height self.notify = notify_func - await self.notify(height, set()) - - async def on_mempool(self, touched, height): - self._touched_mp[height] = touched + await self.notify( + height=height, + touched_hashxs=set(), + touched_outpoints=set(), + ) + + async def on_mempool( + self, + *, + touched_hashxs: Set[bytes], + touched_outpoints: Set[Tuple[bytes, int]], + height: int, + ): + self._touched_hashxs_mp[height] = touched_hashxs + self._touched_outpoints_mp[height] = touched_outpoints await self._maybe_notify() - async def on_block(self, touched, height): - self._touched_bp[height] = touched + async def on_block( + self, + *, + touched_hashxs: Set[bytes], + touched_outpoints: Set[Tuple[bytes, int]], + height: int, + ): + self._touched_hashxs_bp[height] = touched_hashxs + self._touched_outpoints_bp[height] = touched_outpoints self._highest_block = height await self._maybe_notify() diff --git a/src/electrumx/server/daemon.py b/src/electrumx/server/daemon.py index c76ae91ef..f8b806628 100644 --- a/src/electrumx/server/daemon.py +++ b/src/electrumx/server/daemon.py @@ -13,7 +13,7 @@ import time from calendar import timegm from struct import pack -from typing import TYPE_CHECKING, Type, Sequence +from typing import TYPE_CHECKING, Type, Sequence, Any import aiohttp from aiorpcx import JSONRPC @@ -348,6 +348,13 @@ async def getrawtransactions(self, hex_hashes, replace_errs=True): # Convert hex strings to bytes return [hex_to_bytes(tx) if tx else None for tx in txs] + async def gettxspendingprevout(self, prev_txhash: str, txout_idx: int) -> dict[str, Any]: + """Query the daemon to find (if any) the spender of given outpoint.""" + outpoints = [{"txid": prev_txhash, "vout": txout_idx}, ] + options = {"mempool_only": False} + tx_items = await self._send_single('gettxspendingprevout', (outpoints, options)) + return tx_items[0] + async def broadcast_transaction(self, raw_tx): '''Broadcast a transaction to the network.''' return await self._send_single('sendrawtransaction', (raw_tx, )) diff --git a/src/electrumx/server/db.py b/src/electrumx/server/db.py index bd86424b5..0b4b8be7b 100644 --- a/src/electrumx/server/db.py +++ b/src/electrumx/server/db.py @@ -16,7 +16,7 @@ from bisect import bisect_right from dataclasses import dataclass from glob import glob -from typing import Dict, List, Sequence, Tuple, Optional, TYPE_CHECKING +from typing import Dict, List, Sequence, Tuple, Optional, TYPE_CHECKING, Union import attr from aiorpcx import run_in_thread, sleep @@ -28,6 +28,7 @@ formatted_time, pack_be_uint16, pack_be_uint32, pack_le_uint64, pack_le_uint32, unpack_le_uint32, unpack_be_uint32, unpack_le_uint64 ) +from electrumx.lib.tx import TXOSpendStatus from electrumx.server.storage import db_class, Storage from electrumx.server.history import ( History, TXNUM_LEN, TXNUM_PADDING, TXOUTIDX_LEN, TXOUTIDX_PADDING, pack_txnum, unpack_txnum, @@ -389,7 +390,7 @@ def flush_state(self, batch): self.last_flush_tx_count = self.fs_tx_count self.write_utxo_state(batch) - def flush_backup(self, flush_data, touched): + def flush_backup(self, flush_data: FlushData, touched_hashxs): '''Like flush_dbs() but when backing up. All UTXOs are flushed.''' assert not flush_data.headers assert not flush_data.block_tx_hashes @@ -400,7 +401,10 @@ def flush_backup(self, flush_data, touched): tx_delta = flush_data.tx_count - self.last_flush_tx_count self.backup_fs(flush_data.height, flush_data.tx_count) - self.history.backup(touched, flush_data.tx_count) + self.history.backup( + hashXs=touched_hashxs, + tx_count=flush_data.tx_count, + ) with self.utxo_db.write_batch() as batch: self.flush_utxo_db(batch, flush_data) # Flush state last as it reads the wall time. @@ -471,7 +475,7 @@ def read_headers(): return await run_in_thread(read_headers) - def fs_tx_hash(self, tx_num): + def fs_tx_hash(self, tx_num: int) -> Tuple[Optional[bytes], int]: '''Return a pair (tx_hash, tx_height) for the given tx number. If the tx_height is not on disk, returns (None, tx_height).''' diff --git a/src/electrumx/server/mempool.py b/src/electrumx/server/mempool.py index 533233c1e..280531be9 100644 --- a/src/electrumx/server/mempool.py +++ b/src/electrumx/server/mempool.py @@ -21,6 +21,7 @@ from electrumx.lib.hash import hash_to_hex_str, hex_str_to_hash from electrumx.lib.tx import SkipTxDeserialize from electrumx.lib.util import class_logger, chunks, OldTaskGroup +from electrumx.lib.tx import TXOSpendStatus from electrumx.server.db import UTXO if TYPE_CHECKING: @@ -53,17 +54,17 @@ class MemPoolAPI(ABC): and used by it to query DB and blockchain state.''' @abstractmethod - async def height(self): + async def height(self) -> int: '''Query bitcoind for its height.''' @abstractmethod - def cached_height(self): + def cached_height(self) -> Optional[int]: '''Return the height of bitcoind the last time it was queried, for any reason, without actually querying it. ''' @abstractmethod - def db_height(self): + def db_height(self) -> int: '''Return the height flushed to the on-disk DB.''' @abstractmethod @@ -80,17 +81,25 @@ async def raw_transactions(self, hex_hashes): @abstractmethod async def lookup_utxos(self, prevouts): - '''Return a list of (hashX, value) pairs each prevout if unspent, - otherwise return None if spent or not found. + '''Return a list of (hashX, value) pairs, one for each prevout if unspent, + otherwise return None if spent or not found (for the given prevout). - prevouts - an iterable of (hash, index) pairs + prevouts - an iterable of (tx_hash, txout_idx) pairs ''' @abstractmethod - async def on_mempool(self, touched, height): - '''Called each time the mempool is synchronized. touched is a set of - hashXs touched since the previous call. height is the - daemon's height at the time the mempool was obtained.''' + async def on_mempool( + self, + *, + touched_hashxs: Set[bytes], + touched_outpoints: Set[Tuple[bytes, int]], + height: int, + ): + '''Called each time the mempool is synchronized. touched_hashxs and + touched_outpoints are sets of hashXs and tx outpoints touched since + the previous call. height is the daemon's height at the time the + mempool was obtained. + ''' class MemPool: @@ -119,8 +128,9 @@ def __init__( self.coin = coin self.api = api self.logger = class_logger(__name__, self.__class__.__name__) - self.txs = {} # type: Dict[bytes, MemPoolTx] - self.hashXs = defaultdict(set) # None can be a key + self.txs = {} # type: Dict[bytes, MemPoolTx] # txid->tx + self.hashXs = defaultdict(set) # type: Dict[Optional[bytes], Set[bytes]] # hashX->txids + self.txo_to_spender = {} # type: Dict[Tuple[bytes, int], bytes] # prevout->txid self.cached_compact_histogram = [] self.refresh_secs = refresh_secs self.log_status_secs = log_status_secs @@ -137,8 +147,9 @@ async def _logging(self, synchronized_event): self.logger.info(f'synced in {elapsed:.2f}s') while True: mempool_size = sum(tx.size for tx in self.txs.values()) / 1_000_000 - self.logger.info(f'{len(self.txs):,d} txs {mempool_size:.2f} MB ' - f'touching {len(self.hashXs):,d} addresses') + self.logger.info(f'{len(self.txs):,d} txs {mempool_size:.2f} MB, ' + f'touching {len(self.hashXs):,d} addresses. ' + f'{len(self.txo_to_spender):,d} spends.') await sleep(self.log_status_secs) await synchronized_event.wait() @@ -205,7 +216,15 @@ def _compress_histogram( prev_fee_rate = fee_rate return compact - def _accept_transactions(self, tx_map: Dict[bytes, MemPoolTx], utxo_map, touched): + def _accept_transactions( + self, + *, + tx_map: Dict[bytes, MemPoolTx], # txid->tx + utxo_map: Dict[Tuple[bytes, int], Tuple[bytes, int]], # prevout->(hashX,value_in_sats) + touched_hashxs: Set[bytes], # set of hashXs + touched_outpoints: Set[Tuple[bytes, int]], # set of outpoints + ) -> Tuple[Dict[bytes, MemPoolTx], + Dict[Tuple[bytes, int], Tuple[bytes, int]]]: '''Accept transactions in tx_map to the mempool if all their inputs can be found in the existing mempool or a utxo_map from the DB. @@ -214,11 +233,12 @@ def _accept_transactions(self, tx_map: Dict[bytes, MemPoolTx], utxo_map, touched ''' hashXs = self.hashXs txs = self.txs + txo_to_spender = self.txo_to_spender deferred = {} unspent = set(utxo_map) # Try to find all prevouts so we can accept the TX - for hash, tx in tx_map.items(): + for tx_hash, tx in tx_map.items(): in_pairs = [] try: for prevout in tx.prevouts: @@ -229,7 +249,7 @@ def _accept_transactions(self, tx_map: Dict[bytes, MemPoolTx], utxo_map, touched utxo = txs[prev_hash].out_pairs[prev_index] in_pairs.append(utxo) except KeyError: - deferred[hash] = tx + deferred[tx_hash] = tx continue # Spend the prevouts @@ -241,19 +261,25 @@ def _accept_transactions(self, tx_map: Dict[bytes, MemPoolTx], utxo_map, touched # because some in_parts would be missing tx.fee = max(0, (sum(v for _, v in tx.in_pairs) - sum(v for _, v in tx.out_pairs))) - txs[hash] = tx + txs[tx_hash] = tx for hashX, _value in itertools.chain(tx.in_pairs, tx.out_pairs): - touched.add(hashX) - hashXs[hashX].add(hash) + touched_hashxs.add(hashX) + hashXs[hashX].add(tx_hash) + for prevout in tx.prevouts: + txo_to_spender[prevout] = tx_hash + touched_outpoints.add(prevout) + for out_idx, out_pair in enumerate(tx.out_pairs): + touched_outpoints.add((tx_hash, out_idx)) return deferred, {prevout: utxo_map[prevout] for prevout in unspent} async def _refresh_hashes(self, synchronized_event): '''Refresh our view of the daemon's mempool.''' - # Touched accumulates between calls to on_mempool and each + # touched_* accumulates between calls to on_mempool and each # call transfers ownership - touched = set() + touched_hashxs = set() + touched_outpoints = set() while True: height = self.api.cached_height() hex_hashes = await self.api.mempool_hashes() @@ -262,7 +288,12 @@ async def _refresh_hashes(self, synchronized_event): hashes = {hex_str_to_hash(hh) for hh in hex_hashes} try: async with self.lock: - await self._process_mempool(hashes, touched, height) + await self._process_mempool( + all_hashes=hashes, + touched_hashxs=touched_hashxs, + touched_outpoints=touched_outpoints, + mempool_height=height, + ) except DBSyncError: # The UTXO DB is not at the same height as the # mempool; wait and try again @@ -270,14 +301,27 @@ async def _refresh_hashes(self, synchronized_event): else: synchronized_event.set() synchronized_event.clear() - await self.api.on_mempool(touched, height) - touched = set() + await self.api.on_mempool( + touched_hashxs=touched_hashxs, + touched_outpoints=touched_outpoints, + height=height, + ) + touched_hashxs = set() + touched_outpoints = set() await sleep(self.refresh_secs) - async def _process_mempool(self, all_hashes: Set[bytes], touched, mempool_height): + async def _process_mempool( + self, + *, + all_hashes: Set[bytes], # set of txids + touched_hashxs: Set[bytes], # set of hashXs + touched_outpoints: Set[Tuple[bytes, int]], # set of outpoints + mempool_height: int, + ) -> None: # Re-sync with the new set of hashes txs = self.txs hashXs = self.hashXs + txo_to_spender = self.txo_to_spender if mempool_height != self.api.db_height(): raise DBSyncError @@ -285,20 +329,32 @@ async def _process_mempool(self, all_hashes: Set[bytes], touched, mempool_height # First handle txs that have disappeared for tx_hash in (set(txs) - all_hashes): tx = txs.pop(tx_hash) + # hashXs tx_hashXs = {hashX for hashX, value in tx.in_pairs} tx_hashXs.update(hashX for hashX, value in tx.out_pairs) for hashX in tx_hashXs: hashXs[hashX].remove(tx_hash) if not hashXs[hashX]: del hashXs[hashX] - touched |= tx_hashXs + touched_hashxs |= tx_hashXs + # outpoints + for prevout in tx.prevouts: + del txo_to_spender[prevout] + touched_outpoints.add(prevout) + for out_idx, out_pair in enumerate(tx.out_pairs): + touched_outpoints.add((tx_hash, out_idx)) # Process new transactions new_hashes = list(all_hashes.difference(txs)) if new_hashes: group = OldTaskGroup() for hashes in chunks(new_hashes, 200): - coro = self._fetch_and_accept(hashes, all_hashes, touched) + coro = self._fetch_and_accept( + hashes=hashes, + all_hashes=all_hashes, + touched_hashxs=touched_hashxs, + touched_outpoints=touched_outpoints, + ) await group.spawn(coro) if mempool_height != self.api.db_height(): raise DBSyncError @@ -314,14 +370,23 @@ async def _process_mempool(self, all_hashes: Set[bytes], touched, mempool_height # FIXME: this is not particularly efficient while tx_map and len(tx_map) != prior_count: prior_count = len(tx_map) - tx_map, utxo_map = self._accept_transactions(tx_map, utxo_map, - touched) + tx_map, utxo_map = self._accept_transactions( + tx_map=tx_map, + utxo_map=utxo_map, + touched_hashxs=touched_hashxs, + touched_outpoints=touched_outpoints, + ) if tx_map: self.logger.error(f'{len(tx_map)} txs dropped') - return touched - - async def _fetch_and_accept(self, hashes: Sequence[bytes], all_hashes: Set[bytes], touched): + async def _fetch_and_accept( + self, + *, + hashes: Set[bytes], # set of txids + all_hashes: Set[bytes], # set of txids + touched_hashxs: Set[bytes], # set of hashXs + touched_outpoints: Set[Tuple[bytes, int]], # set of outpoints + ): '''Fetch a list of mempool transactions.''' hex_hashes_iter = (hash_to_hex_str(hash) for hash in hashes) raw_txs = await self.api.raw_transactions(hex_hashes_iter) @@ -372,7 +437,12 @@ def deserialize_txs() -> Dict[bytes, MemPoolTx]: utxos = await self.api.lookup_utxos(prevouts) utxo_map = {prevout: utxo for prevout, utxo in zip(prevouts, utxos)} - return self._accept_transactions(tx_map, utxo_map, touched) + return self._accept_transactions( + tx_map=tx_map, + utxo_map=utxo_map, + touched_hashxs=touched_hashxs, + touched_outpoints=touched_outpoints, + ) # # External interface @@ -441,3 +511,38 @@ async def unordered_UTXOs(self, hashX): if hX == hashX: utxos.append(UTXO(-1, pos, tx_hash, 0, value)) return utxos + + async def spender_for_txo(self, prev_txhash: bytes, txout_idx: int) -> 'TXOSpendStatus': + '''For an outpoint, returns its spend-status. + This only considers the mempool, not the DB/blockchain, so e.g. mined + txs are not distinguished from txs that never existed. + ''' + # look up funding tx + prev_tx = self.txs.get(prev_txhash, None) + if prev_tx is None: + # funding tx already mined or never existed + prev_height = None + else: + if len(prev_tx.out_pairs) <= txout_idx: + # output idx out of bounds...? + return TXOSpendStatus(prev_height=None) + prev_has_ui = any(hash in self.txs for hash, idx in prev_tx.prevouts) + prev_height = -prev_has_ui + prevout = (prev_txhash, txout_idx) + # look up spending tx + spender_txhash = self.txo_to_spender.get(prevout, None) + if spender_txhash is None: + return TXOSpendStatus(prev_height=prev_height) + spender_tx = self.txs.get(spender_txhash, None) + if spender_tx is None: + self.logger.warning(f"spender_tx {hash_to_hex_str(spender_txhash)} not in" + f"mempool, but txo_to_spender referenced it as spender " + f"of {hash_to_hex_str(prev_txhash)}:{txout_idx} ?!") + return TXOSpendStatus(prev_height=prev_height) + spender_has_ui = any(hash in self.txs for hash, idx in spender_tx.prevouts) + spender_height = -spender_has_ui + return TXOSpendStatus( + prev_height=prev_height, + spender_txhash=spender_txhash, + spender_height=spender_height, + ) diff --git a/src/electrumx/server/session.py b/src/electrumx/server/session.py index 7a4ad9107..e7e7a7c79 100644 --- a/src/electrumx/server/session.py +++ b/src/electrumx/server/session.py @@ -18,7 +18,7 @@ from collections import defaultdict from functools import partial from ipaddress import IPv4Address, IPv6Address, IPv4Network, IPv6Network -from typing import Iterable, Optional, TYPE_CHECKING, Sequence, Union, Any +from typing import Iterable, Optional, TYPE_CHECKING, Sequence, Union, Any, Tuple, Set, Dict, Mapping import attr from aiorpcx import (Event, JSONRPCAutoDetect, JSONRPCConnection, @@ -34,6 +34,7 @@ hex_str_to_hash, sha256, double_sha256) from electrumx.lib.merkle import MerkleCache from electrumx.lib.text import sessions_lines +from electrumx.lib.tx import TXOSpendStatus from electrumx.server.daemon import DaemonError from electrumx.server.transport import PaddedRSTransport @@ -108,7 +109,7 @@ def assert_list_or_tuple(value: Any) -> None: class SessionGroup: name = attr.ib() weight = attr.ib() - sessions = attr.ib() + sessions = attr.ib() # type: Set[ElectrumX] retained_cost = attr.ib() def session_cost(self): @@ -151,8 +152,8 @@ def __init__( self.shutdown_event = shutdown_event self.logger = util.class_logger(__name__, self.__class__.__name__) self.servers = {} # service->server - self.sessions = {} # session->iterable of its SessionGroups - self.session_groups = {} # group name->SessionGroup instance + self.sessions = {} # type: Dict[ElectrumX, Iterable[SessionGroup]] + self.session_groups = {} # type: Dict[str, SessionGroup] self.txs_sent = 0 # Would use monotonic time, but aiorpcx sessions use Unix time: self.start_time = time.time() @@ -324,7 +325,7 @@ async def _recalc_concurrency(self): # cost_decay_per_sec. for session in self.sessions: # Subs have an on-going cost so decay more slowly with more subs - session.cost_decay_per_sec = hard_limit / (10000 + 5 * session.sub_count()) + session.cost_decay_per_sec = hard_limit / (10000 + 5 * session.sub_count_total()) session.recalc_concurrency() def _get_info(self): @@ -346,11 +347,14 @@ def cache_fmt(cache: LRUCache): 'request total': sum(self._method_counts.values()), 'sessions': { 'count': len(sessions), - 'count with subs': sum(len(getattr(s, 'hashX_subs', ())) > 0 for s in sessions), + 'count with subs_sh': sum(s.sub_count_scripthashes() > 0 for s in sessions), + 'count with subs_txo': sum(s.sub_count_txoutpoints() > 0 for s in sessions), + 'count with subs_any': sum(s.sub_count_total() > 0 for s in sessions), 'errors': sum(s.errors for s in sessions), 'logged': len([s for s in sessions if s.log_me]), 'pending requests': sum(s.unanswered_request_count() for s in sessions), - 'subs': sum(s.sub_count() for s in sessions), + 'subs_sh': sum(s.sub_count_scripthashes() for s in sessions), + 'subs_txo': sum(s.sub_count_txoutpoints() for s in sessions), }, 'txids cache': cache_fmt(self._txids_cache), 'txs sent': self.txs_sent, @@ -371,7 +375,7 @@ def _session_data(self, for_log): session.extra_cost(), session.unanswered_request_count(), session.txs_sent, - session.sub_count(), + session.sub_count_total(), session.recv_count, session.recv_size, session.send_count, session.send_size, now - session.start_time) @@ -388,7 +392,7 @@ def _group_data(self): group.retained_cost, sum(s.unanswered_request_count() for s in sessions), sum(s.txs_sent for s in sessions), - sum(s.sub_count() for s in sessions), + sum(s.sub_count_total() for s in sessions), sum(s.recv_count for s in sessions), sum(s.recv_size for s in sessions), sum(s.send_count for s in sessions), @@ -845,21 +849,32 @@ async def limited_history(self, hashX): raise result return result, cost - async def _notify_sessions(self, height, touched): + async def _notify_sessions( + self, + *, + touched_hashxs: Set[bytes], + touched_outpoints: Set[Tuple[bytes, int]], + height: int, + ): '''Notify sessions about height changes and touched addresses.''' height_changed = height != self.notified_height if height_changed: await self._refresh_hsub_results(height) # Invalidate our history cache for touched hashXs cache = self._history_cache - for hashX in set(cache).intersection(touched): + for hashX in set(cache).intersection(touched_hashxs): del cache[hashX] for session in self.sessions: if self._task_group.joined: # this can happen during shutdown self.logger.warning(f"task group already terminated. not notifying sessions.") return - await self._task_group.spawn(session.notify, touched, height_changed) + coro = session.notify( + touched_hashxs=touched_hashxs, + touched_outpoints=touched_outpoints, + height_changed=height_changed, + ) + await self._task_group.spawn(coro) def _ip_addr_group_name(self, session) -> Optional[str]: host = session.remote_address().host @@ -1002,7 +1017,13 @@ def __init__( self.session_mgr.add_session(self) self.recalc_concurrency() # must be called after session_mgr.add_session - async def notify(self, touched, height_changed): + async def notify( + self, + *, + touched_hashxs: Set[bytes], + touched_outpoints: Set[Tuple[bytes, int]], + height_changed: bool, + ): pass def default_framer(self): @@ -1038,9 +1059,15 @@ async def connection_lost(self): msg = 'disconnected' + msg self.logger.info(msg) - def sub_count(self): + def sub_count_scripthashes(self): + return 0 + + def sub_count_txoutpoints(self): return 0 + def sub_count_total(self): + return self.sub_count_scripthashes() + self.sub_count_txoutpoints() + async def handle_request(self, request): '''Handle an incoming request. ElectrumX doesn't receive notifications from client sessions. @@ -1095,9 +1122,10 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.subscribe_headers = False self.connection.max_response_size = self.env.max_send - self.hashX_subs = {} - self.sv_seen = False - self.mempool_statuses = {} + self.hashX_subs = {} # type: Dict[bytes, bytes] # hashX -> scripthash + self.txoutpoint_subs = set() # type: Set[Tuple[bytes, int]] + self.mempool_hashX_statuses = {} # type: Dict[bytes, str] + self.mempool_txoutpoint_statuses = {} # type: Dict[Tuple[bytes, int], Mapping[str, Any]] self.set_request_handlers(self.PROTOCOL_MIN) self.is_peer = False self.cost = 5.0 # Connection cost @@ -1150,46 +1178,68 @@ def on_disconnect_due_to_excessive_session_cost(self): group_names = [group.name for group in groups] self.logger.info(f"closing session over res usage. ip: {ip_addr}. groups: {group_names}") - def sub_count(self): + def sub_count_scripthashes(self): return len(self.hashX_subs) + def sub_count_txoutpoints(self): + return len(self.txoutpoint_subs) + def unsubscribe_hashX(self, hashX): - self.mempool_statuses.pop(hashX, None) + self.mempool_hashX_statuses.pop(hashX, None) return self.hashX_subs.pop(hashX, None) - async def notify(self, touched, height_changed): + async def notify( + self, + *, + touched_hashxs: Set[bytes], + touched_outpoints: Set[Tuple[bytes, int]], + height_changed: bool, + ): '''Wrap _notify_inner; websockets raises exceptions for unclear reasons.''' try: async with timeout_after(30): - await self._notify_inner(touched, height_changed) + await self._notify_inner( + touched_hashxs=touched_hashxs, + touched_outpoints=touched_outpoints, + height_changed=height_changed, + ) except TaskTimeout: self.logger.warning('timeout notifying client, closing...') await self.close(force_after=1.0) except Exception: self.logger.exception('unexpected exception notifying client') - async def _notify_inner(self, touched, height_changed): + async def _notify_inner( + self, + *, + touched_hashxs: Set[bytes], + touched_outpoints: Set[Tuple[bytes, int]], + height_changed: bool, + ): '''Notify the client about changes to touched addresses (from mempool updates or new blocks) and height. ''' + # block headers if height_changed and self.subscribe_headers: args = (await self.subscribe_headers_result(), ) await self.send_notification('blockchain.headers.subscribe', args) - touched = touched.intersection(self.hashX_subs) - if touched or (height_changed and self.mempool_statuses): + # hashXs + num_hashx_notifs_sent = 0 + touched_hashxs = touched_hashxs.intersection(self.hashX_subs) + if touched_hashxs or (height_changed and self.mempool_hashX_statuses): changed = {} - for hashX in touched: + for hashX in touched_hashxs: alias = self.hashX_subs.get(hashX) if alias: status = await self.subscription_address_status(hashX) changed[alias] = status # Check mempool hashXs - the status is a function of the confirmed state of - # other transactions. - mempool_statuses = self.mempool_statuses.copy() - for hashX, old_status in mempool_statuses.items(): + # other transactions. (this is to detect if height changed from -1 to 0) + mempool_hashX_statuses = self.mempool_hashX_statuses.copy() + for hashX, old_status in mempool_hashX_statuses.items(): alias = self.hashX_subs.get(hashX) if alias: status = await self.subscription_address_status(hashX) @@ -1199,10 +1249,36 @@ async def _notify_inner(self, touched, height_changed): method = 'blockchain.scripthash.subscribe' for alias, status in changed.items(): await self.send_notification(method, (alias, status)) - - if changed: - es = '' if len(changed) == 1 else 'es' - self.logger.info(f'notified of {len(changed):,d} address{es}') + num_hashx_notifs_sent = len(changed) + + # tx outpoints + num_txo_notifs_sent = 0 + touched_outpoints = touched_outpoints.intersection(self.txoutpoint_subs) + if touched_outpoints or (height_changed and self.mempool_txoutpoint_statuses): + method = 'blockchain.outpoint.subscribe' + txo_to_status = {} + for prevout in touched_outpoints: + txo_to_status[prevout] = await self.txoutpoint_status_for_notif(*prevout) + + # Check mempool TXOs - the status is a function of the confirmed state of + # other transactions. (this is to detect if height changed from -1 to 0) + mempool_txoutpoint_statuses = self.mempool_txoutpoint_statuses.copy() + for prevout, old_status in mempool_txoutpoint_statuses.items(): + status = await self.txoutpoint_status_for_notif(*prevout) + if status != old_status: + txo_to_status[prevout] = status + + for tx_hash, txout_idx in touched_outpoints: + spend_status = txo_to_status[(tx_hash, txout_idx)] + tx_hash_hex = hash_to_hex_str(tx_hash) + await self.send_notification(method, (tx_hash_hex, txout_idx, spend_status)) + num_txo_notifs_sent = len(touched_outpoints) + + if num_hashx_notifs_sent + num_txo_notifs_sent > 0: + es1 = '' if num_hashx_notifs_sent == 1 else 'es' + s2 = '' if num_txo_notifs_sent == 1 else 's' + self.logger.info(f'notified of {num_hashx_notifs_sent:,d} address{es1} and ' + f'{num_txo_notifs_sent:,d} outpoint{s2}') async def subscribe_headers_result(self): '''The result of a header subscription or notification.''' @@ -1225,10 +1301,11 @@ async def peers_subscribe(self): self.bump_cost(1.0) return self.peer_mgr.on_peers_subscribe(self.is_tor()) - async def address_status(self, hashX): + async def address_status(self, hashX: bytes) -> Optional[str]: '''Returns an address status. Status is a hex string, but must be None if there is no history. + Side-effect: updates client-last-seen status, used by notifications. ''' # Note both confirmed history and mempool history are ordered # For mempool, height is -1 if it has unconfirmed inputs, otherwise 0 @@ -1250,10 +1327,11 @@ async def address_status(self, hashX): else: status = None + # update status last sent to client if mempool: - self.mempool_statuses[hashX] = status + self.mempool_hashX_statuses[hashX] = status else: - self.mempool_statuses.pop(hashX, None) + self.mempool_hashX_statuses.pop(hashX, None) return status @@ -1266,6 +1344,103 @@ async def subscription_address_status(self, hashX): self.unsubscribe_hashX(hashX) return None + async def _spender_for_txo(self, prev_txhash: bytes, txout_idx: int) -> 'TXOSpendStatus': + """For an outpoint, returns its spend-status (ignoring mempool events). + + Uses daemon (bitcoind) to find the spender_txhash, requiring "txospenderindex=1". + However, mempool events are ignored, as it would be difficult to distinguish block height 0 vs -1 + using only the daemon. Instead, our own mempool data (as opposed to bitcoind's) can be used + separately to enrich the return value. + """ + prev_txid = hash_to_hex_str(prev_txhash) + # 1. call bitcoind "getrawtransaction" to see if prevtx exists/is_mined + self.bump_cost(1) + try: + prevtx_item = await self.session_mgr.daemon.getrawtransaction(prev_txid, verbose=True) # verbose=int(1) + except DaemonError as e: + error, = e.args + ecode = error['code'] + if ecode == -5: # "No such mempool or blockchain transaction." + return TXOSpendStatus(prev_height=None) # utxo never existed + self.logger.debug(f"getrawtransaction errored. {prev_txid=}. {error=}") + raise RPCError(DAEMON_ERROR, f'daemon error: {error!r}') from None # TODO some callers do not expect this + assert prevtx_item.get("txid") == prev_txid, f"{prevtx_item.get('txid')=} != {prev_txid=}" + funder_bhash = prevtx_item.get("blockhash") + funder_bheight = None # type: Optional[int] + if funder_bhash is not None: + funder_bheight = self.db.get_blockheight_from_blockhash(funder_bhash) + if funder_bheight is None: # if in mempool, will defer to mempool.spender_for_txo + return TXOSpendStatus(prev_height=None) # utxo never existed (in chain) + assert isinstance(funder_bheight, int) + # ok, funding tx exists, does the requested output index also exist in this tx? + vouts = prevtx_item.get("vout") or [] + if len(vouts) <= txout_idx: + return TXOSpendStatus(prev_height=None) # txout_idx was out-of-bounds + # by now we know the funding TXO existed in the chain. Let's see if it was spent. + # 2. call bitcoind "gettxspendingprevout" + self.bump_cost(1) + try: + spender_item = await self.session_mgr.daemon.gettxspendingprevout(prev_txid, txout_idx) + except DaemonError as e: + error, = e.args + self.logger.debug(f"gettxspendingprevout errored. txo={prev_txid}:{txout_idx}. {error=}") + raise RPCError(DAEMON_ERROR, f'daemon error: {error!r}') from None # TODO some callers do not expect this + assert spender_item.get("txid") == prev_txid, f"{spender_item.get('txid')=} != {prev_txid=}" + spender_bhash = spender_item.get("blockhash") + spender_bheight = None + if spender_bhash is not None: + spender_bheight = self.db.get_blockheight_from_blockhash(spender_bhash) + if spender_bheight is None: # if in mempool, will defer to mempool.spender_for_txo + return TXOSpendStatus(prev_height=funder_bheight) # utxo funded but unspent (in-chain) + spender_txid = spender_item.get("spendingtxid") + assert spender_txid is not None # we already have a height! + # utxo funded, and spent (in-chain) + return TXOSpendStatus( + prev_height=funder_bheight, + spender_txhash=hex_str_to_hash(spender_txid), + spender_height=spender_bheight, + ) + + async def _calc_txoutpoint_status(self, prev_txhash: bytes, txout_idx: int) -> Dict[str, Any]: + self.bump_cost(0.2) + spend_status = await self._spender_for_txo(prev_txhash, txout_idx) + if spend_status.spender_height is not None: + # TXO was created, was mined, was spent, and spend was mined. + assert spend_status.prev_height > 0 + assert spend_status.spender_height > 0 + assert spend_status.spender_txhash is not None + else: + mp_spend_status = await self.mempool.spender_for_txo(prev_txhash, txout_idx) + if mp_spend_status.prev_height is not None: + spend_status.prev_height = mp_spend_status.prev_height + if mp_spend_status.spender_height is not None: + spend_status.spender_height = mp_spend_status.spender_height + if mp_spend_status.spender_txhash is not None: + spend_status.spender_txhash = mp_spend_status.spender_txhash + # convert to json dict the client expects + status = {} + if spend_status.prev_height is not None: + status['height'] = spend_status.prev_height + if spend_status.spender_txhash is not None: + assert spend_status.spender_height is not None + status['spender_txhash'] = hash_to_hex_str(spend_status.spender_txhash) + status['spender_height'] = spend_status.spender_height + return status + + async def txoutpoint_status_for_notif(self, prev_txhash: bytes, txout_idx: int) -> Dict[str, Any]: + """Side-effect: updates client-last-seen status, used by notifications.""" + status = await self._calc_txoutpoint_status(prev_txhash=prev_txhash, txout_idx=txout_idx) + # update status last sent to client + prevout = (prev_txhash, txout_idx) + prev_height = status.get('height') # type: Optional[int] + spender_height = status.get('spender_height') # type: Optional[int] + if ((prev_height is not None and prev_height <= 0) + or (spender_height is not None and spender_height <= 0)): + self.mempool_txoutpoint_statuses[prevout] = status + else: + self.mempool_txoutpoint_statuses.pop(prevout, None) + return status + async def hashX_listunspent(self, hashX): '''Return the list of UTXOs of a script hash, including mempool effects.''' @@ -1345,6 +1520,47 @@ async def scripthash_unsubscribe(self, scripthash): hashX = scripthash_to_hashX(scripthash) return self.unsubscribe_hashX(hashX) is not None + + async def txoutpoint_get_status(self, tx_hash, txout_idx, spk_hint=None): + '''Return the status of an outpoint, without subscribing. + + spk_hint: scriptPubKey corresponding to the outpoint. Might be used by + other servers, but we don't need and hence ignore it. + ''' + tx_hash = assert_tx_hash(tx_hash) + txout_idx = non_negative_integer(txout_idx) + if spk_hint is not None: + assert_hex_str(spk_hint) + # calc status (but do not side-effect client-last-seen status) + spend_status = await self._calc_txoutpoint_status(tx_hash, txout_idx) + return spend_status + + async def txoutpoint_subscribe(self, tx_hash, txout_idx, spk_hint=None): + '''Subscribe to an outpoint. + + spk_hint: scriptPubKey corresponding to the outpoint. Might be used by + other servers, but we don't need and hence ignore it. + ''' + tx_hash = assert_tx_hash(tx_hash) + txout_idx = non_negative_integer(txout_idx) + if spk_hint is not None: + assert_hex_str(spk_hint) + # calc status, update client-last-seen status, and sub to outpoint + spend_status = await self.txoutpoint_status_for_notif(tx_hash, txout_idx) + self.txoutpoint_subs.add((tx_hash, txout_idx)) + return spend_status + + async def txoutpoint_unsubscribe(self, tx_hash, txout_idx): + '''Unsubscribe from an outpoint.''' + tx_hash = assert_tx_hash(tx_hash) + txout_idx = non_negative_integer(txout_idx) + self.bump_cost(0.1) + prevout = (tx_hash, txout_idx) + was_subscribed = prevout in self.txoutpoint_subs + self.txoutpoint_subs.discard(prevout) + self.mempool_txoutpoint_statuses.pop(prevout, None) + return was_subscribed + async def _merkle_proof(self, cp_height, height): max_height = self.db.db_height if not height <= cp_height <= max_height: @@ -1759,6 +1975,12 @@ def set_request_handlers(self, ptuple): else: handlers['blockchain.relayfee'] = self.relayfee # removed in 1.6 + # experimental: + if ptuple >= (1, 7): + handlers['blockchain.outpoint.subscribe'] = self.txoutpoint_subscribe + handlers['blockchain.outpoint.get_status'] = self.txoutpoint_get_status + handlers['blockchain.outpoint.unsubscribe'] = self.txoutpoint_unsubscribe + self.request_handlers = handlers @@ -1803,9 +2025,19 @@ def set_request_handlers(self, ptuple): 'protx.info': self.protx_info, }) - async def _notify_inner(self, touched, height_changed): + async def _notify_inner( + self, + *, + touched_hashxs, + touched_outpoints, + height_changed, + ): '''Notify the client about changes in masternode list.''' - await super()._notify_inner(touched, height_changed) + await super()._notify_inner( + touched_hashxs=touched_hashxs, + touched_outpoints=touched_outpoints, + height_changed=height_changed, + ) for mn in self.mns.copy(): status = await self.daemon_request('masternode_list', ('status', mn)) diff --git a/tests/server/test_mempool.py b/tests/server/test_mempool.py index 6cbbc81b3..700c0d135 100644 --- a/tests/server/test_mempool.py +++ b/tests/server/test_mempool.py @@ -222,34 +222,20 @@ def cached_height(self): return self._cached_height async def mempool_hashes(self): - '''Query bitcoind for the hashes of all transactions in its - mempool, returned as a list.''' await sleep(0) return [hash_to_hex_str(hash) for hash in self.txs] async def raw_transactions(self, hex_hashes): - '''Query bitcoind for the serialized raw transactions with the given - hashes. Missing transactions are returned as None. - - hex_hashes is an iterable of hexadecimal hash strings.''' await sleep(0) hashes = [hex_str_to_hash(hex_hash) for hex_hash in hex_hashes] return [self.raw_txs.get(hash) for hash in hashes] async def lookup_utxos(self, prevouts): - '''Return a list of (hashX, value) pairs each prevout if unspent, - otherwise return None if spent or not found. - - prevouts - an iterable of (hash, index) pairs - ''' await sleep(0) return [self.db_utxos.get(prevout) for prevout in prevouts] - async def on_mempool(self, touched, height): - '''Called each time the mempool is synchronized. touched is a set of - hashXs touched since the previous call. height is the - daemon's height at the time the mempool was obtained.''' - self.on_mempool_calls.append((touched, height)) + async def on_mempool(self, *, touched_hashxs, touched_outpoints, height): + self.on_mempool_calls.append((touched_hashxs, height)) await sleep(0) diff --git a/tests/server/test_notifications.py b/tests/server/test_notifications.py index c8c55b311..ee14f237b 100644 --- a/tests/server/test_notifications.py +++ b/tests/server/test_notifications.py @@ -7,15 +7,15 @@ async def test_simple_mempool(): n = Notifications() notified = [] - async def notify(height, touched): - notified.append((height, touched)) + async def notify(*, touched_hashxs, touched_outpoints, height): + notified.append((height, touched_hashxs)) await n.start(5, notify) - mtouched = {'a', 'b'} - btouched = {'b', 'c'} - await n.on_mempool(mtouched, 6) + mtouched = {b'a', b'b'} + btouched = {b'b', b'c'} + await n.on_mempool(touched_hashxs=mtouched, height=6, touched_outpoints=set()) assert notified == [(5, set())] - await n.on_block(btouched, 6) + await n.on_block(touched_hashxs=btouched, height=6, touched_outpoints=set()) assert notified == [(5, set()), (6, set.union(mtouched, btouched))] @@ -23,23 +23,23 @@ async def notify(height, touched): async def test_enter_mempool_quick_blocks_2(): n = Notifications() notified = [] - async def notify(height, touched): - notified.append((height, touched)) + async def notify(*, touched_hashxs, touched_outpoints, height): + notified.append((height, touched_hashxs)) await n.start(5, notify) # Suppose a gets in block 6 and blocks 7,8 found right after and # the block processor processes them together. - await n.on_mempool({'a'}, 5) - assert notified == [(5, set()), (5, {'a'})] + await n.on_mempool(touched_hashxs={b'a'}, height=5, touched_outpoints=set()) + assert notified == [(5, set()), (5, {b'a'})] # Mempool refreshes with daemon on block 6 - await n.on_mempool({'a'}, 6) - assert notified == [(5, set()), (5, {'a'})] + await n.on_mempool(touched_hashxs={b'a'}, height=6, touched_outpoints=set()) + assert notified == [(5, set()), (5, {b'a'})] # Blocks 6, 7 processed together - await n.on_block({'a', 'b'}, 7) - assert notified == [(5, set()), (5, {'a'})] + await n.on_block(touched_hashxs={b'a', b'b'}, height=7, touched_outpoints=set()) + assert notified == [(5, set()), (5, {b'a'})] # Then block 8 processed - await n.on_block({'c'}, 8) - assert notified == [(5, set()), (5, {'a'})] + await n.on_block(touched_hashxs={b'c'}, height=8, touched_outpoints=set()) + assert notified == [(5, set()), (5, {b'a'})] # Now mempool refreshes - await n.on_mempool(set(), 8) - assert notified == [(5, set()), (5, {'a'}), (8, {'a', 'b', 'c'})] + await n.on_mempool(touched_hashxs=set(), height=8, touched_outpoints=set()) + assert notified == [(5, set()), (5, {b'a'}), (8, {b'a', b'b', b'c'})] From 093dcb733f642760f954477c7ea95447d64b0443 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Thu, 7 May 2026 13:33:00 +0000 Subject: [PATCH 10/27] daemon: require "txospenderindex" for bitcoind, raise if missing --- src/electrumx/lib/coins.py | 9 +++++++-- src/electrumx/server/controller.py | 1 + src/electrumx/server/daemon.py | 19 +++++++++++++++++++ 3 files changed, 27 insertions(+), 2 deletions(-) diff --git a/src/electrumx/lib/coins.py b/src/electrumx/lib/coins.py index d459151d7..1f8a468c1 100644 --- a/src/electrumx/lib/coins.py +++ b/src/electrumx/lib/coins.py @@ -115,6 +115,7 @@ class Coin: NAME: str NET: str MIN_REQUIRED_DAEMON_VERSION: Optional[str] = None + REQUIRED_DAEMON_INDEXES: Sequence[str] = tuple() # only used for initial db sync ETAs: TX_COUNT_HEIGHT: int # at a given snapshot of the chain, @@ -308,7 +309,9 @@ class Bitcoin(BitcoinMixin, Coin): TX_PER_BLOCK = 2200 CRASH_CLIENT_VER = (3, 2, 3) # core version 28 introduced 1p1c package relay required for protocol 1.6 - MIN_REQUIRED_DAEMON_VERSION = "28.0" + # core version 31 introduced "txospenderindex", required for protocol 1.7 + MIN_REQUIRED_DAEMON_VERSION = "31.0" + REQUIRED_DAEMON_INDEXES = ("txindex", "txospenderindex",) BLACKLIST_URL = 'https://electrum.org/blacklist.json' PEERS = [ 'electrum.vom-stausee.de s t', @@ -386,7 +389,9 @@ class BitcoinTestnet(BitcoinTestnetMixin, Coin): NAME = "Bitcoin" DESERIALIZER = lib_tx.DeserializerSegWit CRASH_CLIENT_VER = (3, 2, 3) - MIN_REQUIRED_DAEMON_VERSION = "28.0" + # core version 31 introduced "txospenderindex", required for protocol 1.7 + MIN_REQUIRED_DAEMON_VERSION = "31.0" + REQUIRED_DAEMON_INDEXES = ("txindex", "txospenderindex",) PEERS = [ 'testnet.hsmiths.com t53011 s53012', 'testnet.qtornado.com s t', diff --git a/src/electrumx/server/controller.py b/src/electrumx/server/controller.py index 36408edc0..d09420397 100644 --- a/src/electrumx/server/controller.py +++ b/src/electrumx/server/controller.py @@ -173,6 +173,7 @@ def get_db_height(): # Check if daemon is recent enough await daemon.check_daemon_version() + await daemon.check_daemon_indexes() caught_up_event = Event() mempool_event = Event() diff --git a/src/electrumx/server/daemon.py b/src/electrumx/server/daemon.py index f8b806628..25b3233a9 100644 --- a/src/electrumx/server/daemon.py +++ b/src/electrumx/server/daemon.py @@ -108,6 +108,20 @@ async def check_daemon_version(self): if daemon_version < required_version: raise RuntimeError(f"Bitcoin Core {daemon_version=} < {required_version=}.") + async def check_daemon_indexes(self): + assert self.session is not None and self.coin is not None, f"{self.session=}, {self.coin=}" + if not self.coin.REQUIRED_DAEMON_INDEXES: + return + index_info = await self.getindexinfo() + for req_index in self.coin.REQUIRED_DAEMON_INDEXES: + if req_index not in index_info: + raise RuntimeError( + f"bitcoind missing required index: {req_index}. " + f"You should set {req_index}=1 in your bitcoin.conf config file, and reindex.") + if not index_info[req_index]["synced"]: + # Should we raise? Not raising allows syncing a fresh bitcoind and e-x "in parallel". + self.logger.warning(f"bitcoind required index {req_index!r} is still syncing!") + def set_url(self, url): '''Set the URLS to the given list, and switch to the first one.''' urls = url.split(',') @@ -312,6 +326,11 @@ async def getmempoolinfo(self): self._mempoolinfo_cache = (val, time.time()) return val + async def getindexinfo(self): + """Return the result of the 'getindexinfo' RPC call.""" + # note: not cached as it's not exposed to client sessions + return await self._send_single('getindexinfo') + async def relayfee(self): """Same as getmempoolinfo['minrelaytxfee']. The minimum fee required for a transaction to be relayed on by the daemon to the From f4ff47ca0d3bdcc7e9eb613b3430b0118634dad7 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Wed, 26 Feb 2025 17:23:26 +0000 Subject: [PATCH 11/27] session: add "blockchain.scriptpubkey.*" RPCs --- src/electrumx/server/session.py | 60 ++++++++++++++++++++++++++++----- 1 file changed, 51 insertions(+), 9 deletions(-) diff --git a/src/electrumx/server/session.py b/src/electrumx/server/session.py index e7e7a7c79..9d3f6f287 100644 --- a/src/electrumx/server/session.py +++ b/src/electrumx/server/session.py @@ -15,6 +15,7 @@ import os import ssl import time +import collections from collections import defaultdict from functools import partial from ipaddress import IPv4Address, IPv6Address, IPv4Network, IPv6Network @@ -51,7 +52,7 @@ DAEMON_ERROR = 2 -def scripthash_to_hashX(scripthash): +def scripthash_to_hashX(scripthash: str) -> bytes: try: bin_hash = hex_str_to_hash(scripthash) if len(bin_hash) == 32: @@ -61,6 +62,13 @@ def scripthash_to_hashX(scripthash): raise RPCError(BAD_REQUEST, f'{scripthash} is not a valid script hash') +def spk_to_scripthash(spk: str) -> str: + """Converts scriptPubKey to scripthash.""" + assert_hex_str(spk) + h = sha256(bytes.fromhex(spk)) + return h[::-1].hex() + + def non_negative_integer(value): '''Return param value it is or can be converted to a non-negative integer, otherwise raise an RPCError.''' @@ -1246,7 +1254,10 @@ async def _notify_inner( if status != old_status: changed[alias] = status - method = 'blockchain.scripthash.subscribe' + if self.protocol_tuple >= (1, 7): + method = 'blockchain.scriptpubkey.subscribe' + else: + method = 'blockchain.scripthash.subscribe' for alias, status in changed.items(): await self.send_notification(method, (alias, status)) num_hashx_notifs_sent = len(changed) @@ -1459,7 +1470,7 @@ async def hashX_listunspent(self, hashX): async def hashX_subscribe(self, hashX, alias): # Store the subscription only after address_status succeeds result = await self.address_status(hashX) - self.hashX_subs[hashX] = alias + self.hashX_subs[hashX] = alias # TODO rename alias to scripthash return result async def get_balance(self, hashX): @@ -1520,6 +1531,29 @@ async def scripthash_unsubscribe(self, scripthash): hashX = scripthash_to_hashX(scripthash) return self.unsubscribe_hashX(hashX) is not None + def scriptpubkey_get_balance(self, spk: str) -> collections.abc.Awaitable[dict]: + scripthash = spk_to_scripthash(spk) + return self.scripthash_get_balance(scripthash) + + def scriptpubkey_get_history(self, spk: str) -> collections.abc.Awaitable[list]: + scripthash = spk_to_scripthash(spk) + return self.scripthash_get_history(scripthash) + + def scriptpubkey_get_mempool(self, spk: str) -> collections.abc.Awaitable[list]: + scripthash = spk_to_scripthash(spk) + return self.scripthash_get_mempool(scripthash) + + def scriptpubkey_listunspent(self, spk: str) -> collections.abc.Awaitable[list]: + scripthash = spk_to_scripthash(spk) + return self.scripthash_listunspent(scripthash) + + def scriptpubkey_subscribe(self, spk: str) -> collections.abc.Awaitable[Optional[str]]: + scripthash = spk_to_scripthash(spk) + return self.scripthash_subscribe(scripthash) + + def scriptpubkey_unsubscribe(self, spk: str) -> collections.abc.Awaitable[bool]: + scripthash = spk_to_scripthash(spk) + return self.scripthash_unsubscribe(scripthash) async def txoutpoint_get_status(self, tx_hash, txout_idx, spk_hint=None): '''Return the status of an outpoint, without subscribing. @@ -1947,11 +1981,6 @@ def set_request_handlers(self, ptuple): 'blockchain.block.headers': self.block_headers, 'blockchain.estimatefee': self.estimatefee, 'blockchain.headers.subscribe': self.headers_subscribe, - 'blockchain.scripthash.get_balance': self.scripthash_get_balance, - 'blockchain.scripthash.get_history': self.scripthash_get_history, - 'blockchain.scripthash.get_mempool': self.scripthash_get_mempool, - 'blockchain.scripthash.listunspent': self.scripthash_listunspent, - 'blockchain.scripthash.subscribe': self.scripthash_subscribe, 'blockchain.transaction.broadcast': self.transaction_broadcast, 'blockchain.transaction.get': self.transaction_get, 'blockchain.transaction.get_merkle': self.transaction_merkle, @@ -1966,7 +1995,14 @@ def set_request_handlers(self, ptuple): 'server.version': self.server_version, } - if ptuple >= (1, 4, 2): + if ptuple < (1, 7): + handlers['blockchain.scripthash.get_balance'] = self.scripthash_get_balance + handlers['blockchain.scripthash.get_history'] = self.scripthash_get_history + handlers['blockchain.scripthash.get_mempool'] = self.scripthash_get_mempool + handlers['blockchain.scripthash.listunspent'] = self.scripthash_listunspent + handlers['blockchain.scripthash.subscribe'] = self.scripthash_subscribe + + if (1, 4, 2) <= ptuple < (1, 7): handlers['blockchain.scripthash.unsubscribe'] = self.scripthash_unsubscribe if ptuple >= (1, 6): @@ -1980,6 +2016,12 @@ def set_request_handlers(self, ptuple): handlers['blockchain.outpoint.subscribe'] = self.txoutpoint_subscribe handlers['blockchain.outpoint.get_status'] = self.txoutpoint_get_status handlers['blockchain.outpoint.unsubscribe'] = self.txoutpoint_unsubscribe + handlers['blockchain.scriptpubkey.get_balance'] = self.scriptpubkey_get_balance + handlers['blockchain.scriptpubkey.get_history'] = self.scriptpubkey_get_history + handlers['blockchain.scriptpubkey.get_mempool'] = self.scriptpubkey_get_mempool + handlers['blockchain.scriptpubkey.listunspent'] = self.scriptpubkey_listunspent + handlers['blockchain.scriptpubkey.subscribe'] = self.scriptpubkey_subscribe + handlers['blockchain.scriptpubkey.unsubscribe'] = self.scriptpubkey_unsubscribe self.request_handlers = handlers From 56a5a37212182f9ecbe2fbe58438cc3923390465 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Wed, 26 Feb 2025 17:54:54 +0000 Subject: [PATCH 12/27] session: add 'block_hash' field to 'transaction.get_merkle' --- src/electrumx/lib/coins.py | 2 +- src/electrumx/server/db.py | 4 ++-- src/electrumx/server/session.py | 27 ++++++++++++++++++++------- 3 files changed, 23 insertions(+), 10 deletions(-) diff --git a/src/electrumx/lib/coins.py b/src/electrumx/lib/coins.py index 1f8a468c1..1a08854ca 100644 --- a/src/electrumx/lib/coins.py +++ b/src/electrumx/lib/coins.py @@ -235,7 +235,7 @@ def privkey_WIF(cls, privkey_bytes, compressed): return cls.ENCODE_CHECK(payload) @classmethod - def header_hash(cls, header): + def header_hash(cls, header: bytes) -> bytes: '''Given a header return hash''' return double_sha256(header) diff --git a/src/electrumx/server/db.py b/src/electrumx/server/db.py index 0b4b8be7b..747b9906c 100644 --- a/src/electrumx/server/db.py +++ b/src/electrumx/server/db.py @@ -444,14 +444,14 @@ def backup_fs(self, height, tx_count): # Truncate header_mc: header count is 1 more than the height. self.header_mc.truncate(height + 1) - async def raw_header(self, height): + async def raw_header(self, height: int) -> bytes: '''Return the binary header at the given height.''' header, n = await self.read_headers(height, 1) if n != 1: raise IndexError(f'height {height:,d} out of range') return header - async def read_headers(self, start_height, count): + async def read_headers(self, start_height: int, count: int) -> Tuple[bytes, int]: '''Requires start_height >= 0, count >= 0. Reads as many headers as are available starting at start_height up to count. This would be zero if start_height is beyond self.db_height, for diff --git a/src/electrumx/server/session.py b/src/electrumx/server/session.py index 9d3f6f287..ecf2db800 100644 --- a/src/electrumx/server/session.py +++ b/src/electrumx/server/session.py @@ -757,8 +757,11 @@ async def tx_hashes_func(start, count): branch = [hash_to_hex_str(hash) for hash in branch] return branch, cost / 2500 - async def merkle_branch_for_tx_hash(self, height, tx_hash): - '''Return a triple (branch, tx_pos, cost).''' + async def merkle_branch_for_tx_hash( + self, *, tx_hash: bytes, height: int, + ) -> Tuple[Sequence[str], int, bytes, float]: + '''Return (branch, tx_pos, block_header, cost).''' + block_header = await self.raw_header(height) tx_hashes, tx_hashes_cost = await self.tx_hashes_at_blockheight(height) try: tx_pos = tx_hashes.index(tx_hash) @@ -766,7 +769,11 @@ async def merkle_branch_for_tx_hash(self, height, tx_hash): raise RPCError(BAD_REQUEST, f'tx {hash_to_hex_str(tx_hash)} not in block at height {height:,d}') branch, merkle_cost = await self._merkle_branch(height, tx_hashes, tx_pos) - return branch, tx_pos, tx_hashes_cost + merkle_cost + if block_header != await self.raw_header(height): + # there was a reorg while processing the request... TODO maybe retry? + raise RPCError(BAD_REQUEST, + f'tx {hash_to_hex_str(tx_hash)} was reorged while processing request') + return branch, tx_pos, block_header, tx_hashes_cost + merkle_cost async def merkle_branch_for_tx_pos(self, height, tx_pos): '''Return a triple (branch, tx_hash_hex, cost).''' @@ -816,7 +823,7 @@ async def daemon_request(self, method, *args): except DaemonError as e: raise RPCError(DAEMON_ERROR, f'daemon error: {e!r}') from None - async def raw_header(self, height): + async def raw_header(self, height: int) -> bytes: '''Return the binary header at the given height.''' try: return await self.db.raw_header(height) @@ -1939,11 +1946,17 @@ async def transaction_merkle(self, tx_hash, height): tx_hash = assert_tx_hash(tx_hash) height = non_negative_integer(height) - branch, tx_pos, cost = await self.session_mgr.merkle_branch_for_tx_hash( - height, tx_hash) + branch, tx_pos, block_header, cost = await self.session_mgr.merkle_branch_for_tx_hash( + tx_hash=tx_hash, height=height) self.bump_cost(cost) + blockhash = hash_to_hex_str(self.coin.header_hash(block_header)) - return {"block_height": height, "merkle": branch, "pos": tx_pos} + return { + "block_height": height, + "block_hash": blockhash, + "merkle": branch, + "pos": tx_pos, + } async def transaction_id_from_pos(self, height, tx_pos, merkle=False): '''Return the txid and optionally a merkle proof, given From c6fe38fb1ef5d2ac2273ecea9efd897c9279179d Mon Sep 17 00:00:00 2001 From: SomberNight Date: Wed, 26 Feb 2025 19:30:22 +0000 Subject: [PATCH 13/27] implement new "server.ping" semantics for protocol 1.7 --- src/electrumx/server/session.py | 42 +++++++++++++++++++++++++-------- 1 file changed, 32 insertions(+), 10 deletions(-) diff --git a/src/electrumx/server/session.py b/src/electrumx/server/session.py index ecf2db800..97298f725 100644 --- a/src/electrumx/server/session.py +++ b/src/electrumx/server/session.py @@ -20,12 +20,15 @@ from functools import partial from ipaddress import IPv4Address, IPv6Address, IPv4Network, IPv6Network from typing import Iterable, Optional, TYPE_CHECKING, Sequence, Union, Any, Tuple, Set, Dict, Mapping +from typing import Callable import attr from aiorpcx import (Event, JSONRPCAutoDetect, JSONRPCConnection, ReplyAndDisconnect, Request, RPCError, RPCSession, Service, handler_invocation, serve_rs, serve_ws, sleep, - NewlineFramer, TaskTimeout, timeout_after, run_in_thread) + NewlineFramer, TaskTimeout, timeout_after, run_in_thread, + Notification) +from aiorpcx.jsonrpc import SingleRequest import electrumx import electrumx.lib.util as util @@ -991,6 +994,8 @@ class SessionBase(RPCSessionWithTaskGroup): MAX_CHUNK_SIZE = 2016 session_counter = itertools.count() log_new = False + request_handlers: Dict[str, Callable] + notification_handlers: Dict[str, Callable] def __init__( self, @@ -1083,14 +1088,13 @@ def sub_count_txoutpoints(self): def sub_count_total(self): return self.sub_count_scripthashes() + self.sub_count_txoutpoints() - async def handle_request(self, request): - '''Handle an incoming request. ElectrumX doesn't receive - notifications from client sessions. - ''' + async def handle_request(self, request: SingleRequest): + '''Handle an incoming request.''' + handler = None if isinstance(request, Request): handler = self.request_handlers.get(request.method) - else: - handler = None + elif isinstance(request, Notification): + handler = self.notification_handlers.get(request.method) method = 'invalid method' if handler is None else request.method # Version negotiation must happen before any other messages. @@ -1800,12 +1804,25 @@ async def estimatefee(self, number, mode=None): cache[(number, mode)] = (blockhash, feerate, lock) return feerate - async def ping(self): + async def ping(self, pong_len=0, data=""): '''Serves as a connection keep-alive mechanism and for the client to - confirm the server is still responding. + confirm the server is still responding. It can also be used to obfuscate + traffic patterns. ''' self.bump_cost(0.1) - return None + if self.protocol_tuple < (1, 7): + return None + assert_hex_str(data) + pong_len = non_negative_integer(pong_len) + if pong_len > self.env.max_send: + raise RPCError(BAD_REQUEST, f'pong_len value too high') + pong_data = pong_len * "0" + return {"data": pong_data} + + async def on_ping_notification(self, data=""): + self.bump_cost(0.1) # note: the bw cost for receiving 'data' has already been incurred + assert_hex_str(data) + # nothing to do async def server_version( self, @@ -2007,6 +2024,7 @@ def set_request_handlers(self, ptuple): 'server.ping': self.ping, 'server.version': self.server_version, } + notif_handlers = {} if ptuple < (1, 7): handlers['blockchain.scripthash.get_balance'] = self.scripthash_get_balance @@ -2035,8 +2053,10 @@ def set_request_handlers(self, ptuple): handlers['blockchain.scriptpubkey.listunspent'] = self.scriptpubkey_listunspent handlers['blockchain.scriptpubkey.subscribe'] = self.scriptpubkey_subscribe handlers['blockchain.scriptpubkey.unsubscribe'] = self.scriptpubkey_unsubscribe + notif_handlers['server.ping'] = self.on_ping_notification self.request_handlers = handlers + self.notification_handlers = notif_handlers class LocalRPC(SessionBase): @@ -2050,6 +2070,8 @@ def __init__(self, *args, **kwargs): self.sv_negotiated.set() self.client = 'RPC' self.connection.max_response_size = 0 + # note: self.request_handlers are set on the class, in SessionManager.__init__ + self.notification_handlers = {} def protocol_version_string(self): return 'RPC' From 041fb9791ae6cde4da34821363e01904a8bd537c Mon Sep 17 00:00:00 2001 From: SomberNight Date: Mon, 4 May 2026 15:32:04 +0000 Subject: [PATCH 14/27] session: add "bc.tx.testmempoolaccept" RPC --- src/electrumx/server/daemon.py | 6 ++++++ src/electrumx/server/session.py | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+) diff --git a/src/electrumx/server/daemon.py b/src/electrumx/server/daemon.py index 25b3233a9..a1a6cea01 100644 --- a/src/electrumx/server/daemon.py +++ b/src/electrumx/server/daemon.py @@ -382,6 +382,12 @@ async def broadcast_package(self, raw_txs: Sequence[str]): """Broadcast a package of transactions to the network using 'submitpackage'.""" return await self._send_single('submitpackage', (raw_txs, )) + async def testmempoolaccept(self, raw_txs: Sequence[str]): + """Query the daemon to test mempool acceptance of txs, + without adding them to the mempool or broadcasting them. + """ + return await self._send_single('testmempoolaccept', (raw_txs, )) + async def height(self): '''Query the daemon for its current height.''' self._height = await self._send_single('getblockcount') diff --git a/src/electrumx/server/session.py b/src/electrumx/server/session.py index 97298f725..16126ce82 100644 --- a/src/electrumx/server/session.py +++ b/src/electrumx/server/session.py @@ -1940,6 +1940,37 @@ async def package_broadcast(self, tx_package: Sequence[str], verbose: bool = Fal response['errors'] = errors return response + async def transaction_testmempoolaccept(self, raw_txs: Sequence[str]) -> Sequence[dict]: + """Returns result of mempool acceptance tests indicating if txs would be accepted by mempool. + + raw_txs: a list of raw transactions as hexadecimal strings + """ + assert_list_or_tuple(raw_txs) + for raw_tx in raw_txs: + assert_hex_str(raw_tx) + self.bump_cost(0.25 + sum(len(tx) / 5000 for tx in raw_txs)) + daemon_result = await self.daemon_request("testmempoolaccept", raw_txs) + + response: list[dict] = [] + for orig_item in daemon_result: # one item for each tx + new_item = { + "txid": orig_item["txid"], + "wtxid": orig_item["wtxid"], + } + # optional: "allowed" field + if orig_item.get("allowed") in (True, False): + new_item["allowed"] = orig_item["allowed"] + # optional: "reason" field + reason_str = ( + orig_item.get("package-error") + or orig_item.get("reject-details") + or orig_item.get("reject-reason") + or None) + if reason_str is not None: + new_item["reason"] = reason_str + response.append(new_item) + return response + async def transaction_get(self, tx_hash, verbose=False): '''Return the serialized raw transaction given its hash @@ -2044,6 +2075,7 @@ def set_request_handlers(self, ptuple): # experimental: if ptuple >= (1, 7): + handlers['blockchain.transaction.testmempoolaccept'] = self.transaction_testmempoolaccept handlers['blockchain.outpoint.subscribe'] = self.txoutpoint_subscribe handlers['blockchain.outpoint.get_status'] = self.txoutpoint_get_status handlers['blockchain.outpoint.unsubscribe'] = self.txoutpoint_unsubscribe From fe5e170b2f7e7df6722d83ad79d1893bc8550d12 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Mon, 17 Nov 2025 19:48:08 +0000 Subject: [PATCH 15/27] session: add "mempool.recent" RPC --- src/electrumx/server/mempool.py | 17 +++++++++++++++++ src/electrumx/server/session.py | 14 ++++++++++++++ tests/server/test_mempool.py | 27 +++++++++++++++++++++++++++ 3 files changed, 58 insertions(+) diff --git a/src/electrumx/server/mempool.py b/src/electrumx/server/mempool.py index 280531be9..8040d0e95 100644 --- a/src/electrumx/server/mempool.py +++ b/src/electrumx/server/mempool.py @@ -12,6 +12,7 @@ from abc import ABC, abstractmethod from asyncio import Lock from collections import defaultdict +from dataclasses import dataclass from typing import Sequence, Tuple, TYPE_CHECKING, Type, Dict, Optional, Set import math @@ -45,6 +46,13 @@ class MemPoolTxSummary: has_unconfirmed_inputs = attr.ib() # type: bool +@dataclass(slots=True, frozen=True, kw_only=True) +class RecentMemPoolTx: + hash: bytes + fee: int # in sats + vsize: int # in vbytes + + class DBSyncError(Exception): pass @@ -546,3 +554,12 @@ async def spender_for_txo(self, prev_txhash: bytes, txout_idx: int) -> 'TXOSpend spender_txhash=spender_txhash, spender_height=spender_height, ) + + async def get_recently_added_txs(self, *, count: int) -> Sequence[RecentMemPoolTx]: + # note: inefficient for large "count"s + it = reversed(self.txs.items()) + count = min(count, len(self.txs)) + mempool_txs = [next(it) for _ in range(count)] + return [ + RecentMemPoolTx(hash=hash, fee=mtx.fee, vsize=mtx.size) + for hash, mtx in mempool_txs] diff --git a/src/electrumx/server/session.py b/src/electrumx/server/session.py index 16126ce82..6dfa90fd7 100644 --- a/src/electrumx/server/session.py +++ b/src/electrumx/server/session.py @@ -1761,6 +1761,19 @@ async def mempool_info(self) -> dict[str, float]: self.bump_cost(1.0) return await self.daemon_request('mempool_info') + async def mempool_recent(self) -> list[dict[str, Any]]: + """ + mempool.recent, introduced in protocol 1.6.1. + Return a list of the last 10 transactions to enter the mempool. + """ + self.bump_cost(1.0) + recent_txs = await self.mempool.get_recently_added_txs(count=10) + return [{ + "txid": hash_to_hex_str(tx.hash), + "fee": tx.fee, + "vsize": tx.vsize, + } for tx in recent_txs] + async def estimatefee(self, number, mode=None): '''The estimated transaction fee per kilobyte to be paid for a transaction to be included within a certain number of blocks. @@ -2085,6 +2098,7 @@ def set_request_handlers(self, ptuple): handlers['blockchain.scriptpubkey.listunspent'] = self.scriptpubkey_listunspent handlers['blockchain.scriptpubkey.subscribe'] = self.scriptpubkey_subscribe handlers['blockchain.scriptpubkey.unsubscribe'] = self.scriptpubkey_unsubscribe + handlers['mempool.recent'] = self.mempool_recent notif_handlers['server.ping'] = self.on_ping_notification self.request_handlers = handlers diff --git a/tests/server/test_mempool.py b/tests/server/test_mempool.py index 700c0d135..396c42c24 100644 --- a/tests/server/test_mempool.py +++ b/tests/server/test_mempool.py @@ -549,6 +549,33 @@ async def test_notifications(caplog): await group.cancel_remaining() +@pytest.mark.asyncio +async def test_get_recently_added_txs(): + mempool_size_target = 50 + api = API() + api.initialize(mempool_size=mempool_size_target) + mempool = MemPool(coin, api, refresh_secs=0.001, log_status_secs=0) + event = Event() + + raw_txs = api.raw_txs.copy() + txs = api.txs.copy() + + async with OldTaskGroup() as group: + api.raw_txs = {} + api.txs = {} + await group.spawn(mempool.keep_synchronized, event) + for cur_size in range(mempool_size_target): + api.raw_txs = {hash: raw_txs[hash] for hash in api.ordered_adds[:cur_size]} + api.txs = {hash: txs[hash] for hash in api.ordered_adds[:cur_size]} + async with ignore_after(max(mempool.refresh_secs * 2, 0.5)): + await event.wait() + recent_txs = await mempool.get_recently_added_txs(count=10) + start_idx = max(0, cur_size-10) + recent_adds = api.ordered_adds[start_idx:cur_size] + assert [tx.hash for tx in recent_txs][::-1] == recent_adds + await group.cancel_remaining() + + @pytest.mark.asyncio async def test_dropped_txs(caplog): api = API() From 81f6809d805d0279b27ebff49590501580b75c71 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Tue, 12 May 2026 12:43:09 +0000 Subject: [PATCH 16/27] lrucache: sync with electrum client module (add type hints) --- src/electrumx/lib/lrucache.py | 45 +++++++++++++++++++---------------- 1 file changed, 24 insertions(+), 21 deletions(-) diff --git a/src/electrumx/lib/lrucache.py b/src/electrumx/lib/lrucache.py index e2651bd5e..c3eea8029 100644 --- a/src/electrumx/lib/lrucache.py +++ b/src/electrumx/lib/lrucache.py @@ -26,6 +26,7 @@ import collections import collections.abc +from typing import TypeVar, Dict class _DefaultSize: @@ -42,19 +43,21 @@ def pop(self, _): return 1 -class Cache(collections.abc.MutableMapping): +_KT = TypeVar("_KT") +_VT = TypeVar("_VT") +class Cache(collections.abc.MutableMapping[_KT, _VT]): """Mutable mapping to serve as a simple cache or cache base class.""" __marker = object() __size = _DefaultSize() - def __init__(self, maxsize, getsizeof=None): + def __init__(self, maxsize: int, getsizeof=None): if getsizeof: self.getsizeof = getsizeof if self.getsizeof is not Cache.getsizeof: self.__size = dict() - self.__data = dict() + self.__data = dict() # type: Dict[_KT, _VT] self.__currsize = 0 self.__maxsize = maxsize # these can be used externally to keep track of cache hit statistics: @@ -70,13 +73,13 @@ def __repr__(self): self.__currsize, ) - def __getitem__(self, key): + def __getitem__(self, key: _KT) -> _VT: try: return self.__data[key] except KeyError: return self.__missing__(key) - def __setitem__(self, key, value): + def __setitem__(self, key: _KT, value: _VT) -> None: maxsize = self.__maxsize size = self.getsizeof(value) if size > maxsize: @@ -92,15 +95,15 @@ def __setitem__(self, key, value): self.__size[key] = size self.__currsize += diffsize - def __delitem__(self, key): + def __delitem__(self, key: _KT) -> None: size = self.__size.pop(key) del self.__data[key] self.__currsize -= size - def __contains__(self, key): + def __contains__(self, key: _KT) -> bool: return key in self.__data - def __missing__(self, key): + def __missing__(self, key: _KT): raise KeyError(key) def __iter__(self): @@ -109,13 +112,13 @@ def __iter__(self): def __len__(self): return len(self.__data) - def get(self, key, default=None): + def get(self, key: _KT, default: _VT = None) -> _VT | None: if key in self: return self[key] else: return default - def pop(self, key, default=__marker): + def pop(self, key: _KT, default=__marker) -> _VT: if key in self: value = self[key] del self[key] @@ -125,7 +128,7 @@ def pop(self, key, default=__marker): value = default return value - def setdefault(self, key, default=None): + def setdefault(self, key: _KT, default: _VT = None) -> _VT | None: if key in self: value = self[key] else: @@ -133,43 +136,43 @@ def setdefault(self, key, default=None): return value @property - def maxsize(self): + def maxsize(self) -> int: """The maximum size of the cache.""" return self.__maxsize @property - def currsize(self): + def currsize(self) -> int: """The current size of the cache.""" return self.__currsize @staticmethod - def getsizeof(value): + def getsizeof(value) -> int: """Return the size of a cache element's value.""" return 1 -class LRUCache(Cache): +class LRUCache(Cache[_KT, _VT]): """Least Recently Used (LRU) cache implementation.""" - def __init__(self, maxsize, getsizeof=None): + def __init__(self, maxsize: int, getsizeof=None): Cache.__init__(self, maxsize, getsizeof) self.__order = collections.OrderedDict() - def __getitem__(self, key, cache_getitem=Cache.__getitem__): + def __getitem__(self, key: _KT, cache_getitem=Cache.__getitem__) -> _VT | None: value = cache_getitem(self, key) if key in self: # __missing__ may not store item self.__update(key) return value - def __setitem__(self, key, value, cache_setitem=Cache.__setitem__): + def __setitem__(self, key: _KT, value, cache_setitem=Cache.__setitem__) -> None: cache_setitem(self, key, value) self.__update(key) - def __delitem__(self, key, cache_delitem=Cache.__delitem__): + def __delitem__(self, key: _KT, cache_delitem=Cache.__delitem__) -> None: cache_delitem(self, key) del self.__order[key] - def popitem(self): + def popitem(self) -> tuple[_KT, _VT]: """Remove and return the `(key, value)` pair least recently used.""" try: key = next(iter(self.__order)) @@ -178,7 +181,7 @@ def popitem(self): else: return (key, self.pop(key)) - def __update(self, key): + def __update(self, key: _KT) -> None: try: self.__order.move_to_end(key) except KeyError: From 60c84c75e20bf2e387e343f0ae4d73e17a2b40fa Mon Sep 17 00:00:00 2001 From: SomberNight Date: Tue, 12 May 2026 14:10:38 +0000 Subject: [PATCH 17/27] introduce txid_rev, txid_hum terms; add type hints all over txid_rev: txid in same endianness as prevouts are serialised in a raw tx txid_hum: txid in endianness used for human-readable display typically (these are reverses of each other) --- contrib/query.py | 2 +- src/electrumx/lib/coins.py | 112 +++---- src/electrumx/lib/hash.py | 12 +- src/electrumx/lib/peer.py | 3 +- src/electrumx/lib/tx.py | 148 ++++----- src/electrumx/lib/tx_axe.py | 4 +- src/electrumx/lib/tx_dash.py | 4 +- src/electrumx/server/block_processor.py | 122 ++++---- src/electrumx/server/controller.py | 2 +- src/electrumx/server/daemon.py | 58 ++-- src/electrumx/server/db.py | 120 ++++---- src/electrumx/server/env.py | 6 +- src/electrumx/server/mempool.py | 149 ++++----- src/electrumx/server/peers.py | 4 +- src/electrumx/server/session.py | 392 ++++++++++++------------ tests/lib/test_tx_zcoin.py | 2 +- tests/server/test_daemon.py | 4 +- tests/server/test_mempool.py | 37 +-- tests/test_blocks.py | 6 +- tests/test_transactions.py | 12 +- 20 files changed, 610 insertions(+), 589 deletions(-) diff --git a/contrib/query.py b/contrib/query.py index c27ddbd3a..077fd6cac 100755 --- a/contrib/query.py +++ b/contrib/query.py @@ -81,7 +81,7 @@ async def query(args): n = None utxos = await db.all_utxos(hashX) for n, utxo in enumerate(utxos, start=1): - print(f'UTXO #{n:,d}: tx_hash {hash_to_hex_str(utxo.tx_hash)} ' + print(f'UTXO #{n:,d}: tx_hash {hash_to_hex_str(utxo.txid_rev)} ' f'tx_pos {utxo.tx_pos:,d} height {utxo.height:,d} ' f'value {utxo.value:,d}') if n == limit: diff --git a/src/electrumx/lib/coins.py b/src/electrumx/lib/coins.py index 1a08854ca..89e82f153 100644 --- a/src/electrumx/lib/coins.py +++ b/src/electrumx/lib/coins.py @@ -35,7 +35,7 @@ from decimal import Decimal from functools import partial from hashlib import sha256 -from typing import Sequence, Tuple, Optional +from typing import Sequence, Tuple, Optional, Type import electrumx.lib.util as util from electrumx.lib.hash import Base58, double_sha256, hash_to_hex_str @@ -89,7 +89,7 @@ class Coin: ENCODE_CHECK = Base58.encode_check DECODE_CHECK = Base58.decode_check GENESIS_HASH = ('000000000019d6689c085ae165831e93' - '4ff763ae46a2a6c172b3f1b60a8ce26f') + '4ff763ae46a2a6c172b3f1b60a8ce26f') # 'hum' byte-order GENESIS_ACTIVATION = 100_000_000 # max byte-size of single jsonrpc message @@ -123,7 +123,7 @@ class Coin: TX_PER_BLOCK: int # and from that height onwards, we guess this many txs per block @classmethod - def lookup_coin_class(cls, name, net): + def lookup_coin_class(cls, name: str, net: str) -> Type['Coin']: '''Return a coin class given name and network. Raise an exception if unrecognised.''' @@ -144,7 +144,7 @@ def lookup_coin_class(cls, name, net): raise CoinError(f'unknown coin {name} and network {net} combination') @classmethod - def sanitize_url(cls, url): + def sanitize_url(cls, url: str): # Remove surrounding ws and trailing /s url = url.strip().rstrip('/') match = cls.RPC_URL_REGEX.match(url) @@ -157,19 +157,19 @@ def sanitize_url(cls, url): return url + '/' @classmethod - def max_fetch_blocks(cls, height): + def max_fetch_blocks(cls, height: int) -> int: if height < 130000: return 1000 return 100 @classmethod - def genesis_block(cls, block): + def genesis_block(cls, block: bytes) -> bytes: '''Check the Genesis block is the right one for this coin. Return the block less its unspendable coinbase. ''' header = cls.block_header(block, 0) - header_hex_hash = hash_to_hex_str(cls.header_hash(header)) + header_hex_hash = hash_to_hex_str(cls.header_hash_rev(header)) if header_hex_hash != cls.GENESIS_HASH: raise CoinError(f'genesis block has hash {header_hex_hash} ' f'expected {cls.GENESIS_HASH}') @@ -235,17 +235,17 @@ def privkey_WIF(cls, privkey_bytes, compressed): return cls.ENCODE_CHECK(payload) @classmethod - def header_hash(cls, header: bytes) -> bytes: - '''Given a header return hash''' + def header_hash_rev(cls, header: bytes) -> bytes: + '''Given a header return hash. (reverse of human-readable blockhash)''' return double_sha256(header) @classmethod - def header_prevhash(cls, header): - '''Given a header return previous hash''' + def header_prevhash_rev(cls, header: bytes) -> bytes: + '''Given a header return previous hash. (reverse of human-readable blockhash)''' return header[4:36] @classmethod - def static_header_offset(cls, height): + def static_header_offset(cls, height: int) -> int: '''Given a header height return its offset in the headers file. If header sizes change at some point, this is the only code @@ -254,25 +254,25 @@ def static_header_offset(cls, height): return height * cls.BASIC_HEADER_SIZE @classmethod - def static_header_len(cls, height): + def static_header_len(cls, height: int) -> int: '''Given a header height return its length.''' return (cls.static_header_offset(height + 1) - cls.static_header_offset(height)) @classmethod - def block_header(cls, block, height): + def block_header(cls, block: bytes, height: int) -> bytes: '''Returns the block header given a block and its height.''' return block[:cls.static_header_len(height)] @classmethod - def block(cls, raw_block, height): + def block(cls, raw_block: bytes, height: int) -> 'Block': '''Return a Block namedtuple given a raw block and its height.''' header = cls.block_header(raw_block, height) txs = cls.DESERIALIZER(raw_block, start=len(header)).read_tx_block() return Block(raw_block, header, txs) @classmethod - def decimal_value(cls, value): + def decimal_value(cls, value: int) -> Decimal: '''Return the number of standard coin units as a Decimal given a quantity of smallest units. @@ -469,7 +469,7 @@ class AuxPowMixin: DEFAULT_MAX_SEND = 10000000 @classmethod - def header_hash(cls, header): + def header_hash_rev(cls, header): '''Given a header return hash''' return double_sha256(header[:cls.BASIC_HEADER_SIZE]) @@ -731,7 +731,7 @@ class Verge(Coin): DESERIALIZER = lib_tx.DeserializerVerge @classmethod - def header_hash(cls, header): + def header_hash_rev(cls, header): '''Given a header return the hash.''' import scrypt return scrypt.hash(header, header, 1024, 1, 1, 32) @@ -815,7 +815,7 @@ class BitcoinGold(EquihashMixin, BitcoinMixin, Coin): ] @classmethod - def header_hash(cls, header): + def header_hash_rev(cls, header): '''Given a header return hash''' height, = util.unpack_le_uint32_from(header, 68) if height >= cls.FORK_HEIGHT: @@ -923,7 +923,7 @@ def block_header(cls, block, height): return block[:cls.static_header_len(height)] @classmethod - def header_hash(cls, header): + def header_hash_rev(cls, header): '''Given a header return hash''' return double_sha256(header[:cls.BASIC_HEADER_SIZE]) @@ -1335,7 +1335,7 @@ class Dash(Coin): DESERIALIZER = lib_tx_dash.DeserializerDash @classmethod - def header_hash(cls, header): + def header_hash_rev(cls, header): '''Given a header return the hash.''' import dash_hash return dash_hash.getPoWHash(header) @@ -1742,7 +1742,7 @@ class DeepOnion(Coin): PEERS = [] @classmethod - def header_hash(cls, header): + def header_hash_rev(cls, header): ''' Given a header return the hash for DeepOnion. Need to download `x13_hash` module @@ -1832,7 +1832,7 @@ def genesis_block(cls, block): return header + b'\0' @classmethod - def header_hash(cls, header): + def header_hash_rev(cls, header): '''Given a header return the hash.''' return cls.HEADER_HASH(header) @@ -2051,7 +2051,7 @@ class Bitzeny(Coin): REORG_LIMIT = 1000 @classmethod - def header_hash(cls, header): + def header_hash_rev(cls, header): '''Given a header return the hash.''' import zny_yespower_0_5 return zny_yespower_0_5.getPoWHash(header) @@ -2098,7 +2098,7 @@ class Denarius(Coin): TX_PER_BLOCK = 4000 @classmethod - def header_hash(cls, header): + def header_hash_rev(cls, header): '''Given a header return the hash.''' import tribushashm return tribushashm.getPoWHash(header) @@ -2137,7 +2137,7 @@ class Sibcoin(Dash): PEERS = [] @classmethod - def header_hash(cls, header): + def header_hash_rev(cls, header): ''' Given a header return the hash for sibcoin. Need to download `x11_gost_hash` module @@ -2322,7 +2322,7 @@ class BitcoinAtom(Coin): REORG_LIMIT = 5000 @classmethod - def header_hash(cls, header): + def header_hash_rev(cls, header): '''Given a header return hash''' header_to_be_hashed = header[:cls.BASIC_HEADER_SIZE] # New block header format has some extra flags in the end @@ -2375,7 +2375,7 @@ class Decred(Coin): RPC_PORT = 9109 @classmethod - def header_hash(cls, header): + def header_hash_rev(cls, header): '''Given a header return the hash.''' return cls.HEADER_HASH(header) @@ -2429,7 +2429,7 @@ class Axe(Dash): PEERS = [] @classmethod - def header_hash(cls, header): + def header_hash_rev(cls, header): ''' Given a header return the hash for AXE. Need to download `axe_hash` module @@ -2487,7 +2487,7 @@ class Xuez(Coin): PEERS = [] @classmethod - def header_hash(cls, header): + def header_hash_rev(cls, header): ''' Given a header return the hash for Xuez. Need to download `xevan_hash` module @@ -2541,10 +2541,10 @@ def static_header_offset(cls, height): return height * cls.BASIC_HEADER_SIZE @classmethod - def header_hash(cls, header): + def header_hash_rev(cls, header): version, = util.unpack_le_uint32_from(header) if version >= 4: - return super().header_hash(header) + return super().header_hash_rev(header) else: import quark_hash return quark_hash.getPoWHash(header) @@ -2575,7 +2575,7 @@ class Pac(Coin): RELAY_FEE = 0.00001 @classmethod - def header_hash(cls, header): + def header_hash_rev(cls, header): '''Given a header return the hash.''' import dash_hash return dash_hash.getPoWHash(header) @@ -2644,7 +2644,7 @@ def block_header(cls, block, height): return block[:sz] @classmethod - def header_hash(cls, header): + def header_hash_rev(cls, header): sz = cls.BASIC_HEADER_SIZE if cls.is_mtp(header): sz += cls.MTP_HEADER_EXTRA_SIZE @@ -2687,7 +2687,7 @@ class Polis(Coin): DAEMON = daemon.DashDaemon @classmethod - def header_hash(cls, header): + def header_hash_rev(cls, header): '''Given a header return the hash.''' import dash_hash return dash_hash.getPoWHash(header) @@ -2715,7 +2715,7 @@ class MNPCoin(Coin): DAEMON = daemon.DashDaemon @classmethod - def header_hash(cls, header): + def header_hash_rev(cls, header): '''Given a header return the hash.''' import quark_hash return quark_hash.getPoWHash(header) @@ -2756,10 +2756,10 @@ def static_header_offset(cls, height): return height * cls.BASIC_HEADER_SIZE @classmethod - def header_hash(cls, header): + def header_hash_rev(cls, header): version, = util.unpack_le_uint32_from(header) if version >= 5: - return super().header_hash(header) + return super().header_hash_rev(header) else: import quark_hash return quark_hash.getPoWHash(header) @@ -2820,7 +2820,7 @@ def grshash(data): return groestlcoin_hash.getHash(data, len(data)) @classmethod - def header_hash(cls, header): + def header_hash_rev(cls, header): '''Given a header return the hash.''' return cls.grshash(header) @@ -2900,11 +2900,11 @@ def static_header_len(cls, height): return cls.BASIC_HEADER_SIZE @classmethod - def header_hash(cls, header): + def header_hash_rev(cls, header): '''Given a header return the hash.''' version, = struct.unpack('= cls.ZEROCOIN_BLOCK_VERSION: - return super().header_hash(header) + return super().header_hash_rev(header) else: import quark_hash return quark_hash.getPoWHash(header) @@ -2956,7 +2956,7 @@ class Bitg(Coin): SESSIONCLS = DashElectrumX @classmethod - def header_hash(cls, header): + def header_hash_rev(cls, header): '''Given a header return the hash.''' import quark_hash return quark_hash.getPoWHash(header) @@ -2994,7 +2994,7 @@ class EXOS(Coin): DESERIALIZER = lib_tx.DeserializerTxTime @classmethod - def header_hash(cls, header): + def header_hash_rev(cls, header): version, = util.unpack_le_uint32_from(header) if version > 2: @@ -3016,7 +3016,7 @@ class EXOSTestnet(EXOS): RPC_PORT = 14561 @classmethod - def header_hash(cls, header): + def header_hash_rev(cls, header): version, = util.unpack_le_uint32_from(header) if version > 2: @@ -3049,7 +3049,7 @@ class SmartCash(Coin): SESSIONCLS = SmartCashElectrumX @classmethod - def header_hash(cls, header): + def header_hash_rev(cls, header): '''Given a header return the hash.''' return cls.HEADER_HASH(header) @@ -3121,7 +3121,7 @@ class BitcoinPlus(Coin): REORG_LIMIT = 2000 @classmethod - def header_hash(cls, header): + def header_hash_rev(cls, header): '''Given a header return the hash.''' import x13_hash return x13_hash.getPoWHash(header) @@ -3183,7 +3183,7 @@ class Bitsend(Coin): ] @classmethod - def header_hash(cls, header): + def header_hash_rev(cls, header): timestamp, = util.unpack_le_uint32_from(header, 68) if timestamp > cls.XEVAN_TIMESTAMP: import xevan_hash @@ -3195,7 +3195,7 @@ def header_hash(cls, header): @classmethod def genesis_block(cls, block): header = cls.block_header(block, 0) - header_hex_hash = hash_to_hex_str(cls.header_hash(header)) + header_hex_hash = hash_to_hex_str(cls.header_hash_rev(header)) if header_hex_hash != cls.GENESIS_HASH: raise CoinError(f'genesis block has hash {header_hex_hash} ' f'expected {cls.GENESIS_HASH}') @@ -3236,7 +3236,7 @@ def static_header_offset(cls, height): return result @classmethod - def header_hash(cls, header): + def header_hash_rev(cls, header): '''Given a header return the hash.''' timestamp = util.unpack_le_uint32_from(header, 68)[0] assert cls.KAWPOW_ACTIVATION_TIME > 0 @@ -3307,7 +3307,7 @@ class Bolivarcoin(Coin): DAEMON = daemon.DashDaemon @classmethod - def header_hash(cls, header): + def header_hash_rev(cls, header): '''Given a header return the hash.''' import dash_hash return dash_hash.getPoWHash(header) @@ -3332,7 +3332,7 @@ class Onixcoin(Coin): DAEMON = daemon.DashDaemon @classmethod - def header_hash(cls, header): + def header_hash_rev(cls, header): '''Given a header return the hash.''' import dash_hash return dash_hash.getPoWHash(header) @@ -3357,7 +3357,7 @@ class Electra(Coin): DESERIALIZER = lib_tx.DeserializerElectra @classmethod - def header_hash(cls, header): + def header_hash_rev(cls, header): '''Given a header return the hash.''' version, = util.unpack_le_uint32_from(header) import nist5_hash @@ -3385,7 +3385,7 @@ class ECCoin(Coin): RPC_PORT = 19119 @classmethod - def header_hash(cls, header): + def header_hash_rev(cls, header): # Requires OpenSSL 1.1.0+ from hashlib import scrypt return scrypt(header, salt=header, n=1024, r=1, p=1, dklen=32) @@ -3494,7 +3494,7 @@ class Simplicity(Coin): DESERIALIZER = lib_tx.DeserializerSimplicity @classmethod - def header_hash(cls, header): + def header_hash_rev(cls, header): '''Given a header return the hash.''' version, = util.unpack_le_uint32_from(header) @@ -3555,7 +3555,7 @@ class Myce(Coin): DESERIALIZER = lib_tx.DeserializerSimplicity @classmethod - def header_hash(cls, header): + def header_hash_rev(cls, header): '''Given a header return the hash.''' version, = util.unpack_le_uint32_from(header) @@ -3586,7 +3586,7 @@ class Navcoin(Coin): REORG_LIMIT = 1000 @classmethod - def header_hash(cls, header): + def header_hash_rev(cls, header): if int.from_bytes(header[:4], "little") > 6: return double_sha256(header) else: diff --git a/src/electrumx/lib/hash.py b/src/electrumx/lib/hash.py index fe11d9137..66d1db3f7 100644 --- a/src/electrumx/lib/hash.py +++ b/src/electrumx/lib/hash.py @@ -37,26 +37,26 @@ HASHX_LEN = 11 -def sha256(x): +def sha256(x: bytes) -> bytes: '''Simple wrapper of hashlib sha256.''' return _sha256(x).digest() -def double_sha256(x): +def double_sha256(x: bytes) -> bytes: '''SHA-256 of SHA-256, as used extensively in bitcoin.''' return sha256(sha256(x)) -def hash_to_hex_str(x): - '''Convert a big-endian binary hash to displayed hex string. +def hash_to_hex_str(x: bytes) -> str: + '''Convert a big-endian binary hash to displayed hex string. (rev-to-hum) Display form of a binary hash is reversed and converted to hex. ''' return bytes(reversed(x)).hex() -def hex_str_to_hash(x): - '''Convert a displayed hex string to a binary hash.''' +def hex_str_to_hash(x: str) -> bytes: + '''Convert a displayed hex string to a binary hash. (hum-to-rev)''' return bytes(reversed(hex_to_bytes(x))) diff --git a/src/electrumx/lib/peer.py b/src/electrumx/lib/peer.py index bc1391dd5..2c5c849df 100644 --- a/src/electrumx/lib/peer.py +++ b/src/electrumx/lib/peer.py @@ -27,6 +27,7 @@ from ipaddress import ip_address, IPv4Address, IPv6Address, IPv4Network, IPv6Network from socket import AF_INET, AF_INET6 +from typing import Any from aiorpcx import is_valid_hostname from electrumx.lib.util import cachedproperty, protocol_tuple, version_string @@ -71,7 +72,7 @@ def __init__(self, host, features, source='unknown', ip_addr=None, self.other_port_pairs = set() @classmethod - def peers_from_features(cls, features, source): + def peers_from_features(cls, features: dict[str, Any] | Any, source): peers = [] if isinstance(features, dict): hosts = features.get('hosts') diff --git a/src/electrumx/lib/tx.py b/src/electrumx/lib/tx.py index d0ff1758d..e1ed9f9c3 100644 --- a/src/electrumx/lib/tx.py +++ b/src/electrumx/lib/tx.py @@ -58,10 +58,10 @@ class Tx: inputs: Sequence['TxInput'] outputs: Sequence['TxOutput'] locktime: int - # The hashes need to be reversed for human display; - # for efficiency we process it in the natural serialized order. - txid: bytes - wtxid: bytes + # For efficiency, we store/process tx hashes in the natural serialized order ("rev"). + # This is the reverse of the usual human display byteorder ("hum"). + txid_rev: bytes + wtxid_rev: bytes def serialize(self): return b''.join(( @@ -77,24 +77,24 @@ def serialize(self): @dataclass(kw_only=True, slots=True) class TxInput: '''Class representing a transaction input.''' - prev_hash: bytes + prev_txid_rev: bytes prev_idx: int script: bytes sequence: int def __str__(self): script = self.script.hex() - prev_hash = hash_to_hex_str(self.prev_hash) + prev_hash = hash_to_hex_str(self.prev_txid_rev) return (f"Input({prev_hash}, {self.prev_idx:d}, script={script}, " f"sequence={self.sequence:d})") def is_generation(self): '''Test if an input is generation/coinbase like''' - return self.prev_idx == MINUS_1 and self.prev_hash == ZERO + return self.prev_idx == MINUS_1 and self.prev_txid_rev == ZERO def serialize(self): return b''.join(( - self.prev_hash, + self.prev_txid_rev, pack_le_uint32(self.prev_idx), pack_varbytes(self.script), pack_le_uint32(self.sequence), @@ -116,7 +116,7 @@ def serialize(self): @dataclass(kw_only=True, slots=True) class TXOSpendStatus: prev_height: Optional[int] # block height TXO is mined at. None if the outpoint never existed - spender_txhash: bytes = None + spender_txid_rev: bytes = None spender_height: int = None @@ -130,7 +130,7 @@ class Deserializer: millions of times during sync. ''' - TX_HASH_FN = staticmethod(double_sha256) + TX_HASH_FN = staticmethod(double_sha256) # returns txid_rev def __init__(self, binary, start=0): assert isinstance(binary, bytes) @@ -146,12 +146,12 @@ def read_tx(self) -> Tx: inputs=self._read_inputs(), outputs=self._read_outputs(), locktime=self._read_le_uint32(), - txid=None, - wtxid=None, + txid_rev=None, + wtxid_rev=None, ) txid = self.TX_HASH_FN(self.binary[start:self.cursor]) - tx.txid = txid - tx.wtxid = txid + tx.txid_rev = txid + tx.wtxid_rev = txid return tx def read_tx_and_vsize(self) -> Tuple[Tx, int]: @@ -175,7 +175,7 @@ def _read_inputs(self): def _read_input(self): return TxInput( - prev_hash=self._read_nbytes(32), + prev_txid_rev=self._read_nbytes(32), prev_idx=self._read_le_uint32(), script=self._read_varbytes(), sequence=self._read_le_uint32(), @@ -310,8 +310,8 @@ def _read_tx_parts(self) -> Tuple[Tx, int]: outputs=outputs, witness=witness, locktime=locktime, - txid=txid, - wtxid=wtxid), vsize + txid_rev=txid, + wtxid_rev=wtxid), vsize def read_tx(self): return self._read_tx_parts()[0] @@ -418,8 +418,8 @@ def _read_tx_parts(self): outputs=outputs, witness=witness, locktime=locktime, - txid=txid, - wtxid=wtxid), vsize + txid_rev=txid, + wtxid_rev=wtxid), vsize def read_tx(self): return self._read_tx_parts()[0] @@ -509,8 +509,8 @@ def read_tx(self): inputs=self._read_inputs(), outputs=self._read_outputs(), locktime=self._read_le_uint32(), - txid=None, - wtxid=None, + txid_rev=None, + wtxid_rev=None, ) if is_overwinter_v3 or is_sapling_v4: @@ -537,7 +537,7 @@ def read_tx(self): if is_sapling_v4 and has_shielded: self.cursor += 64 # bindingSig - base_tx.txid = base_tx.wtxid = self.TX_HASH_FN(self.binary[orig_start:self.cursor]) + base_tx.txid_rev = base_tx.wtxid_rev = self.TX_HASH_FN(self.binary[orig_start:self.cursor]) return base_tx @@ -579,8 +579,8 @@ def read_tx(self): inputs=self._read_inputs(), outputs=self._read_outputs(), locktime=self._read_le_uint32(), - txid=None, - wtxid=None, + txid_rev=None, + wtxid_rev=None, ) if version >= 3: # >= sapling @@ -594,7 +594,7 @@ def read_tx(self): if (tx_type > 0): self.cursor += 2 # extraPayload - base_tx.txid = base_tx.wtxid = self.TX_HASH_FN(self.binary[orig_start:self.cursor]) + base_tx.txid_rev = base_tx.wtxid_rev = self.TX_HASH_FN(self.binary[orig_start:self.cursor]) return base_tx @@ -614,10 +614,10 @@ def read_tx(self): inputs=self._read_inputs(), outputs=self._read_outputs(), locktime=self._read_le_uint32(), - txid=None, - wtxid=None, + txid_rev=None, + wtxid_rev=None, ) - tx.txid = tx.wtxid = self.TX_HASH_FN(self.binary[orig_start:self.cursor]) + tx.txid_rev = tx.wtxid_rev = self.TX_HASH_FN(self.binary[orig_start:self.cursor]) return tx @@ -679,8 +679,8 @@ def _read_tx_parts(self): outputs=outputs, witness=witness, locktime=locktime, - txid=txid, - wtxid=wtxid, + txid_rev=txid, + wtxid_rev=wtxid, ) return tx, vsize @@ -714,8 +714,8 @@ def read_tx_no_segwit(self): inputs=inputs, outputs=outputs, locktime=locktime, - txid=txid, - wtxid=txid, + txid_rev=txid, + wtxid_rev=txid, ) def _read_tx_parts(self): @@ -765,8 +765,8 @@ def _read_tx_parts(self): outputs=outputs, witness=witness, locktime=locktime, - txid=txid, - wtxid=wtxid, + txid_rev=txid, + wtxid_rev=wtxid, ) return tx, vsize @@ -802,8 +802,8 @@ def read_tx(self): outputs=outputs, locktime=locktime, txcomment=txcomment, - txid=txid, - wtxid=txid, + txid_rev=txid, + wtxid_rev=txid, ) @staticmethod @@ -856,8 +856,8 @@ def read_tx(self): inputs=self._read_inputs(), outputs=self._read_outputs(), locktime=self._read_le_uint32(), - txid=None, - wtxid=None, + txid_rev=None, + wtxid_rev=None, ) else: tx = Tx( @@ -865,10 +865,10 @@ def read_tx(self): inputs=self._read_inputs(), outputs=self._read_outputs(), locktime=self._read_le_uint32(), - txid=None, - wtxid=None, + txid_rev=None, + wtxid_rev=None, ) - tx.txid = tx.wtxid = self.TX_HASH_FN(self.binary[orig_start:self.cursor]) + tx.txid_rev = tx.wtxid_rev = self.TX_HASH_FN(self.binary[orig_start:self.cursor]) return tx @@ -891,8 +891,8 @@ def read_tx(self): inputs=inputs, outputs=outputs, locktime=locktime, - txid=txid, - wtxid=txid, + txid_rev=txid, + wtxid_rev=txid, ) @@ -912,8 +912,8 @@ def read_tx(self): inputs=inputs, outputs=outputs, locktime=locktime, - txid=txid, - wtxid=txid, + txid_rev=txid, + wtxid_rev=txid, ) @@ -1020,7 +1020,7 @@ class DeserializerTokenPay(DeserializerTxTime): def _read_input(self): txin = TxInputTokenPay( - prev_hash=self._read_nbytes(32), + prev_txid_rev=self._read_nbytes(32), prev_idx=self._read_le_uint32(), script=self._read_varbytes(), sequence=self._read_le_uint32(), @@ -1139,7 +1139,7 @@ def _read_tx_parts(self): # TxSerializeNoWitness << 16 == 0x10000 no_witness_header = pack_le_uint32(0x10000 | (version & 0xffff)) prefix_tx = no_witness_header + self.binary[start+4:end_prefix] - tx_hash = self.blake256(prefix_tx) + txid_rev = self.blake256(prefix_tx) return TxDcr( version=version, @@ -1148,8 +1148,8 @@ def _read_tx_parts(self): locktime=locktime, expiry=expiry, witness=witness, - txid=tx_hash, - wtxid=tx_hash, + txid_rev=txid_rev, + wtxid_rev=txid_rev, ), self.cursor - start @@ -1182,8 +1182,8 @@ def read_tx(self): inputs=self._read_inputs(), outputs=self._read_outputs(), locktime=self._read_le_uint32(), - txid=None, - wtxid=None, + txid_rev=None, + wtxid_rev=None, ) else: tx = TxBitcoinDiamond( @@ -1192,11 +1192,11 @@ def read_tx(self): inputs=self._read_inputs(), outputs=self._read_outputs(), locktime=self._read_le_uint32(), - txid=None, - wtxid=None, + txid_rev=None, + wtxid_rev=None, ) txid = self.TX_HASH_FN(self.binary[start:self.cursor]) - tx.txid = tx.wtxid = txid + tx.txid_rev = tx.wtxid_rev = txid return tx def _get_version(self): @@ -1264,8 +1264,8 @@ def _read_tx_parts(self): outputs=outputs, witness=witness, locktime=locktime, - txid=txid, - wtxid=wtxid), vsize + txid_rev=txid, + wtxid_rev=wtxid), vsize else: return TxSegWit( version=version, @@ -1275,8 +1275,8 @@ def _read_tx_parts(self): outputs=outputs, witness=witness, locktime=locktime, - txid=txid, - wtxid=wtxid), vsize + txid_rev=txid, + wtxid_rev=wtxid), vsize def read_tx(self): '''Return a (Deserialized TX, TX_HASH) pair. @@ -1305,8 +1305,8 @@ def read_tx(self): inputs=self._read_inputs(), outputs=self._read_outputs(), locktime=self._read_le_uint32(), - txid=None, - wtxid=None, + txid_rev=None, + wtxid_rev=None, ) else: tx = Tx( @@ -1314,10 +1314,10 @@ def read_tx(self): inputs=self._read_inputs(), outputs=self._read_outputs(), locktime=self._read_le_uint32(), - txid=None, - wtxid=None, + txid_rev=None, + wtxid_rev=None, ) - tx.txid = tx.wtxid = self.TX_HASH_FN(self.binary[orig_start:self.cursor]) + tx.txid_rev = tx.wtxid_rev = self.TX_HASH_FN(self.binary[orig_start:self.cursor]) return tx @@ -1332,32 +1332,32 @@ def read_tx(self): inputs=self._read_inputs(), outputs=self._read_outputs(), locktime=self._read_le_uint32(), - txid=None, - wtxid=None, + txid_rev=None, + wtxid_rev=None, ) if tx_version > 1: self.cursor += 32 - tx.txid = tx.wtxid = self.TX_HASH_FN(self.binary[orig_start:self.cursor]) + tx.txid_rev = tx.wtxid_rev = self.TX_HASH_FN(self.binary[orig_start:self.cursor]) return tx class DeserializerZcoin(Deserializer): def _read_input(self): tx_input = TxInput( - prev_hash=self._read_nbytes(32), + prev_txid_rev=self._read_nbytes(32), prev_idx=self._read_le_uint32(), script=self._read_varbytes(), sequence=self._read_le_uint32(), ) - if tx_input.prev_idx == MINUS_1 and tx_input.prev_hash == ZERO: + if tx_input.prev_idx == MINUS_1 and tx_input.prev_txid_rev == ZERO: return tx_input if tx_input.script[0] == 0xc4: # This is a Sigma spend - mimic a generation tx return TxInput( - prev_hash=ZERO, + prev_txid_rev=ZERO, prev_idx=MINUS_1, script=tx_input.script, sequence=tx_input.sequence @@ -1418,8 +1418,8 @@ def read_tx(self): inputs=self._read_inputs(), outputs=self._read_outputs(), locktime=self._read_le_uint32(), - txid=None, - wtxid=None, + txid_rev=None, + wtxid_rev=None, ) else: tx = Tx( @@ -1427,10 +1427,10 @@ def read_tx(self): inputs=self._read_inputs(), outputs=self._read_outputs(), locktime=self._read_le_uint32(), - txid=None, - wtxid=None, + txid_rev=None, + wtxid_rev=None, ) - tx.txid = tx.wtxid = self.TX_HASH_FN(self.binary[orig_start:self.cursor]) + tx.txid_rev = tx.wtxid_rev = self.TX_HASH_FN(self.binary[orig_start:self.cursor]) return tx diff --git a/src/electrumx/lib/tx_axe.py b/src/electrumx/lib/tx_axe.py index be96f95a5..180172cc6 100644 --- a/src/electrumx/lib/tx_axe.py +++ b/src/electrumx/lib/tx_axe.py @@ -483,7 +483,7 @@ def read_tx(self): locktime=locktime, tx_type=tx_type, extra_payload=extra_payload, - txid=txid, - wtxid=txid, + txid_rev=txid, + wtxid_rev=txid, ) return tx diff --git a/src/electrumx/lib/tx_dash.py b/src/electrumx/lib/tx_dash.py index 8c3a11106..b2b682917 100644 --- a/src/electrumx/lib/tx_dash.py +++ b/src/electrumx/lib/tx_dash.py @@ -436,7 +436,7 @@ def read_tx(self): locktime=locktime, tx_type=tx_type, extra_payload=extra_payload, - txid=txid, - wtxid=txid, + txid_rev=txid, + wtxid_rev=txid, ) return tx diff --git a/src/electrumx/server/block_processor.py b/src/electrumx/server/block_processor.py index d49df6ac7..9ef72eee8 100644 --- a/src/electrumx/server/block_processor.py +++ b/src/electrumx/server/block_processor.py @@ -196,8 +196,8 @@ def __init__(self, env: 'Env', db: DB, daemon: Daemon, notifications: 'Notificat self._caught_up_event = None # Caches of unflushed items. - self.headers = [] - self.tx_hashes = [] # type: List[bytes] + self.headers = [] # type: list[bytes] + self.txids_rev = [] # type: List[bytes] self.undo_infos = [] # type: List[Tuple[Sequence[bytes], int]] # UTXO cache @@ -232,8 +232,8 @@ async def check_and_advance_blocks(self, raw_blocks: Sequence[bytes]) -> None: blocks = [self.coin.block(raw_block, first + n) for n, raw_block in enumerate(raw_blocks)] headers = [block.header for block in blocks] - hprevs = [self.coin.header_prevhash(h) for h in headers] - chain = [self.tip] + [self.coin.header_hash(h) for h in headers[:-1]] + hprevs = [self.coin.header_prevhash_rev(h) for h in headers] + chain = [self.tip] + [self.coin.header_hash_rev(h) for h in headers[:-1]] if hprevs == chain: start = time.monotonic() @@ -264,7 +264,7 @@ async def check_and_advance_blocks(self, raw_blocks: Sequence[bytes]) -> None: 'resetting the prefetcher') await self.prefetcher.reset_height(self.height) - async def reorg_chain(self, count=None): + async def reorg_chain(self, count=None) -> Sequence[bytes]: '''Handle a chain reorganisation. Count is the number of blocks to simulate a reorg, or None for @@ -275,14 +275,14 @@ async def reorg_chain(self, count=None): self.logger.info(f'faking a reorg of {count:,d} blocks') await self.flush(True) - async def get_raw_blocks(last_height, hex_hashes) -> Sequence[bytes]: - heights = range(last_height, last_height - len(hex_hashes), -1) + async def get_raw_blocks(last_height: int, bhashes_hum: Sequence[str]) -> Sequence[bytes]: + heights = range(last_height, last_height - len(bhashes_hum), -1) try: blocks = [self.db.read_raw_block(height) for height in heights] self.logger.info(f'read {len(blocks)} blocks from disk') return blocks except FileNotFoundError: - return await self.daemon.raw_blocks(hex_hashes) + return await self.daemon.raw_blocks(bhashes_hum) def flush_backup(): # self.touched_hashxs can include other addresses which is @@ -290,11 +290,11 @@ def flush_backup(): self.touched_hashxs.discard(None) self.db.flush_backup(self.flush_data(), self.touched_hashxs) - _start, last, hashes = await self.reorg_hashes(count) + _start, last, bhashes_rev = await self.reorg_hashes(count) # Reverse and convert to hex strings. - hashes = [hash_to_hex_str(hash) for hash in reversed(hashes)] - for hex_hashes in chunks(hashes, 50): - raw_blocks = await get_raw_blocks(last, hex_hashes) + bhashes_hum = [hash_to_hex_str(bhash_rev) for bhash_rev in reversed(bhashes_rev)] + for bhash_hum in chunks(bhashes_hum, 50): + raw_blocks = await get_raw_blocks(last, bhash_hum) await self.run_in_thread_with_lock(self.backup_blocks, raw_blocks) await self.run_in_thread_with_lock(flush_backup) last -= len(raw_blocks) @@ -302,7 +302,7 @@ def flush_backup(): self.backed_up_event.set() self.backed_up_event.clear() - async def reorg_hashes(self, count): + async def reorg_hashes(self, count: Optional[int]) -> tuple[int, int, Sequence[bytes]]: '''Return a pair (start, last, hashes) of blocks to back up during a reorg. @@ -315,9 +315,9 @@ async def reorg_hashes(self, count): self.logger.info(f'chain was reorganised replacing {count:,d} ' f'block{s} at heights {start:,d}-{last:,d}') - return start, last, await self.db.fs_block_hashes(start, count) + return start, last, await self.db.fs_block_hashes_rev(start, count) - async def calc_reorg_range(self, count): + async def calc_reorg_range(self, count: Optional[int]) -> tuple[int, int]: '''Calculate the reorg range''' def diff_pos(hashes1, hashes2): @@ -326,17 +326,17 @@ def diff_pos(hashes1, hashes2): for n, (hash1, hash2) in enumerate(zip(hashes1, hashes2)): if hash1 != hash2: return n - return len(hashes) + return len(bhashes_rev) if count is None: # A real reorg start = self.height - 1 count = 1 while start > 0: - hashes = await self.db.fs_block_hashes(start, count) - hex_hashes = [hash_to_hex_str(hash) for hash in hashes] - d_hex_hashes = await self.daemon.block_hex_hashes(start, count) - n = diff_pos(hex_hashes, d_hex_hashes) + bhashes_rev = await self.db.fs_block_hashes_rev(start, count) + bhashes_hum = [hash_to_hex_str(bhash) for bhash in bhashes_rev] + d_bhashes_hum = await self.daemon.block_hex_hashes(start, count) + n = diff_pos(bhashes_hum, d_bhashes_hum) if n > 0: start += n break @@ -367,7 +367,7 @@ def flush_data(self): height=self.height, tx_count=self.tx_count, headers=self.headers, - block_tx_hashes=self.tx_hashes, + block_txids_rev=self.txids_rev, undo_infos=self.undo_infos, adds=self.utxo_cache, deletes=self.db_deletes, @@ -401,10 +401,10 @@ def check_cache_size(self) -> Optional[bool]: db_deletes_size = len(self.db_deletes) * 57 hist_cache_size = self.db.history.unflushed_memsize() # Roughly ntxs * 32 + nblocks * 42 - tx_hash_size = ((self.tx_count - self.db.fs_tx_count) * 32 + txids_size = ((self.tx_count - self.db.fs_tx_count) * 32 + (self.height - self.db.fs_height) * 42) utxo_MB = (db_deletes_size + utxo_cache_size) // one_MB - hist_MB = (hist_cache_size + tx_hash_size) // one_MB + hist_MB = (hist_cache_size + txids_size) // one_MB self.logger.info(f'our height: {self.height:,d} daemon: ' f'{self.daemon.cached_height():,d} ' @@ -429,7 +429,7 @@ def advance_blocks(self, blocks: Sequence['Block']): for block in blocks: height += 1 - header_hash = coin.header_hash(block.header) + header_hash = coin.header_hash_rev(block.header) is_unspendable = (is_unspendable_genesis if height >= genesis_activation else is_unspendable_legacy) undo_info = self.advance_txs(block.transactions, is_unspendable) @@ -441,7 +441,7 @@ def advance_blocks(self, blocks: Sequence['Block']): headers = [block.header for block in blocks] self.height = height self.headers += headers - self.tip = self.coin.header_hash(headers[-1]) + self.tip = self.coin.header_hash_rev(headers[-1]) self.tip_advanced_event.set() self.tip_advanced_event.clear() @@ -450,7 +450,7 @@ def advance_txs( txs: Sequence[Tx], is_unspendable: Callable[[bytes], bool], ) -> Sequence[bytes]: - self.tx_hashes.append(b''.join(tx.txid for tx in txs)) + self.txids_rev.append(b''.join(tx.txid_rev for tx in txs)) # Use local vars for speed in the loops undo_info = [] @@ -468,7 +468,7 @@ def advance_txs( _pack_txnum = pack_txnum for tx in txs: - tx_hash = tx.txid + txid_rev = tx.txid_rev hashXs = [] append_hashX = hashXs.append tx_numb = _pack_txnum(tx_num) @@ -477,10 +477,10 @@ def advance_txs( for txin in tx.inputs: if txin.is_generation(): continue - cache_value = spend_utxo(txin.prev_hash, txin.prev_idx) + cache_value = spend_utxo(txin.prev_txid_rev, txin.prev_idx) undo_info_append(cache_value) append_hashX(cache_value[:HASHX_LEN]) - prevout_tuple = (txin.prev_hash, txin.prev_idx) + prevout_tuple = (txin.prev_txid_rev, txin.prev_idx) add_touched_outpoint(prevout_tuple) # Add the new UTXOs @@ -492,9 +492,9 @@ def advance_txs( # Get the hashX hashX = script_hashX(txout.pk_script) append_hashX(hashX) - put_utxo(tx_hash + to_le_uint32(idx)[:TXOUTIDX_LEN], + put_utxo(txid_rev + to_le_uint32(idx)[:TXOUTIDX_LEN], hashX + tx_numb + to_le_uint64(txout.value)) - add_touched_outpoint((tx_hash, idx)) + add_touched_outpoint((txid_rev, idx)) append_hashXs(hashXs) update_touched_hashxs(hashXs) @@ -521,13 +521,13 @@ def backup_blocks(self, raw_blocks: Sequence[bytes]): for raw_block in raw_blocks: # Check and update self.tip block = coin.block(raw_block, self.height) - header_hash = coin.header_hash(block.header) + header_hash = coin.header_hash_rev(block.header) if header_hash != self.tip: raise ChainError( f'backup block {hash_to_hex_str(header_hash)} not tip ' f'{hash_to_hex_str(self.tip)} at height {self.height:,d}' ) - self.tip = coin.header_prevhash(block.header) + self.tip = coin.header_prevhash_rev(block.header) is_unspendable = (is_unspendable_genesis if self.height >= genesis_activation else is_unspendable_legacy) self.backup_txs(block.transactions, is_unspendable) @@ -558,7 +558,7 @@ def backup_txs( undo_entry_len = HASHX_LEN + TXNUM_LEN + 8 for tx in reversed(txs): - tx_hash = tx.txid + txid_rev = tx.txid_rev for idx, txout in enumerate(tx.outputs): # Spend the TX outputs. Be careful with unspendable # outputs - we didn't save those in the first place. @@ -566,10 +566,10 @@ def backup_txs( continue # Get the hashX - cache_value = spend_utxo(tx_hash, idx) + cache_value = spend_utxo(txid_rev, idx) hashX = cache_value[:HASHX_LEN] add_touched_hashx(hashX) - add_touched_outpoint((tx_hash, idx)) + add_touched_outpoint((txid_rev, idx)) # Restore the inputs for txin in reversed(tx.inputs): @@ -577,11 +577,11 @@ def backup_txs( continue n -= undo_entry_len undo_item = undo_info[n:n + undo_entry_len] - prevout = txin.prev_hash + pack_le_uint32(txin.prev_idx)[:TXOUTIDX_LEN] + prevout = txin.prev_txid_rev + pack_le_uint32(txin.prev_idx)[:TXOUTIDX_LEN] put_utxo(prevout, undo_item) hashX = undo_item[:HASHX_LEN] add_touched_hashx(hashX) - add_touched_outpoint((txin.prev_hash, txin.prev_idx)) + add_touched_outpoint((txin.prev_txid_rev, txin.prev_idx)) assert n == 0 self.tx_count -= len(txs) @@ -619,7 +619,7 @@ def backup_txs( 1. Given an address be able to list its UTXOs and their values so its balance can be efficiently computed. - 2. When processing transactions, for each prevout spent - a (tx_hash, + 2. When processing transactions, for each prevout spent - a (txid_rev, idx) pair - we have to be able to remove it from the DB. To send notifications to clients we also need to know any address it paid to. @@ -640,7 +640,7 @@ def backup_txs( collision rate is low (<0.1%). ''' - def spend_utxo(self, tx_hash: bytes, tx_idx: int) -> bytes: + def spend_utxo(self, txid_rev: bytes, tx_idx: int) -> bytes: '''Spend a UTXO and return (hashX + tx_num + value_sats). If the UTXO is not in the cache it must be on disk. We store @@ -649,7 +649,7 @@ def spend_utxo(self, tx_hash: bytes, tx_idx: int) -> bytes: ''' # Fast track is it being in the cache idx_packed = pack_le_uint32(tx_idx)[:TXOUTIDX_LEN] - cache_value = self.utxo_cache.pop(tx_hash + idx_packed, None) + cache_value = self.utxo_cache.pop(txid_rev + idx_packed, None) if cache_value: return cache_value @@ -657,7 +657,7 @@ def spend_utxo(self, tx_hash: bytes, tx_idx: int) -> bytes: # Key: b'h' + compressed_tx_hash + tx_idx + tx_num # Value: hashX - prefix = b'h' + tx_hash[:COMP_TXID_LEN] + idx_packed + prefix = b'h' + txid_rev[:COMP_TXID_LEN] + idx_packed candidates = {db_key: hashX for db_key, hashX in self.db.utxo_db.iterator(prefix=prefix)} @@ -666,8 +666,8 @@ def spend_utxo(self, tx_hash: bytes, tx_idx: int) -> bytes: if len(candidates) > 1: tx_num = unpack_txnum(tx_num_packed) - hash, _height = self.db.fs_tx_hash(tx_num) - if hash != tx_hash: + hash, _height = self.db.fs_txid_rev(tx_num) + if hash != txid_rev: assert hash is not None # Should always be found continue @@ -681,10 +681,10 @@ def spend_utxo(self, tx_hash: bytes, tx_idx: int) -> bytes: self.db_deletes.append(udb_key) return hashX + tx_num_packed + utxo_value_packed - raise ChainError(f'UTXO {hash_to_hex_str(tx_hash)} / {tx_idx:,d} not ' + raise ChainError(f'UTXO {hash_to_hex_str(txid_rev)} / {tx_idx:,d} not ' f'found in "h" table') - async def _process_prefetched_blocks(self): + async def _process_prefetched_blocks(self) -> None: '''Loop forever processing blocks as they arrive.''' while True: if self.height == self.daemon.cached_height(): @@ -700,7 +700,7 @@ async def _process_prefetched_blocks(self): blocks = self.prefetcher.get_prefetched_blocks() await self.check_and_advance_blocks(blocks) - async def _first_caught_up(self): + async def _first_caught_up(self) -> None: self.logger.info(f'caught up to height {self.height}') # Flush everything but with first_sync->False state. first_sync = self.db.first_sync @@ -712,7 +712,7 @@ async def _first_caught_up(self): # Reopen for serving await self.db.open_for_serving() - async def _first_open_dbs(self): + async def _first_open_dbs(self) -> None: await self.db.open_for_sync() self.height = self.db.db_height self.tip = self.db.db_tip @@ -720,7 +720,7 @@ async def _first_open_dbs(self): # --- External API - async def fetch_and_process_blocks(self, caught_up_event): + async def fetch_and_process_blocks(self, caught_up_event: asyncio.Event) -> None: '''Fetch, process and index blocks from the daemon. Sets caught_up_event when first caught up. Flushes to disk @@ -744,7 +744,7 @@ async def fetch_and_process_blocks(self, caught_up_event): self.logger.info('flushing to DB for a clean shutdown...') await self.flush(True) - def force_chain_reorg(self, count): + def force_chain_reorg(self, count: int) -> bool: '''Force a reorg of the given number of blocks. Returns True if a reorg is queued, false if not caught up. @@ -800,7 +800,7 @@ def advance_txs(self, txs, is_unspendable): class LTORBlockProcessor(BlockProcessor): def advance_txs(self, txs, is_unspendable): - self.tx_hashes.append(b''.join(tx.txid for tx in txs)) + self.txids_rev.append(b''.join(tx.txid_rev for tx in txs)) # Use local vars for speed in the loops undo_info = [] @@ -819,7 +819,7 @@ def advance_txs(self, txs, is_unspendable): # Add the new UTXOs for tx, hashXs in zip(txs, hashXs_by_tx): - tx_hash = tx.txid + txid_rev = tx.txid_rev add_hashXs = hashXs.add tx_numb = _pack_txnum(tx_num) @@ -831,9 +831,9 @@ def advance_txs(self, txs, is_unspendable): # Get the hashX hashX = script_hashX(txout.pk_script) add_hashXs(hashX) - put_utxo(tx_hash + to_le_uint32(idx)[:TXOUTIDX_LEN], + put_utxo(txid_rev + to_le_uint32(idx)[:TXOUTIDX_LEN], hashX + tx_numb + to_le_uint64(txout.value)) - add_touched_outpoint((tx_hash, idx)) + add_touched_outpoint((txid_rev, idx)) tx_num += 1 # Spend the inputs @@ -843,10 +843,10 @@ def advance_txs(self, txs, is_unspendable): for txin in tx.inputs: if txin.is_generation(): continue - cache_value = spend_utxo(txin.prev_hash, txin.prev_idx) + cache_value = spend_utxo(txin.prev_txid_rev, txin.prev_idx) undo_info_append(cache_value) add_hashXs(cache_value[:HASHX_LEN]) - prevout_tuple = (txin.prev_hash, txin.prev_idx) + prevout_tuple = (txin.prev_txid_rev, txin.prev_idx) add_touched_outpoint(prevout_tuple) # Update touched set for notifications @@ -882,17 +882,17 @@ def backup_txs(self, txs, is_unspendable): if txin.is_generation(): continue undo_item = undo_info[n:n + undo_entry_len] - prevout = txin.prev_hash + pack_le_uint32(txin.prev_idx)[:TXOUTIDX_LEN] + prevout = txin.prev_txid_rev + pack_le_uint32(txin.prev_idx)[:TXOUTIDX_LEN] put_utxo(prevout, undo_item) add_touched_hashx(undo_item[:HASHX_LEN]) - add_touched_outpoint((txin.prev_hash, txin.prev_idx)) + add_touched_outpoint((txin.prev_txid_rev, txin.prev_idx)) n += undo_entry_len assert n == len(undo_info) # Remove tx outputs made in this block, by spending them. for tx in txs: - tx_hash = tx.txid + txid_rev = tx.txid_rev for idx, txout in enumerate(tx.outputs): # Spend the TX outputs. Be careful with unspendable # outputs - we didn't save those in the first place. @@ -900,9 +900,9 @@ def backup_txs(self, txs, is_unspendable): continue # Get the hashX - cache_value = spend_utxo(tx_hash, idx) + cache_value = spend_utxo(txid_rev, idx) hashX = cache_value[:HASHX_LEN] add_touched_hashx(hashX) - add_touched_outpoint((tx_hash, idx)) + add_touched_outpoint((txid_rev, idx)) self.tx_count -= len(txs) diff --git a/src/electrumx/server/controller.py b/src/electrumx/server/controller.py index d09420397..b65b561c4 100644 --- a/src/electrumx/server/controller.py +++ b/src/electrumx/server/controller.py @@ -149,7 +149,7 @@ def get_db_height(): notifications.height = daemon.height notifications.db_height = get_db_height notifications.cached_height = daemon.cached_height - notifications.mempool_hashes = daemon.mempool_hashes + notifications.mempool_txids_hum = daemon.mempool_txids_hum notifications.raw_transactions = daemon.getrawtransactions notifications.lookup_utxos = db.lookup_utxos MemPoolAPI.register(Notifications) diff --git a/src/electrumx/server/daemon.py b/src/electrumx/server/daemon.py index a1a6cea01..8c95cdcb8 100644 --- a/src/electrumx/server/daemon.py +++ b/src/electrumx/server/daemon.py @@ -13,7 +13,7 @@ import time from calendar import timegm from struct import pack -from typing import TYPE_CHECKING, Type, Sequence, Any +from typing import TYPE_CHECKING, Type, Sequence, Any, Iterable, Optional import aiohttp from aiorpcx import JSONRPC @@ -122,7 +122,7 @@ async def check_daemon_indexes(self): # Should we raise? Not raising allows syncing a fresh bitcoind and e-x "in parallel". self.logger.warning(f"bitcoind required index {req_index!r} is still syncing!") - def set_url(self, url): + def set_url(self, url: str) -> None: '''Set the URLS to the given list, and switch to the first one.''' urls = url.split(',') urls = [self.coin.sanitize_url(url) for url in urls] @@ -133,7 +133,7 @@ def set_url(self, url): self.url_index = 0 self.urls = urls - def current_url(self): + def current_url(self) -> str: '''Returns the current daemon URL.''' return self.urls[self.url_index] @@ -255,7 +255,7 @@ def processor(result): return await self._send(payload, processor) return [] - async def _is_rpc_available(self, method): + async def _is_rpc_available(self, method: str) -> bool: '''Return whether given RPC method is available in the daemon. Results are cached and the daemon will generally not be queried with @@ -272,23 +272,23 @@ async def _is_rpc_available(self, method): self.available_rpcs[method] = available return available - async def block_hex_hashes(self, first, count) -> Sequence[str]: - '''Return the hex hashes of count block starting at height first.''' + async def block_hex_hashes(self, first: int, count: int) -> Sequence[str]: + '''Return the hex hashes (hum) of count block starting at height first.''' params_iterable = ((h, ) for h in range(first, first + count)) return await self._send_vector('getblockhash', params_iterable) - async def deserialised_block(self, hex_hash): + async def deserialised_block(self, bhash_hum: str) -> dict: '''Return the deserialised block with the given hex hash.''' - return await self._send_single('getblock', (hex_hash, True)) + return await self._send_single('getblock', (bhash_hum, True)) - async def raw_blocks(self, hex_hashes: Sequence[str]) -> Sequence[bytes]: + async def raw_blocks(self, bhashes_hum: Sequence[str]) -> Sequence[bytes]: '''Return the raw binary blocks with the given hex hashes.''' - params_iterable = ((h, False) for h in hex_hashes) + params_iterable = ((h, False) for h in bhashes_hum) blocks = await self._send_vector('getblock', params_iterable) # Convert hex string to bytes return [hex_to_bytes(block) for block in blocks] - async def mempool_hashes(self): + async def mempool_txids_hum(self) -> Sequence[str]: '''Update our record of the daemon's mempool hashes.''' return await self._send_single('getrawmempool') @@ -351,30 +351,30 @@ async def mempool_info(self) -> dict[str, float]: 'incrementalrelayfee': mempool_info['incrementalrelayfee'], } - async def getrawtransaction(self, hex_hash, verbose=False): + async def getrawtransaction(self, txid_hum: str, verbose=False): '''Return the serialized raw transaction with the given hash.''' # Cast to int because some coin daemons are old and require it return await self._send_single('getrawtransaction', - (hex_hash, int(verbose))) + (txid_hum, int(verbose))) - async def getrawtransactions(self, hex_hashes, replace_errs=True): + async def getrawtransactions(self, txids_hum: Iterable[str], replace_errs=True) -> Sequence[bytes | None]: '''Return the serialized raw transactions with the given hashes. Replaces errors with None by default.''' - params_iterable = ((hex_hash, 0) for hex_hash in hex_hashes) + params_iterable = ((txid_hum, 0) for txid_hum in txids_hum) txs = await self._send_vector('getrawtransaction', params_iterable, replace_errs=replace_errs) # Convert hex strings to bytes return [hex_to_bytes(tx) if tx else None for tx in txs] - async def gettxspendingprevout(self, prev_txhash: str, txout_idx: int) -> dict[str, Any]: + async def gettxspendingprevout(self, prev_txid_hum: str, txout_idx: int) -> dict[str, Any]: """Query the daemon to find (if any) the spender of given outpoint.""" - outpoints = [{"txid": prev_txhash, "vout": txout_idx}, ] + outpoints = [{"txid": prev_txid_hum, "vout": txout_idx}, ] options = {"mempool_only": False} tx_items = await self._send_single('gettxspendingprevout', (outpoints, options)) return tx_items[0] - async def broadcast_transaction(self, raw_tx): + async def broadcast_transaction(self, raw_tx: str) -> str: '''Broadcast a transaction to the network.''' return await self._send_single('sendrawtransaction', (raw_tx, )) @@ -388,12 +388,12 @@ async def testmempoolaccept(self, raw_txs: Sequence[str]): """ return await self._send_single('testmempoolaccept', (raw_txs, )) - async def height(self): + async def height(self) -> int: '''Query the daemon for its current height.''' self._height = await self._send_single('getblockcount') return self._height - def cached_height(self): + def cached_height(self) -> Optional[int]: '''Return the cached daemon height. If the daemon has not been queried yet this returns None.''' @@ -438,9 +438,9 @@ class LegacyRPCDaemon(Daemon): as in the underlying blockchain but it is good enough for our indexing purposes.''' - async def raw_blocks(self, hex_hashes): + async def raw_blocks(self, bhashes_hum): '''Return the raw binary blocks with the given hex hashes.''' - params_iterable = ((h, ) for h in hex_hashes) + params_iterable = ((h, ) for h in bhashes_hum) block_info = await self._send_vector('getblock', params_iterable) blocks = [] @@ -494,10 +494,10 @@ class FakeEstimateLegacyRPCDaemon(LegacyRPCDaemon, FakeEstimateFeeDaemon): class DecredDaemon(Daemon): - async def raw_blocks(self, hex_hashes): + async def raw_blocks(self, bhashes_hum): '''Return the raw binary blocks with the given hex hashes.''' - params_iterable = ((h, False) for h in hex_hashes) + params_iterable = ((h, False) for h in bhashes_hum) blocks = await self._send_vector('getblock', params_iterable) raw_blocks = [] @@ -512,7 +512,7 @@ async def raw_blocks(self, hex_hashes): valid_tx_tree[prev] = self.is_valid_tx_tree(votebits) processed_raw_blocks = [] - for hash, raw_block in zip(hex_hashes, raw_blocks): + for hash, raw_block in zip(bhashes_hum, raw_blocks): if hash in valid_tx_tree: is_valid = valid_tx_tree[hash] else: @@ -559,8 +559,8 @@ async def height(self): self._height = height return height - async def mempool_hashes(self): - mempool = await super().mempool_hashes() + async def mempool_txids_hum(self): + mempool = await super().mempool_txids_hum() # Add current tip transactions to the 'fake' mempool. real_height = await self._send_single('getblockcount') tip_hash = await self._send_single('getblockhash', (real_height,)) @@ -612,9 +612,9 @@ def strip_mtp_data(self, raw_block): raw_block[self.coin.MTP_HEADER_DATA_END*2:] return raw_block - async def raw_blocks(self, hex_hashes): + async def raw_blocks(self, bhashes_hum): '''Return the raw binary blocks with the given hex hashes.''' - params_iterable = ((h, False) for h in hex_hashes) + params_iterable = ((h, False) for h in bhashes_hum) blocks = await self._send_vector('getblock', params_iterable) # Convert hex string to bytes return [hex_to_bytes(self.strip_mtp_data(block)) for block in blocks] diff --git a/src/electrumx/server/db.py b/src/electrumx/server/db.py index 747b9906c..864419527 100644 --- a/src/electrumx/server/db.py +++ b/src/electrumx/server/db.py @@ -42,7 +42,7 @@ class UTXO: tx_num: int # index of tx in chain order tx_pos: int # tx output idx - tx_hash: bytes # txid + txid_rev: bytes | None height: int # block height value: int # in satoshis @@ -52,7 +52,7 @@ class FlushData: height = attr.ib() tx_count = attr.ib() headers = attr.ib() - block_tx_hashes = attr.ib() # type: List[bytes] + block_txids_rev = attr.ib() # type: List[bytes] # The following are flushed to the UTXO DB if undo_infos is not None undo_infos = attr.ib() # type: List[Tuple[Sequence[bytes], int]] adds = attr.ib() # type: Dict[bytes, bytes] # txid+out_idx -> hashX+tx_num+value_sats @@ -124,20 +124,20 @@ def __init__(self, env: 'Env'): # Header merkle cache self.merkle = Merkle() - self.header_mc = MerkleCache(self.merkle, self.fs_block_hashes) + self.header_mc = MerkleCache(self.merkle, self.fs_block_hashes_rev) # on-disk: raw block headers in chain order self.headers_file = util.LogicalFile('meta/headers', 2, 16000000) # on-disk: cumulative number of txs at the end of height N self.tx_counts = None # type: Optional[array] # in-memory self.tx_counts_file = util.LogicalFile('meta/txcounts', 2, 2000000) - # on-disk: 32 byte txids in chain order, allows (tx_num -> txid) map + # on-disk: 32 byte txids in chain order, allows (tx_num -> txid_rev) map self.hashes_file = util.LogicalFile('meta/hashes', 4, 16000000) if not self.coin.STATIC_BLOCK_HEADERS: self.headers_offsets_file = util.LogicalFile( 'meta/headers_offsets', 2, 16000000) - # in-memory: (block_hash -> block_height) map + # in-memory: (block_hash_rev -> block_height) map self.bhash_to_bheight = None # type: Optional[dict[bytes, int]] async def _read_tx_counts(self): @@ -183,7 +183,7 @@ async def _open_dbs(self, *, for_sync: bool): # Now prepare in-memory structures. # - Read TX counts (requires meta directory) await self._read_tx_counts() - # - (block_hash -> block_height) map + # - (block_hash_rev -> block_height) map await self._prep_bhash_to_bheight_map() async def open_for_sync(self): @@ -225,7 +225,7 @@ async def _prep_bhash_to_bheight_map(self) -> None: return self.bhash_to_bheight = {} count = self.db_height + 1 - block_hashes = await self.fs_block_hashes(0, count) + block_hashes = await self.fs_block_hashes_rev(0, count) if len(block_hashes) != count: raise Exception( f"failed to prep bhash_to_bheight. " @@ -234,9 +234,9 @@ async def _prep_bhash_to_bheight_map(self) -> None: self.bhash_to_bheight[bhash] = bheight # note: for new blocks, the block_processor will keep the map up-to-date - def get_blockheight_from_blockhash(self, block_hash: str) -> Optional[int]: - bhash = hex_str_to_hash(block_hash) - return self.bhash_to_bheight.get(bhash, None) + def get_blockheight_from_blockhash(self, block_hash_hum: str) -> Optional[int]: + bhash_rev = hex_str_to_hash(block_hash_hum) + return self.bhash_to_bheight.get(bhash_rev, None) # Flushing def assert_flushed(self, flush_data): @@ -245,7 +245,7 @@ def assert_flushed(self, flush_data): assert flush_data.height == self.fs_height == self.db_height assert flush_data.tip == self.db_tip assert not flush_data.headers - assert not flush_data.block_tx_hashes + assert not flush_data.block_txids_rev assert not flush_data.adds assert not flush_data.deletes assert not flush_data.undo_infos @@ -303,15 +303,15 @@ def flush_fs(self, flush_data): ''' prior_tx_count = (self.tx_counts[self.fs_height] if self.fs_height >= 0 else 0) - assert len(flush_data.block_tx_hashes) == len(flush_data.headers) + assert len(flush_data.block_txids_rev) == len(flush_data.headers) assert flush_data.height == self.fs_height + len(flush_data.headers) assert flush_data.tx_count == (self.tx_counts[-1] if self.tx_counts else 0) assert len(self.tx_counts) == flush_data.height + 1 - hashes = b''.join(flush_data.block_tx_hashes) - flush_data.block_tx_hashes.clear() - assert len(hashes) % 32 == 0 - assert len(hashes) // 32 == flush_data.tx_count - prior_tx_count + txids_rev = b''.join(flush_data.block_txids_rev) + flush_data.block_txids_rev.clear() + assert len(txids_rev) % 32 == 0 + assert len(txids_rev) // 32 == flush_data.tx_count - prior_tx_count # Write the headers, tx counts, and tx hashes start_time = time.monotonic() @@ -325,7 +325,7 @@ def flush_fs(self, flush_data): self.tx_counts_file.write(offset, self.tx_counts[height_start:].tobytes()) offset = prior_tx_count * 32 - self.hashes_file.write(offset, hashes) + self.hashes_file.write(offset, txids_rev) self.fs_height = flush_data.height self.fs_tx_count = flush_data.tx_count @@ -393,7 +393,7 @@ def flush_state(self, batch): def flush_backup(self, flush_data: FlushData, touched_hashxs): '''Like flush_dbs() but when backing up. All UTXOs are flushed.''' assert not flush_data.headers - assert not flush_data.block_tx_hashes + assert not flush_data.block_txids_rev assert flush_data.height < self.db_height self.history.assert_flushed() @@ -475,19 +475,19 @@ def read_headers(): return await run_in_thread(read_headers) - def fs_tx_hash(self, tx_num: int) -> Tuple[Optional[bytes], int]: - '''Return a pair (tx_hash, tx_height) for the given tx number. + def fs_txid_rev(self, tx_num: int) -> Tuple[Optional[bytes], int]: + '''Return a pair (txid_rev, tx_height) for the given tx number. If the tx_height is not on disk, returns (None, tx_height).''' tx_height = bisect_right(self.tx_counts, tx_num) if tx_height > self.db_height: - tx_hash = None + txid_rev = None else: - tx_hash = self.hashes_file.read(tx_num * 32, 32) - return tx_hash, tx_height + txid_rev = self.hashes_file.read(tx_num * 32, 32) + return txid_rev, tx_height - def fs_tx_hashes_at_blockheight(self, block_height): - '''Return a list of tx_hashes at given block height, + def fs_txids_rev_at_blockheight(self, block_height: int) -> Sequence[bytes]: + '''Return a list of txids_rev at given block height, in the same order as in the block. ''' if block_height > self.db_height: @@ -498,14 +498,14 @@ def fs_tx_hashes_at_blockheight(self, block_height): else: first_tx_num = 0 num_txs_in_block = self.tx_counts[block_height] - first_tx_num - tx_hashes = self.hashes_file.read(first_tx_num * 32, num_txs_in_block * 32) - assert num_txs_in_block == len(tx_hashes) // 32 - return [tx_hashes[idx * 32: (idx+1) * 32] for idx in range(num_txs_in_block)] + txids_rev = self.hashes_file.read(first_tx_num * 32, num_txs_in_block * 32) + assert num_txs_in_block == len(txids_rev) // 32 + return [txids_rev[idx * 32: (idx+1) * 32] for idx in range(num_txs_in_block)] - async def tx_hashes_at_blockheight(self, block_height): - return await run_in_thread(self.fs_tx_hashes_at_blockheight, block_height) + async def txids_rev_at_blockheight(self, block_height: int) -> Sequence[bytes]: + return await run_in_thread(self.fs_txids_rev_at_blockheight, block_height) - async def fs_block_hashes(self, height, count): + async def fs_block_hashes_rev(self, height: int, count: int) -> Sequence[bytes]: headers_concat, headers_count = await self.read_headers(height, count) if headers_count != count: raise self.DBError(f'only got {headers_count:,d} headers starting ' @@ -517,10 +517,10 @@ async def fs_block_hashes(self, height, count): headers.append(headers_concat[offset:offset + hlen]) offset += hlen - return [self.coin.header_hash(header) for header in headers] + return [self.coin.header_hash_rev(header) for header in headers] - async def limited_history(self, hashX, *, limit=1000): - '''Return an unpruned, sorted list of (tx_hash, height) tuples of + async def limited_history(self, hashX: bytes, *, limit: int = 1000) -> Sequence[tuple[bytes, int]]: + '''Return an unpruned, sorted list of (txid_rev, height) tuples of confirmed transactions that touched the address, earliest in the blockchain first. Includes both spending and receiving transactions. By default returns at most 1000 entries. Set @@ -528,12 +528,12 @@ async def limited_history(self, hashX, *, limit=1000): ''' def read_history(): tx_nums = list(self.history.get_txnums(hashX, limit)) - fs_tx_hash = self.fs_tx_hash - return [fs_tx_hash(tx_num) for tx_num in tx_nums] + fs_txid_rev = self.fs_txid_rev + return [fs_txid_rev(tx_num) for tx_num in tx_nums] while True: history = await run_in_thread(read_history) - if all(hash is not None for hash, height in history): + if all(txid_rev is not None for txid_rev, height in history): return history self.logger.warning(f'limited_history: tx hash ' f'not found (reorg?), retrying...') @@ -541,7 +541,7 @@ def read_history(): # -- Undo information - def min_undo_height(self, max_height): + def min_undo_height(self, max_height: int) -> int: '''Returns a height from which we should store undo info.''' return max_height - self.env.reorg_limit + 1 @@ -549,30 +549,30 @@ def undo_key(self, height: int) -> bytes: '''DB key for undo information at the given height.''' return b'U' + pack_be_uint32(height) - def read_undo_info(self, height): + def read_undo_info(self, height: int) -> bytes: '''Read undo information from a file for the current height.''' return self.utxo_db.get(self.undo_key(height)) def flush_undo_infos( self, batch_put, undo_infos: Sequence[Tuple[Sequence[bytes], int]] - ): + ) -> None: '''undo_infos is a list of (undo_info, height) pairs.''' for undo_info, height in undo_infos: batch_put(self.undo_key(height), b''.join(undo_info)) - def raw_block_prefix(self): + def raw_block_prefix(self) -> str: return 'meta/block' - def raw_block_path(self, height): + def raw_block_path(self, height: int) -> str: return f'{self.raw_block_prefix()}{height:d}' - def read_raw_block(self, height): + def read_raw_block(self, height: int) -> bytes: '''Returns a raw block read from disk. Raises FileNotFoundError if the block isn't on-disk.''' with util.open_file(self.raw_block_path(height)) as f: return f.read(-1) - def write_raw_block(self, block, height): + def write_raw_block(self, block: bytes, height: int) -> None: '''Write a raw block to disk.''' with util.open_truncate(self.raw_block_path(height)) as f: f.write(block) @@ -583,7 +583,7 @@ def write_raw_block(self, block, height): except FileNotFoundError: pass - def clear_excess_undo_info(self): + def clear_excess_undo_info(self) -> None: '''Clear excess undo info. Only most recent N are kept.''' prefix = b'U' min_height = self.min_undo_height(self.db_height) @@ -615,7 +615,7 @@ def clear_excess_undo_info(self): # -- UTXO database - def read_utxo_state(self): + def read_utxo_state(self) -> None: state = self.utxo_db.get(b'\0state') if not state: self.db_height = -1 @@ -669,7 +669,7 @@ def read_utxo_state(self): f'sync time so far: {util.formatted_time(self.wall_time)}' ) - def write_utxo_state(self, batch): + def write_utxo_state(self, batch) -> None: '''Write (UTXO) state to the batch.''' state = { 'genesis': self.coin.GENESIS_HASH, @@ -682,7 +682,7 @@ def write_utxo_state(self, batch): } batch.put(b'\0state', repr(state).encode()) - async def all_utxos(self, hashX): + async def all_utxos(self, hashX: bytes) -> Sequence[UTXO]: '''Return all UTXOs for an address sorted in no particular order.''' def read_utxos(): utxos = [] @@ -694,47 +694,49 @@ def read_utxos(): txout_idx, = unpack_le_uint32(db_key[-TXNUM_LEN-TXOUTIDX_LEN:-TXNUM_LEN] + TXOUTIDX_PADDING) tx_num = unpack_txnum(db_key[-TXNUM_LEN:]) value, = unpack_le_uint64(db_value) - tx_hash, height = self.fs_tx_hash(tx_num) - utxos_append(UTXO(tx_num, txout_idx, tx_hash, height, value)) + txid_rev, height = self.fs_txid_rev(tx_num) + utxos_append(UTXO(tx_num, txout_idx, txid_rev, height, value)) return utxos while True: utxos = await run_in_thread(read_utxos) - if all(utxo.tx_hash is not None for utxo in utxos): + if all(utxo.txid_rev is not None for utxo in utxos): return utxos self.logger.warning(f'all_utxos: tx hash not ' f'found (reorg?), retrying...') await sleep(0.25) - async def lookup_utxos(self, prevouts): + async def lookup_utxos(self, prevouts: Sequence[tuple[bytes, int]]) -> Sequence[Optional[tuple[bytes, int]]]: '''For each prevout, lookup it up in the DB and return a (hashX, value) pair or None if not found. Used by the mempool code. ''' - def lookup_hashXs(): + def lookup_hashXs() -> Sequence[tuple[Optional[bytes], Optional[bytes]]]: '''Return (hashX, suffix) pairs, or None if not found, for each prevout. ''' - def lookup_hashX(tx_hash, tx_idx): + def lookup_hashX(txid_rev: bytes, tx_idx: int) -> tuple[Optional[bytes], Optional[bytes]]: idx_packed = pack_le_uint32(tx_idx)[:TXOUTIDX_LEN] # Key: b'h' + compressed_tx_hash + tx_idx + tx_num # Value: hashX - prefix = b'h' + tx_hash[:COMP_TXID_LEN] + idx_packed + prefix = b'h' + txid_rev[:COMP_TXID_LEN] + idx_packed # Find which entry, if any, the TX_HASH matches. for db_key, hashX in self.utxo_db.iterator(prefix=prefix): tx_num_packed = db_key[-TXNUM_LEN:] tx_num = unpack_txnum(tx_num_packed) - hash, _height = self.fs_tx_hash(tx_num) - if hash == tx_hash: + hash, _height = self.fs_txid_rev(tx_num) + if hash == txid_rev: return hashX, idx_packed + tx_num_packed return None, None return [lookup_hashX(*prevout) for prevout in prevouts] - def lookup_utxos(hashX_pairs): - def lookup_utxo(hashX, suffix): + def lookup_utxos( + hashX_pairs: Sequence[tuple[Optional[bytes], Optional[bytes]]], + ) -> Sequence[Optional[tuple[bytes, int]]]: + def lookup_utxo(hashX: bytes, suffix: bytes) -> Optional[tuple[bytes, int]]: if not hashX: # This can happen when the daemon is a block ahead # of us and has mempool txs spending outputs from diff --git a/src/electrumx/server/env.py b/src/electrumx/server/env.py index 36f5afc00..273582220 100644 --- a/src/electrumx/server/env.py +++ b/src/electrumx/server/env.py @@ -10,7 +10,7 @@ import re from ipaddress import IPv4Address, IPv6Address -from typing import Type +from typing import Type, Sequence from aiorpcx import Service, ServicePart from electrumx.lib.coins import Coin @@ -154,7 +154,7 @@ def _check_and_fix_cost_limits(self): "bumping COST_HARD_LIMIT by 1.") self.cost_hard_limit = self.cost_soft_limit + 1 - def _parse_services(self, services_str, default_func): + def _parse_services(self, services_str, default_func) -> Sequence[Service]: result = [] for service_str in services_str.split(','): if not service_str: @@ -177,7 +177,7 @@ def _parse_services(self, services_str, default_func): return result - def services_to_run(self): + def services_to_run(self) -> Sequence[Service]: def default_part(protocol, part): return default_services.get(protocol, {}).get(part) diff --git a/src/electrumx/server/mempool.py b/src/electrumx/server/mempool.py index 8040d0e95..d49a4a582 100644 --- a/src/electrumx/server/mempool.py +++ b/src/electrumx/server/mempool.py @@ -10,10 +10,11 @@ import itertools import time from abc import ABC, abstractmethod +import asyncio from asyncio import Lock from collections import defaultdict from dataclasses import dataclass -from typing import Sequence, Tuple, TYPE_CHECKING, Type, Dict, Optional, Set +from typing import Sequence, Tuple, TYPE_CHECKING, Type, Dict, Optional, Set, Iterable import math import attr @@ -31,7 +32,7 @@ @attr.s(slots=True) class MemPoolTx: - prevouts = attr.ib() # type: Sequence[Tuple[bytes, int]] # (txid, txout_idx) + prevouts = attr.ib() # type: Sequence[Tuple[bytes, int]] # (txid_rev, txout_idx) # A pair is a (hashX, value) tuple in_pairs = attr.ib() # type: Optional[Sequence[Tuple[bytes, int]]] # (hashX, value_in_sats) out_pairs = attr.ib() # type: Sequence[Tuple[bytes, int]] # (hashX, value_in_sats) @@ -41,16 +42,16 @@ class MemPoolTx: @attr.s(slots=True) class MemPoolTxSummary: - hash = attr.ib() # type: bytes + txid_rev = attr.ib() # type: bytes fee = attr.ib() # type: int # in sats has_unconfirmed_inputs = attr.ib() # type: bool @dataclass(slots=True, frozen=True, kw_only=True) class RecentMemPoolTx: - hash: bytes - fee: int # in sats - vsize: int # in vbytes + txid_rev: bytes + fee: int # in sats + vsize: int # in vbytes class DBSyncError(Exception): @@ -76,23 +77,23 @@ def db_height(self) -> int: '''Return the height flushed to the on-disk DB.''' @abstractmethod - async def mempool_hashes(self): - '''Query bitcoind for the hashes of all transactions in its + async def mempool_txids_hum(self) -> Sequence[str]: + '''Query bitcoind for the txids of all transactions in its mempool, returned as a list.''' @abstractmethod - async def raw_transactions(self, hex_hashes): + async def raw_transactions(self, txids_hum: Iterable[str]) -> Sequence[bytes | None]: '''Query bitcoind for the serialized raw transactions with the given - hashes. Missing transactions are returned as None. + txids. Missing transactions are returned as None. - hex_hashes is an iterable of hexadecimal hash strings.''' + txids_hum is an iterable of hexadecimal hash strings.''' @abstractmethod - async def lookup_utxos(self, prevouts): + async def lookup_utxos(self, prevouts: Sequence[Tuple[bytes, int]]) -> Sequence[Optional[Tuple[bytes, int]]]: '''Return a list of (hashX, value) pairs, one for each prevout if unspent, otherwise return None if spent or not found (for the given prevout). - prevouts - an iterable of (tx_hash, txout_idx) pairs + prevouts - an iterable of (txid_rev, txout_idx) pairs ''' @abstractmethod @@ -120,8 +121,8 @@ class MemPool: response to the calls in the external interface. To that end we maintain the following maps: - tx: tx_hash -> MemPoolTx - hashXs: hashX -> set of all hashes of txs touching the hashX + tx: txid_rev -> MemPoolTx + hashXs: hashX -> set of all txids_rev of txs touching the hashX ''' def __init__( @@ -136,10 +137,10 @@ def __init__( self.coin = coin self.api = api self.logger = class_logger(__name__, self.__class__.__name__) - self.txs = {} # type: Dict[bytes, MemPoolTx] # txid->tx - self.hashXs = defaultdict(set) # type: Dict[Optional[bytes], Set[bytes]] # hashX->txids - self.txo_to_spender = {} # type: Dict[Tuple[bytes, int], bytes] # prevout->txid - self.cached_compact_histogram = [] + self.txs = {} # type: Dict[bytes, MemPoolTx] # txid_rev->tx + self.hashXs = defaultdict(set) # type: Dict[Optional[bytes], Set[bytes]] # hashX->txids_rev + self.txo_to_spender = {} # type: Dict[Tuple[bytes, int], bytes] # prevout->txid_rev + self.cached_compact_histogram = [] # type: Sequence[tuple[float, int]] self.refresh_secs = refresh_secs self.log_status_secs = log_status_secs # Prevents mempool refreshes during fee histogram calculation @@ -227,7 +228,7 @@ def _compress_histogram( def _accept_transactions( self, *, - tx_map: Dict[bytes, MemPoolTx], # txid->tx + tx_map: Dict[bytes, MemPoolTx], # txid_rev->tx utxo_map: Dict[Tuple[bytes, int], Tuple[bytes, int]], # prevout->(hashX,value_in_sats) touched_hashxs: Set[bytes], # set of hashXs touched_outpoints: Set[Tuple[bytes, int]], # set of outpoints @@ -246,7 +247,7 @@ def _accept_transactions( deferred = {} unspent = set(utxo_map) # Try to find all prevouts so we can accept the TX - for tx_hash, tx in tx_map.items(): + for txid_rev, tx in tx_map.items(): in_pairs = [] try: for prevout in tx.prevouts: @@ -257,7 +258,7 @@ def _accept_transactions( utxo = txs[prev_hash].out_pairs[prev_index] in_pairs.append(utxo) except KeyError: - deferred[tx_hash] = tx + deferred[txid_rev] = tx continue # Spend the prevouts @@ -269,16 +270,16 @@ def _accept_transactions( # because some in_parts would be missing tx.fee = max(0, (sum(v for _, v in tx.in_pairs) - sum(v for _, v in tx.out_pairs))) - txs[tx_hash] = tx + txs[txid_rev] = tx for hashX, _value in itertools.chain(tx.in_pairs, tx.out_pairs): touched_hashxs.add(hashX) - hashXs[hashX].add(tx_hash) + hashXs[hashX].add(txid_rev) for prevout in tx.prevouts: - txo_to_spender[prevout] = tx_hash + txo_to_spender[prevout] = txid_rev touched_outpoints.add(prevout) for out_idx, out_pair in enumerate(tx.out_pairs): - touched_outpoints.add((tx_hash, out_idx)) + touched_outpoints.add((txid_rev, out_idx)) return deferred, {prevout: utxo_map[prevout] for prevout in unspent} @@ -290,14 +291,14 @@ async def _refresh_hashes(self, synchronized_event): touched_outpoints = set() while True: height = self.api.cached_height() - hex_hashes = await self.api.mempool_hashes() + txids_hum = await self.api.mempool_txids_hum() if height != await self.api.height(): continue - hashes = {hex_str_to_hash(hh) for hh in hex_hashes} + txids_rev = {hex_str_to_hash(hh) for hh in txids_hum} try: async with self.lock: await self._process_mempool( - all_hashes=hashes, + all_txids_rev=txids_rev, touched_hashxs=touched_hashxs, touched_outpoints=touched_outpoints, mempool_height=height, @@ -321,7 +322,7 @@ async def _refresh_hashes(self, synchronized_event): async def _process_mempool( self, *, - all_hashes: Set[bytes], # set of txids + all_txids_rev: Set[bytes], # set of txids_rev touched_hashxs: Set[bytes], # set of hashXs touched_outpoints: Set[Tuple[bytes, int]], # set of outpoints mempool_height: int, @@ -335,13 +336,13 @@ async def _process_mempool( raise DBSyncError # First handle txs that have disappeared - for tx_hash in (set(txs) - all_hashes): - tx = txs.pop(tx_hash) + for txid_rev in (set(txs) - all_txids_rev): + tx = txs.pop(txid_rev) # hashXs tx_hashXs = {hashX for hashX, value in tx.in_pairs} tx_hashXs.update(hashX for hashX, value in tx.out_pairs) for hashX in tx_hashXs: - hashXs[hashX].remove(tx_hash) + hashXs[hashX].remove(txid_rev) if not hashXs[hashX]: del hashXs[hashX] touched_hashxs |= tx_hashXs @@ -350,16 +351,16 @@ async def _process_mempool( del txo_to_spender[prevout] touched_outpoints.add(prevout) for out_idx, out_pair in enumerate(tx.out_pairs): - touched_outpoints.add((tx_hash, out_idx)) + touched_outpoints.add((txid_rev, out_idx)) # Process new transactions - new_hashes = list(all_hashes.difference(txs)) + new_hashes = list(all_txids_rev.difference(txs)) if new_hashes: group = OldTaskGroup() for hashes in chunks(new_hashes, 200): coro = self._fetch_and_accept( - hashes=hashes, - all_hashes=all_hashes, + new_txids_rev=hashes, + all_txids_rev=all_txids_rev, touched_hashxs=touched_hashxs, touched_outpoints=touched_outpoints, ) @@ -390,14 +391,14 @@ async def _process_mempool( async def _fetch_and_accept( self, *, - hashes: Set[bytes], # set of txids - all_hashes: Set[bytes], # set of txids + new_txids_rev: Set[bytes], # new txs being added to mempool + all_txids_rev: Set[bytes], # existing txs in mempool touched_hashxs: Set[bytes], # set of hashXs touched_outpoints: Set[Tuple[bytes, int]], # set of outpoints ): '''Fetch a list of mempool transactions.''' - hex_hashes_iter = (hash_to_hex_str(hash) for hash in hashes) - raw_txs = await self.api.raw_transactions(hex_hashes_iter) + txids_hum_iter = (hash_to_hex_str(hash) for hash in new_txids_rev) + raw_txs = await self.api.raw_transactions(txids_hum_iter) def deserialize_txs() -> Dict[bytes, MemPoolTx]: """This function is pure""" @@ -405,7 +406,7 @@ def deserialize_txs() -> Dict[bytes, MemPoolTx]: deserializer = self.coin.DESERIALIZER txs = {} # type: Dict[bytes, MemPoolTx] - for hash, raw_tx in zip(hashes, raw_txs): + for txid_rev, raw_tx in zip(new_txids_rev, raw_txs): # The daemon may have evicted the tx from its # mempool or it may have gotten in a block if not raw_tx: @@ -413,16 +414,16 @@ def deserialize_txs() -> Dict[bytes, MemPoolTx]: try: tx, tx_size = deserializer(raw_tx).read_tx_and_vsize() except SkipTxDeserialize as ex: - self.logger.debug(f'skipping tx {hash_to_hex_str(hash)}: {ex}') + self.logger.debug(f'skipping tx {hash_to_hex_str(txid_rev)}: {ex}') continue # Convert the inputs and outputs into (hashX, value) pairs # Drop generation-like inputs from MemPoolTx.prevouts - txin_pairs = tuple((txin.prev_hash, txin.prev_idx) + txin_pairs = tuple((txin.prev_txid_rev, txin.prev_idx) for txin in tx.inputs if not txin.is_generation()) txout_pairs = tuple((to_hashX(txout.pk_script), txout.value) for txout in tx.outputs) - txs[hash] = MemPoolTx( + txs[txid_rev] = MemPoolTx( prevouts=txin_pairs, in_pairs=None, out_pairs=txout_pairs, @@ -441,7 +442,7 @@ def deserialize_txs() -> Dict[bytes, MemPoolTx]: # generation-like. prevouts = tuple(prevout for tx in tx_map.values() for prevout in tx.prevouts - if prevout[0] not in all_hashes) + if prevout[0] not in all_txids_rev) utxos = await self.api.lookup_utxos(prevouts) utxo_map = {prevout: utxo for prevout, utxo in zip(prevouts, utxos)} @@ -456,31 +457,31 @@ def deserialize_txs() -> Dict[bytes, MemPoolTx]: # External interface # - async def keep_synchronized(self, synchronized_event): + async def keep_synchronized(self, synchronized_event: asyncio.Event) -> None: '''Keep the mempool synchronized with the daemon.''' async with OldTaskGroup() as group: await group.spawn(self._refresh_hashes(synchronized_event)) await group.spawn(self._refresh_histogram(synchronized_event)) await group.spawn(self._logging(synchronized_event)) - async def balance_delta(self, hashX): + async def balance_delta(self, hashX: bytes) -> int: '''Return the unconfirmed amount in the mempool for hashX. Can be positive or negative. ''' value = 0 if hashX in self.hashXs: - for hash in self.hashXs[hashX]: - tx = self.txs[hash] + for txid_rev in self.hashXs[hashX]: + tx = self.txs[txid_rev] value -= sum(v for h168, v in tx.in_pairs if h168 == hashX) value += sum(v for h168, v in tx.out_pairs if h168 == hashX) return value - async def compact_fee_histogram(self): + async def compact_fee_histogram(self) -> Sequence[tuple[float, int]]: '''Return a compact fee histogram of the current mempool.''' return self.cached_compact_histogram - async def potential_spends(self, hashX): + async def potential_spends(self, hashX: bytes) -> set[tuple[bytes, int]]: '''Return a set of (prev_hash, prev_idx) pairs from mempool transactions that touch hashX. @@ -488,24 +489,24 @@ async def potential_spends(self, hashX): actual spends of it (in the DB or mempool) will be included. ''' result = set() - for tx_hash in self.hashXs.get(hashX, ()): - tx = self.txs[tx_hash] + for txid_rev in self.hashXs.get(hashX, ()): + tx = self.txs[txid_rev] result.update(tx.prevouts) return result - async def transaction_summaries(self, hashX): + async def transaction_summaries(self, hashX: bytes) -> Sequence[MemPoolTxSummary]: '''Return a list of MemPoolTxSummary objects for the hashX, sorted as expected by protocol methods. ''' - result = [] - for tx_hash in self.hashXs.get(hashX, ()): - tx = self.txs[tx_hash] + result = [] # type: list[MemPoolTxSummary] + for txid_rev in self.hashXs.get(hashX, ()): + tx = self.txs[txid_rev] has_ui = any(hash in self.txs for hash, idx in tx.prevouts) - result.append(MemPoolTxSummary(tx_hash, tx.fee, has_ui)) - result.sort(key=lambda x: (x.has_unconfirmed_inputs, x.hash[::-1])) + result.append(MemPoolTxSummary(txid_rev, tx.fee, has_ui)) + result.sort(key=lambda x: (x.has_unconfirmed_inputs, x.txid_rev[::-1])) return result - async def unordered_UTXOs(self, hashX): + async def unordered_UTXOs(self, hashX: bytes) -> Sequence[UTXO]: '''Return an unordered list of UTXO named tuples from mempool transactions that pay to hashX. @@ -513,20 +514,20 @@ async def unordered_UTXOs(self, hashX): the outputs. ''' utxos = [] - for tx_hash in self.hashXs.get(hashX, ()): - tx = self.txs.get(tx_hash) + for txid_rev in self.hashXs.get(hashX, ()): + tx = self.txs.get(txid_rev) for pos, (hX, value) in enumerate(tx.out_pairs): if hX == hashX: - utxos.append(UTXO(-1, pos, tx_hash, 0, value)) + utxos.append(UTXO(-1, pos, txid_rev, 0, value)) return utxos - async def spender_for_txo(self, prev_txhash: bytes, txout_idx: int) -> 'TXOSpendStatus': + async def spender_for_txo(self, prev_txid_rev: bytes, txout_idx: int) -> 'TXOSpendStatus': '''For an outpoint, returns its spend-status. This only considers the mempool, not the DB/blockchain, so e.g. mined txs are not distinguished from txs that never existed. ''' # look up funding tx - prev_tx = self.txs.get(prev_txhash, None) + prev_tx = self.txs.get(prev_txid_rev, None) if prev_tx is None: # funding tx already mined or never existed prev_height = None @@ -536,22 +537,22 @@ async def spender_for_txo(self, prev_txhash: bytes, txout_idx: int) -> 'TXOSpend return TXOSpendStatus(prev_height=None) prev_has_ui = any(hash in self.txs for hash, idx in prev_tx.prevouts) prev_height = -prev_has_ui - prevout = (prev_txhash, txout_idx) + prevout = (prev_txid_rev, txout_idx) # look up spending tx - spender_txhash = self.txo_to_spender.get(prevout, None) - if spender_txhash is None: + spender_txid_rev = self.txo_to_spender.get(prevout, None) + if spender_txid_rev is None: return TXOSpendStatus(prev_height=prev_height) - spender_tx = self.txs.get(spender_txhash, None) + spender_tx = self.txs.get(spender_txid_rev, None) if spender_tx is None: - self.logger.warning(f"spender_tx {hash_to_hex_str(spender_txhash)} not in" + self.logger.warning(f"spender_tx {hash_to_hex_str(spender_txid_rev)} not in" f"mempool, but txo_to_spender referenced it as spender " - f"of {hash_to_hex_str(prev_txhash)}:{txout_idx} ?!") + f"of {hash_to_hex_str(prev_txid_rev)}:{txout_idx} ?!") return TXOSpendStatus(prev_height=prev_height) spender_has_ui = any(hash in self.txs for hash, idx in spender_tx.prevouts) spender_height = -spender_has_ui return TXOSpendStatus( prev_height=prev_height, - spender_txhash=spender_txhash, + spender_txid_rev=spender_txid_rev, spender_height=spender_height, ) @@ -561,5 +562,5 @@ async def get_recently_added_txs(self, *, count: int) -> Sequence[RecentMemPoolT count = min(count, len(self.txs)) mempool_txs = [next(it) for _ in range(count)] return [ - RecentMemPoolTx(hash=hash, fee=mtx.fee, vsize=mtx.size) + RecentMemPoolTx(txid_rev=hash, fee=mtx.fee, vsize=mtx.size) for hash, mtx in mempool_txs] diff --git a/src/electrumx/server/peers.py b/src/electrumx/server/peers.py index 6d693cbff..6eba31eec 100644 --- a/src/electrumx/server/peers.py +++ b/src/electrumx/server/peers.py @@ -14,7 +14,7 @@ import time from collections import Counter, defaultdict from ipaddress import IPv4Address, IPv6Address -from typing import TYPE_CHECKING, Type +from typing import TYPE_CHECKING, Type, Any, Optional from functools import partial import aiohttp @@ -519,7 +519,7 @@ async def add_localRPC_peer(self, real_name): '''Add a peer passed by the admin over LocalRPC.''' await self._note_peers([Peer.from_real_name(real_name, 'RPC')], check_ports=True) - async def on_add_peer(self, features, source_addr): + async def on_add_peer(self, features: dict[str, Any] | Any, source_addr: Optional[str]): '''Add a peer (but only if the peer resolves to the source).''' if self.env.peer_discovery != self.env.PD_ON: return False diff --git a/src/electrumx/server/session.py b/src/electrumx/server/session.py index 6dfa90fd7..242fd8749 100644 --- a/src/electrumx/server/session.py +++ b/src/electrumx/server/session.py @@ -23,6 +23,7 @@ from typing import Callable import attr +import aiorpcx from aiorpcx import (Event, JSONRPCAutoDetect, JSONRPCConnection, ReplyAndDisconnect, Request, RPCError, RPCSession, Service, handler_invocation, serve_rs, serve_ws, sleep, @@ -49,6 +50,7 @@ from electrumx.server.daemon import Daemon from electrumx.server.mempool import MemPool from electrumx.server.peers import PeerManager + from electrumx.server.controller import Controller, Notifications BAD_REQUEST = 1 @@ -72,7 +74,7 @@ def spk_to_scripthash(spk: str) -> str: return h[::-1].hex() -def non_negative_integer(value): +def non_negative_integer(value: Any | int) -> int: '''Return param value it is or can be converted to a non-negative integer, otherwise raise an RPCError.''' try: @@ -85,17 +87,17 @@ def non_negative_integer(value): f'{value} should be a non-negative integer') -def assert_boolean(value): +def assert_boolean(value: Any | bool) -> bool: '''Return param value it is boolean otherwise raise an RPCError.''' if value in (False, True): return value raise RPCError(BAD_REQUEST, f'{value} should be a boolean value') -def assert_tx_hash(value): - '''Raise an RPCError if the value is not a valid hexadecimal transaction hash. +def assert_txid_hum(value: Any | str) -> bytes: + '''Raise an RPCError if the value is not a valid hexadecimal txid_hum. - If it is valid, return it as 32-byte binary hash. + If it is valid, return it as 32-byte binary txid_rev. ''' try: raw_hash = hex_str_to_hash(value) @@ -106,7 +108,7 @@ def assert_tx_hash(value): raise RPCError(BAD_REQUEST, f'{value} should be a transaction hash') -def assert_hex_str(value: Any) -> None: +def assert_hex_str(value: Any | str) -> None: if not is_hex_str(value): raise RPCError(BAD_REQUEST, f'{value} should be a hex string') @@ -118,25 +120,25 @@ def assert_list_or_tuple(value: Any) -> None: @attr.s(slots=True) class SessionGroup: - name = attr.ib() - weight = attr.ib() - sessions = attr.ib() # type: Set[ElectrumX] - retained_cost = attr.ib() + name = attr.ib() # type: str + weight = attr.ib() # type: float + sessions = attr.ib() # type: Set[SessionBase] + retained_cost = attr.ib() # type: float - def session_cost(self): + def session_cost(self) -> float: return sum(session.cost for session in self.sessions) - def cost(self): + def cost(self) -> float: return self.retained_cost + self.session_cost() @attr.s(slots=True) class SessionReferences: # All attributes are sets but groups is a list - sessions = attr.ib() - groups = attr.ib() - specials = attr.ib() # Lower-case strings - unknown = attr.ib() # Strings + sessions = attr.ib() # type: set[ElectrumX] + groups = attr.ib() # type: Sequence[SessionGroup] + specials = attr.ib() # type: set[str] # Lower-case strings + unknown = attr.ib() # type: set[str] class SessionManager: @@ -162,19 +164,22 @@ def __init__( self.peer_mgr = PeerManager(env, db) self.shutdown_event = shutdown_event self.logger = util.class_logger(__name__, self.__class__.__name__) - self.servers = {} # service->server - self.sessions = {} # type: Dict[ElectrumX, Iterable[SessionGroup]] + self.servers = {} # type: Dict[Service, asyncio.Server] + self.sessions = {} # type: Dict[SessionBase, Iterable[SessionGroup]] self.session_groups = {} # type: Dict[str, SessionGroup] self.txs_sent = 0 # Would use monotonic time, but aiorpcx sessions use Unix time: self.start_time = time.time() self._method_counts = defaultdict(int) self._reorg_count = 0 - self._history_cache = LRUCache(maxsize=1000) - self._txids_cache = LRUCache(maxsize=1000) + self._history_cache = LRUCache(maxsize=1000) # type: LRUCache[bytes, Sequence[tuple[bytes, int]] | RPCError] + self._txids_cache = LRUCache(maxsize=1000) # type: LRUCache[int, Sequence[bytes]] # Really a MerkleCache cache - self._merkle_txid_cache = LRUCache(maxsize=1000) - self.estimatefee_cache = LRUCache(maxsize=1000) + self._merkle_txid_cache = LRUCache(maxsize=1000) # type: LRUCache[int, MerkleCache] + self.estimatefee_cache : LRUCache[ + tuple[int, str | None], + tuple[bytes | None, float | None, asyncio.Lock] + ] = LRUCache(maxsize=1000) self.notified_height = None self.hsub_results = None self._task_group = OldTaskGroup() @@ -190,13 +195,13 @@ def __init__( LocalRPC.request_handlers = {cmd: getattr(self, 'rpc_' + cmd) for cmd in cmds} - def _ssl_context(self): + def _ssl_context(self) -> ssl.SSLContext: if self._sslc is None: self._sslc = ssl.SSLContext(ssl.PROTOCOL_TLS) self._sslc.load_cert_chain(self.env.ssl_certfile, keyfile=self.env.ssl_keyfile) return self._sslc - async def _start_servers(self, services): + async def _start_servers(self, services: Iterable[Service]) -> None: for service in services: kind = service.protocol.upper() if service.protocol in self.env.SSL_PROTOCOLS: @@ -241,7 +246,7 @@ async def _start_external_servers(self): if service.protocol != 'rpc') self.server_listening.set() - async def _stop_servers(self, services): + async def _stop_servers(self, services: Iterable[Service]): '''Stop the servers of the given protocols.''' for service in services: self.logger.info(f'closing down server for {service}') @@ -252,7 +257,7 @@ def _remove_servers(self, services: Iterable[Service]): for service in services: del self.servers[service] - async def _manage_servers(self): + async def _manage_servers(self) -> None: paused = False max_sessions = self.env.max_sessions low_watermark = max_sessions * 19 // 20 @@ -275,7 +280,7 @@ async def _manage_servers(self): await self._start_external_servers() paused = False - async def _log_sessions(self): + async def _log_sessions(self) -> None: '''Periodically log sessions.''' log_interval = self.env.log_sessions if log_interval: @@ -286,14 +291,16 @@ async def _log_sessions(self): self.logger.info(line) self.logger.info(util.json_serialize(self._get_info())) - async def _disconnect_sessions(self, sessions, reason, *, force_after=1.0): + async def _disconnect_sessions( + self, sessions: Sequence['SessionBase'], reason: str, *, force_after: float = 1.0, + ) -> None: if sessions: session_ids = ', '.join(str(session.session_id) for session in sessions) self.logger.info(f'{reason} session ids {session_ids}') for session in sessions: await self._task_group.spawn(session.close(force_after=force_after)) - async def _clear_stale_sessions(self): + async def _clear_stale_sessions(self) -> None: '''Cut off sessions that haven't done anything for 10 minutes.''' while True: await sleep(60) @@ -303,17 +310,17 @@ async def _clear_stale_sessions(self): await self._disconnect_sessions(stale_sessions, 'closing stale') del stale_sessions - async def _handle_chain_reorgs(self): + async def _handle_chain_reorgs(self) -> None: '''Clear certain caches on chain reorgs.''' while True: await self.bp.backed_up_event.wait() - self.logger.info(f'reorg signalled; clearing tx_hashes and merkle caches') + self.logger.info(f'reorg signalled; clearing txids and merkle caches') self._reorg_count += 1 # not: history_cache is cleared in _notify_sessions self._txids_cache.clear() self._merkle_txid_cache.clear() - async def _recalc_concurrency(self): + async def _recalc_concurrency(self) -> None: '''Periodically recalculate session concurrency.''' session_class = self.env.coin.SESSIONCLS period = 300 @@ -339,7 +346,7 @@ async def _recalc_concurrency(self): session.cost_decay_per_sec = hard_limit / (10000 + 5 * session.sub_count_total()) session.recalc_concurrency() - def _get_info(self): + def _get_info(self) -> dict[str, Any]: '''A summary of server state.''' def cache_fmt(cache: LRUCache): return f"{cache.num_lookups} lookups, {cache.num_hits} hits, {len(cache)} entries" @@ -373,7 +380,7 @@ def cache_fmt(cache: LRUCache): 'version': electrumx.version, } - def _session_data(self, for_log): + def _session_data(self, for_log: bool): '''Returned to the RPC 'sessions' call.''' now = time.time() sessions = sorted(self.sessions, key=lambda s: s.start_time) @@ -411,7 +418,7 @@ def _group_data(self): ]) return result - async def _refresh_hsub_results(self, height): + async def _refresh_hsub_results(self, height: int) -> None: '''Refresh the cached header subscription responses to be for height, and record that as notified_height. ''' @@ -421,7 +428,7 @@ async def _refresh_hsub_results(self, height): self.hsub_results = {'hex': raw.hex(), 'height': height} self.notified_height = height - def _session_references(self, items, special_strings): + def _session_references(self, items: Iterable[str] | Any, special_strings): '''Return a SessionReferences object.''' if not isinstance(items, list) or not all(isinstance(item, str) for item in items): raise RPCError(BAD_REQUEST, 'expected a list of session IDs') @@ -456,7 +463,7 @@ def _session_references(self, items, special_strings): # --- LocalRPC command handlers - async def rpc_add_peer(self, real_name): + async def rpc_add_peer(self, real_name: str) -> str: '''Add a peer. real_name: "bch.electrumx.cash t50001 s50002" for example @@ -464,7 +471,7 @@ async def rpc_add_peer(self, real_name): await self.peer_mgr.add_localRPC_peer(real_name) return f"peer '{real_name}' added" - async def rpc_disconnect(self, session_ids): + async def rpc_disconnect(self, session_ids: Iterable[str] | Any) -> Sequence[str]: '''Disconnect sesssions. session_ids: array of session IDs @@ -486,7 +493,7 @@ async def rpc_disconnect(self, session_ids): await self._disconnect_sessions(sessions, 'local RPC request to disconnect') return result - async def rpc_log(self, session_ids): + async def rpc_log(self, session_ids: Iterable[str] | Any) -> Sequence[str]: '''Toggle logging of sesssions. session_ids: array of session or group IDs, or 'all', 'none', 'new' @@ -524,7 +531,7 @@ def add_result(text, value): result.extend(f'unknown: {item}' for item in refs.unknown) return result - async def rpc_daemon_url(self, daemon_url): + async def rpc_daemon_url(self, daemon_url: str): '''Replace the daemon URL.''' daemon_url = daemon_url or self.env.daemon_url try: @@ -587,16 +594,16 @@ def arg_to_hashX(arg): continue n = None history = await db.limited_history(hashX, limit=limit) - for n, (tx_hash, height) in enumerate(history): + for n, (txid_rev, height) in enumerate(history): lines.append(f'History #{n:,d}: height {height:,d} ' - f'tx_hash {hash_to_hex_str(tx_hash)}') + f'txid {hash_to_hex_str(txid_rev)}') if n is None: lines.append('No history found') n = None utxos = await db.all_utxos(hashX) for n, utxo in enumerate(utxos, start=1): - lines.append(f'UTXO #{n:,d}: tx_hash ' - f'{hash_to_hex_str(utxo.tx_hash)} ' + lines.append(f'UTXO #{n:,d}: txid ' + f'{hash_to_hex_str(utxo.txid_rev)} ' f'tx_pos {utxo.tx_pos:,d} height ' f'{utxo.height:,d} value {utxo.value:,d}') if n == limit: @@ -614,7 +621,7 @@ async def rpc_sessions(self): '''Return statistics about connected sessions.''' return self._session_data(for_log=False) - async def rpc_reorg(self, count): + async def rpc_reorg(self, count: int) -> str: '''Force a reorg of the given number of blocks. count: number of blocks to reorg @@ -659,7 +666,7 @@ async def rpc_debug_memusage_get_random_backref_chain(self, objtype: str) -> str # --- External Interface - async def serve(self, notifications, event): + async def serve(self, notifications: 'Notifications', event: asyncio.Event) -> None: '''Start the RPC server if enabled. When the event is triggered, start TCP and SSL servers.''' try: @@ -727,7 +734,7 @@ async def serve(self, notifications, event): servers_to_remove = list(self.servers.keys()) self._remove_servers(servers_to_remove) - def extra_cost(self, session): + def extra_cost(self, session: 'SessionBase') -> float: # Note there is no guarantee that session is still in self.sessions. Example traceback: # notify_sessions->notify->address_status->bump_cost->recalc_concurrency->extra_cost # during which there are many places the sesssion could be removed @@ -736,90 +743,94 @@ def extra_cost(self, session): return 0 return sum((group.cost() - session.cost) * group.weight for group in groups) - async def _merkle_branch(self, height, tx_hashes, tx_pos): - tx_hash_count = len(tx_hashes) - cost = tx_hash_count + async def _merkle_branch( + self, height: int, txids_rev: Sequence[bytes], tx_pos: int, + ) -> tuple[Sequence[str], float]: + tx_count = len(txids_rev) + cost = tx_count - if tx_hash_count >= 200: + if tx_count >= 200: self._merkle_txid_cache.num_lookups += 1 merkle_cache = self._merkle_txid_cache.get(height) if merkle_cache: self._merkle_txid_cache.num_hits += 1 - cost = 10 * math.sqrt(tx_hash_count) + cost = 10 * math.sqrt(tx_count) else: async def tx_hashes_func(start, count): - return tx_hashes[start: start + count] + return txids_rev[start: start + count] merkle_cache = MerkleCache(self.db.merkle, tx_hashes_func) self._merkle_txid_cache[height] = merkle_cache - await merkle_cache.initialize(len(tx_hashes)) - branch, _root = await merkle_cache.branch_and_root(tx_hash_count, tx_pos) + await merkle_cache.initialize(len(txids_rev)) + branch, _root = await merkle_cache.branch_and_root(tx_count, tx_pos) else: - branch, _root = self.db.merkle.branch_and_root(tx_hashes, tx_pos) + branch, _root = self.db.merkle.branch_and_root(txids_rev, tx_pos) branch = [hash_to_hex_str(hash) for hash in branch] return branch, cost / 2500 - async def merkle_branch_for_tx_hash( - self, *, tx_hash: bytes, height: int, + async def merkle_branch_for_txid( + self, *, txid_rev: bytes, height: int, ) -> Tuple[Sequence[str], int, bytes, float]: '''Return (branch, tx_pos, block_header, cost).''' block_header = await self.raw_header(height) - tx_hashes, tx_hashes_cost = await self.tx_hashes_at_blockheight(height) + txids_rev, txids_cost = await self.txids_rev_at_blockheight(height) try: - tx_pos = tx_hashes.index(tx_hash) + tx_pos = txids_rev.index(txid_rev) except ValueError: raise RPCError(BAD_REQUEST, - f'tx {hash_to_hex_str(tx_hash)} not in block at height {height:,d}') - branch, merkle_cost = await self._merkle_branch(height, tx_hashes, tx_pos) + f'tx {hash_to_hex_str(txid_rev)} not in block at height {height:,d}') + branch, merkle_cost = await self._merkle_branch(height, txids_rev, tx_pos) if block_header != await self.raw_header(height): # there was a reorg while processing the request... TODO maybe retry? raise RPCError(BAD_REQUEST, - f'tx {hash_to_hex_str(tx_hash)} was reorged while processing request') - return branch, tx_pos, block_header, tx_hashes_cost + merkle_cost + f'tx {hash_to_hex_str(txid_rev)} was reorged while processing request') + return branch, tx_pos, block_header, txids_cost + merkle_cost - async def merkle_branch_for_tx_pos(self, height, tx_pos): - '''Return a triple (branch, tx_hash_hex, cost).''' - tx_hashes, tx_hashes_cost = await self.tx_hashes_at_blockheight(height) + async def merkle_branch_for_tx_pos(self, height: int, tx_pos: int) -> tuple[Sequence[str], str, float]: + '''Return a triple (branch, txid_hum, cost).''' + txids_rev, txids_cost = await self.txids_rev_at_blockheight(height) try: - tx_hash = tx_hashes[tx_pos] + txid_rev = txids_rev[tx_pos] except IndexError: raise RPCError(BAD_REQUEST, f'no tx at position {tx_pos:,d} in block at height {height:,d}') - branch, merkle_cost = await self._merkle_branch(height, tx_hashes, tx_pos) - return branch, hash_to_hex_str(tx_hash), tx_hashes_cost + merkle_cost + branch, merkle_cost = await self._merkle_branch(height, txids_rev, tx_pos) + txid_hum = hash_to_hex_str(txid_rev) + cost = txids_cost + merkle_cost + return branch, txid_hum, cost - async def tx_hashes_at_blockheight(self, height): - '''Returns a pair (tx_hashes, cost). + async def txids_rev_at_blockheight(self, height: int) -> tuple[Sequence[bytes], float]: + '''Returns a pair (txids_rev, cost). - tx_hashes is an ordered list of binary hashes, cost is an estimated cost of + txids_rev is an ordered list of binary hashes, cost is an estimated cost of getting the hashes; cheaper if in-cache. Raises RPCError. ''' self._txids_cache.num_lookups += 1 - tx_hashes = self._txids_cache.get(height) - if tx_hashes: + txids_rev = self._txids_cache.get(height) + if txids_rev: self._txids_cache.num_hits += 1 - return tx_hashes, 0.1 + return txids_rev, 0.1 - # Ensure the tx_hashes are fresh before placing in the cache + # Ensure the txids_rev are fresh before placing in the cache while True: reorg_count = self._reorg_count try: - tx_hashes = await self.db.tx_hashes_at_blockheight(height) + txids_rev = await self.db.txids_rev_at_blockheight(height) except self.db.DBError as e: raise RPCError(BAD_REQUEST, f'db error: {e!r}') if reorg_count == self._reorg_count: break - self._txids_cache[height] = tx_hashes + self._txids_cache[height] = txids_rev - return tx_hashes, 0.25 + len(tx_hashes) * 0.0001 + return txids_rev, 0.25 + len(txids_rev) * 0.0001 def session_count(self): '''The number of connections that we've sent something to.''' return len(self.sessions) - async def daemon_request(self, method, *args): + async def daemon_request(self, method: str, *args): '''Catch a DaemonError and convert it to an RPCError.''' try: return await getattr(self.daemon, method)(*args) @@ -834,20 +845,20 @@ async def raw_header(self, height: int) -> bytes: raise RPCError(BAD_REQUEST, f'height {height:,d} ' 'out of range') from None - async def broadcast_transaction(self, raw_tx): - hex_hash = await self.daemon.broadcast_transaction(raw_tx) + async def broadcast_transaction(self, raw_tx: str) -> str: + txid_hum = await self.daemon.broadcast_transaction(raw_tx) self.txs_sent += 1 - return hex_hash + return txid_hum async def broadcast_package(self, tx_package: Sequence[str]) -> dict: result = await self.daemon.broadcast_package(tx_package) self.txs_sent += len(tx_package) return result - async def limited_history(self, hashX): + async def limited_history(self, hashX: bytes) -> tuple[Sequence[tuple[bytes, int]], float]: '''Returns a pair (history, cost). - History is a sorted list of (tx_hash, height) tuples, or an RPCError.''' + History is a sorted list of (txid_rev, height) tuples, or an RPCError.''' # History DoS limit. Each element of history is about 99 bytes when encoded # as JSON. limit = self.env.max_send // 99 @@ -863,6 +874,7 @@ async def limited_history(self, hashX): result = RPCError(BAD_REQUEST, f'history too large', cost=cost) self._history_cache[hashX] = result + assert result is not None if isinstance(result, Exception): raise result return result, cost @@ -873,7 +885,7 @@ async def _notify_sessions( touched_hashxs: Set[bytes], touched_outpoints: Set[Tuple[bytes, int]], height: int, - ): + ) -> None: '''Notify sessions about height changes and touched addresses.''' height_changed = height != self.notified_height if height_changed: @@ -918,7 +930,7 @@ def _session_group(self, name: Optional[str], weight: float) -> Optional[Session self.session_groups[name] = group return group - def add_session(self, session): + def add_session(self, session: 'SessionBase') -> None: self.session_event.set() # Return the session groups groups = ( @@ -929,7 +941,7 @@ def add_session(self, session): for group in groups: group.sessions.add(session) - def remove_session(self, session): + def remove_session(self, session: 'SessionBase') -> None: '''Remove a session from our sessions list if there.''' self.session_event.set() groups = self.sessions.pop(session) @@ -960,7 +972,7 @@ async def _start_main_loop(self) -> None: else: raise - async def main_loop(self): + async def main_loop(self) -> None: """Manages taskgroup tied to this session. The session and the taskgroup share a lifecycle, either dying will kill the other. This method must not raise, to avoid killing the manager_taskgroup. @@ -1026,7 +1038,7 @@ def __init__( self.anon_logs = self.env.anon_logs self.txs_sent = 0 self.log_me = SessionBase.log_new - self.session_id = None + self.session_id = None # type: int self.daemon_request = self.session_mgr.daemon_request self.session_id = next(self.session_counter) context = {'conn_id': f'{self.session_id}'} @@ -1043,20 +1055,20 @@ async def notify( touched_hashxs: Set[bytes], touched_outpoints: Set[Tuple[bytes, int]], height_changed: bool, - ): + ) -> None: pass def default_framer(self): return NewlineFramer(max_size=self.env.max_recv) - def remote_address_string(self, *, for_log=True): + def remote_address_string(self, *, for_log: bool = True) -> str: '''Returns the peer's IP address and port as a human-readable string, respecting anon logs if the output is for a log.''' if for_log and self.anon_logs: return 'xx.xx.xx.xx:xx' return str(self.remote_address()) - def flags(self): + def flags(self) -> str: '''Status flags.''' status = self.kind[0] if self.is_closing(): @@ -1079,13 +1091,13 @@ async def connection_lost(self): msg = 'disconnected' + msg self.logger.info(msg) - def sub_count_scripthashes(self): + def sub_count_scripthashes(self) -> int: return 0 - def sub_count_txoutpoints(self): + def sub_count_txoutpoints(self) -> int: return 0 - def sub_count_total(self): + def sub_count_total(self) -> int: return self.sub_count_scripthashes() + self.sub_count_txoutpoints() async def handle_request(self, request: SingleRequest): @@ -1114,7 +1126,9 @@ async def handle_request(self, request: SingleRequest): def protocol_version_string(self) -> str: raise NotImplementedError() - async def maybe_crash_old_client(self, ptuple, crash_client_ver): + async def maybe_crash_old_client( + self, ptuple: Optional[tuple[int, ...]], crash_client_ver: Optional[tuple[int, ...]], + ) -> None: if crash_client_ver: client_ver = util.protocol_tuple(self.client) is_old_protocol = ptuple is None or ptuple <= (1, 2) @@ -1141,8 +1155,8 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.subscribe_headers = False self.connection.max_response_size = self.env.max_send - self.hashX_subs = {} # type: Dict[bytes, bytes] # hashX -> scripthash - self.txoutpoint_subs = set() # type: Set[Tuple[bytes, int]] + self.hashX_subs = {} # type: Dict[bytes, str] # hashX -> scripthash + self.txoutpoint_subs = set() # type: Set[Tuple[bytes, int]] # (txid_rev, txout_idx) self.mempool_hashX_statuses = {} # type: Dict[bytes, str] self.mempool_txoutpoint_statuses = {} # type: Dict[Tuple[bytes, int], Mapping[str, Any]] self.set_request_handlers(self.PROTOCOL_MIN) @@ -1150,12 +1164,13 @@ def __init__(self, *args, **kwargs): self.cost = 5.0 # Connection cost @classmethod - def protocol_min_max_strings(cls): - return [util.version_string(ver) - for ver in (cls.PROTOCOL_MIN, cls.PROTOCOL_MAX)] + def protocol_min_max_strings(cls) -> tuple[str, str]: + return tuple( + util.version_string(ver) + for ver in (cls.PROTOCOL_MIN, cls.PROTOCOL_MAX)) @classmethod - def server_features(cls, env): + def server_features(cls, env: 'Env') -> dict[str, Any]: '''Return the server features dictionary.''' hosts_dict = {} for service in env.report_services: @@ -1175,7 +1190,7 @@ def server_features(cls, env): 'services': [str(service) for service in env.report_services], } - async def server_features_async(self): + async def server_features_async(self) -> dict[str, Any]: self.bump_cost(0.2) return self.server_features(self.env) @@ -1187,7 +1202,7 @@ def server_version_args(cls): def protocol_version_string(self): return util.version_string(self.protocol_tuple) - def extra_cost(self): + def extra_cost(self) -> float: return self.session_mgr.extra_cost(self) def on_disconnect_due_to_excessive_session_cost(self): @@ -1203,7 +1218,7 @@ def sub_count_scripthashes(self): def sub_count_txoutpoints(self): return len(self.txoutpoint_subs) - def unsubscribe_hashX(self, hashX): + def unsubscribe_hashX(self, hashX: bytes) -> Optional[str]: self.mempool_hashX_statuses.pop(hashX, None) return self.hashX_subs.pop(hashX, None) @@ -1234,7 +1249,7 @@ async def _notify_inner( touched_hashxs: Set[bytes], touched_outpoints: Set[Tuple[bytes, int]], height_changed: bool, - ): + ) -> None: '''Notify the client about changes to touched addresses (from mempool updates or new blocks) and height. ''' @@ -1247,7 +1262,7 @@ async def _notify_inner( num_hashx_notifs_sent = 0 touched_hashxs = touched_hashxs.intersection(self.hashX_subs) if touched_hashxs or (height_changed and self.mempool_hashX_statuses): - changed = {} + changed = {} # type: dict[str, Optional[str]] for hashX in touched_hashxs: alias = self.hashX_subs.get(hashX) @@ -1312,7 +1327,7 @@ async def headers_subscribe(self): self.bump_cost(0.25) return await self.subscribe_headers_result() - async def add_peer(self, features): + async def add_peer(self, features: dict[str, Any] | Any): '''Add a peer (but only if the peer resolves to the source).''' self.is_peer = True self.bump_cost(100.0) @@ -1337,7 +1352,7 @@ async def address_status(self, hashX: bytes) -> Optional[str]: status = ''.join(f'{hash_to_hex_str(tx_hash)}:' f'{height:d}:' for tx_hash, height in db_history) - status += ''.join(f'{hash_to_hex_str(tx.hash)}:' + status += ''.join(f'{hash_to_hex_str(tx.txid_rev)}:' f'{-tx.has_unconfirmed_inputs:d}:' for tx in mempool) @@ -1357,7 +1372,7 @@ async def address_status(self, hashX: bytes) -> Optional[str]: return status - async def subscription_address_status(self, hashX): + async def subscription_address_status(self, hashX: bytes) -> Optional[str]: '''As for address_status, but if it can't be calculated the subscription is discarded.''' try: @@ -1366,7 +1381,7 @@ async def subscription_address_status(self, hashX): self.unsubscribe_hashX(hashX) return None - async def _spender_for_txo(self, prev_txhash: bytes, txout_idx: int) -> 'TXOSpendStatus': + async def _spender_for_txo(self, prev_txid_rev: bytes, txout_idx: int) -> 'TXOSpendStatus': """For an outpoint, returns its spend-status (ignoring mempool events). Uses daemon (bitcoind) to find the spender_txhash, requiring "txospenderindex=1". @@ -1374,19 +1389,19 @@ async def _spender_for_txo(self, prev_txhash: bytes, txout_idx: int) -> 'TXOSpen using only the daemon. Instead, our own mempool data (as opposed to bitcoind's) can be used separately to enrich the return value. """ - prev_txid = hash_to_hex_str(prev_txhash) + prev_txid_hum = hash_to_hex_str(prev_txid_rev) # 1. call bitcoind "getrawtransaction" to see if prevtx exists/is_mined self.bump_cost(1) try: - prevtx_item = await self.session_mgr.daemon.getrawtransaction(prev_txid, verbose=True) # verbose=int(1) + prevtx_item = await self.session_mgr.daemon.getrawtransaction(prev_txid_hum, verbose=True) # verbose=int(1) except DaemonError as e: error, = e.args ecode = error['code'] if ecode == -5: # "No such mempool or blockchain transaction." return TXOSpendStatus(prev_height=None) # utxo never existed - self.logger.debug(f"getrawtransaction errored. {prev_txid=}. {error=}") + self.logger.debug(f"getrawtransaction errored. {prev_txid_hum=}. {error=}") raise RPCError(DAEMON_ERROR, f'daemon error: {error!r}') from None # TODO some callers do not expect this - assert prevtx_item.get("txid") == prev_txid, f"{prevtx_item.get('txid')=} != {prev_txid=}" + assert prevtx_item.get("txid") == prev_txid_hum, f"{prevtx_item.get('txid')=} != {prev_txid_hum=}" funder_bhash = prevtx_item.get("blockhash") funder_bheight = None # type: Optional[int] if funder_bhash is not None: @@ -1402,12 +1417,12 @@ async def _spender_for_txo(self, prev_txhash: bytes, txout_idx: int) -> 'TXOSpen # 2. call bitcoind "gettxspendingprevout" self.bump_cost(1) try: - spender_item = await self.session_mgr.daemon.gettxspendingprevout(prev_txid, txout_idx) + spender_item = await self.session_mgr.daemon.gettxspendingprevout(prev_txid_hum, txout_idx) except DaemonError as e: error, = e.args - self.logger.debug(f"gettxspendingprevout errored. txo={prev_txid}:{txout_idx}. {error=}") + self.logger.debug(f"gettxspendingprevout errored. txo={prev_txid_hum}:{txout_idx}. {error=}") raise RPCError(DAEMON_ERROR, f'daemon error: {error!r}') from None # TODO some callers do not expect this - assert spender_item.get("txid") == prev_txid, f"{spender_item.get('txid')=} != {prev_txid=}" + assert spender_item.get("txid") == prev_txid_hum, f"{spender_item.get('txid')=} != {prev_txid_hum=}" spender_bhash = spender_item.get("blockhash") spender_bheight = None if spender_bhash is not None: @@ -1419,41 +1434,41 @@ async def _spender_for_txo(self, prev_txhash: bytes, txout_idx: int) -> 'TXOSpen # utxo funded, and spent (in-chain) return TXOSpendStatus( prev_height=funder_bheight, - spender_txhash=hex_str_to_hash(spender_txid), + spender_txid_rev=hex_str_to_hash(spender_txid), spender_height=spender_bheight, ) - async def _calc_txoutpoint_status(self, prev_txhash: bytes, txout_idx: int) -> Dict[str, Any]: + async def _calc_txoutpoint_status(self, prev_txid_rev: bytes, txout_idx: int) -> Dict[str, Any]: self.bump_cost(0.2) - spend_status = await self._spender_for_txo(prev_txhash, txout_idx) + spend_status = await self._spender_for_txo(prev_txid_rev, txout_idx) if spend_status.spender_height is not None: # TXO was created, was mined, was spent, and spend was mined. assert spend_status.prev_height > 0 assert spend_status.spender_height > 0 - assert spend_status.spender_txhash is not None + assert spend_status.spender_txid_rev is not None else: - mp_spend_status = await self.mempool.spender_for_txo(prev_txhash, txout_idx) + mp_spend_status = await self.mempool.spender_for_txo(prev_txid_rev, txout_idx) if mp_spend_status.prev_height is not None: spend_status.prev_height = mp_spend_status.prev_height if mp_spend_status.spender_height is not None: spend_status.spender_height = mp_spend_status.spender_height - if mp_spend_status.spender_txhash is not None: - spend_status.spender_txhash = mp_spend_status.spender_txhash + if mp_spend_status.spender_txid_rev is not None: + spend_status.spender_txid_rev = mp_spend_status.spender_txid_rev # convert to json dict the client expects status = {} if spend_status.prev_height is not None: status['height'] = spend_status.prev_height - if spend_status.spender_txhash is not None: + if spend_status.spender_txid_rev is not None: assert spend_status.spender_height is not None - status['spender_txhash'] = hash_to_hex_str(spend_status.spender_txhash) + status['spender_txhash'] = hash_to_hex_str(spend_status.spender_txid_rev) status['spender_height'] = spend_status.spender_height return status - async def txoutpoint_status_for_notif(self, prev_txhash: bytes, txout_idx: int) -> Dict[str, Any]: + async def txoutpoint_status_for_notif(self, prev_txid_rev: bytes, txout_idx: int) -> Dict[str, Any]: """Side-effect: updates client-last-seen status, used by notifications.""" - status = await self._calc_txoutpoint_status(prev_txhash=prev_txhash, txout_idx=txout_idx) + status = await self._calc_txoutpoint_status(prev_txid_rev=prev_txid_rev, txout_idx=txout_idx) # update status last sent to client - prevout = (prev_txhash, txout_idx) + prevout = (prev_txid_rev, txout_idx) prev_height = status.get('height') # type: Optional[int] spender_height = status.get('spender_height') # type: Optional[int] if ((prev_height is not None and prev_height <= 0) @@ -1463,7 +1478,7 @@ async def txoutpoint_status_for_notif(self, prev_txhash: bytes, txout_idx: int) self.mempool_txoutpoint_statuses.pop(prevout, None) return status - async def hashX_listunspent(self, hashX): + async def hashX_listunspent(self, hashX: bytes) -> Sequence[dict[str, Any]]: '''Return the list of UTXOs of a script hash, including mempool effects.''' utxos = await self.db.all_utxos(hashX) @@ -1472,71 +1487,71 @@ async def hashX_listunspent(self, hashX): self.bump_cost(1.0 + len(utxos) / 50) spends = await self.mempool.potential_spends(hashX) - return [{'tx_hash': hash_to_hex_str(utxo.tx_hash), + return [{'tx_hash': hash_to_hex_str(utxo.txid_rev), 'tx_pos': utxo.tx_pos, 'height': utxo.height, 'value': utxo.value} for utxo in utxos - if (utxo.tx_hash, utxo.tx_pos) not in spends] + if (utxo.txid_rev, utxo.tx_pos) not in spends] - async def hashX_subscribe(self, hashX, alias): + async def hashX_subscribe(self, hashX: bytes, alias: str) -> Optional[str]: # Store the subscription only after address_status succeeds result = await self.address_status(hashX) self.hashX_subs[hashX] = alias # TODO rename alias to scripthash return result - async def get_balance(self, hashX): + async def get_balance(self, hashX: bytes) -> dict[str, Any]: utxos = await self.db.all_utxos(hashX) confirmed = sum(utxo.value for utxo in utxos) unconfirmed = await self.mempool.balance_delta(hashX) self.bump_cost(1.0 + len(utxos) / 50) return {'confirmed': confirmed, 'unconfirmed': unconfirmed} - async def scripthash_get_balance(self, scripthash): + async def scripthash_get_balance(self, scripthash: str | Any) -> dict[str, Any]: '''Return the confirmed and unconfirmed balance of a scripthash.''' hashX = scripthash_to_hashX(scripthash) return await self.get_balance(hashX) - async def unconfirmed_history(self, hashX): + async def unconfirmed_history(self, hashX: bytes) -> list[dict[str, Any]]: # Note both confirmed history and mempool history are ordered # height is -1 if it has unconfirmed inputs, otherwise 0 - result = [{'tx_hash': hash_to_hex_str(tx.hash), + result = [{'tx_hash': hash_to_hex_str(tx.txid_rev), 'height': -tx.has_unconfirmed_inputs, 'fee': tx.fee} for tx in await self.mempool.transaction_summaries(hashX)] self.bump_cost(0.25 + len(result) / 50) return result - async def confirmed_and_unconfirmed_history(self, hashX): + async def confirmed_and_unconfirmed_history(self, hashX: bytes) -> list[dict[str, Any]]: # Note both confirmed history and mempool history are ordered history, cost = await self.session_mgr.limited_history(hashX) self.bump_cost(cost) - conf = [{'tx_hash': hash_to_hex_str(tx_hash), 'height': height} - for tx_hash, height in history] + conf = [{'tx_hash': hash_to_hex_str(txid_rev), 'height': height} + for txid_rev, height in history] return conf + await self.unconfirmed_history(hashX) - async def scripthash_get_history(self, scripthash): + async def scripthash_get_history(self, scripthash: str | Any) -> list[dict[str, Any]]: '''Return the confirmed and unconfirmed history of a scripthash.''' hashX = scripthash_to_hashX(scripthash) return await self.confirmed_and_unconfirmed_history(hashX) - async def scripthash_get_mempool(self, scripthash): + async def scripthash_get_mempool(self, scripthash: str | Any) -> list[dict[str, Any]]: '''Return the mempool transactions touching a scripthash.''' hashX = scripthash_to_hashX(scripthash) return await self.unconfirmed_history(hashX) - async def scripthash_listunspent(self, scripthash): + async def scripthash_listunspent(self, scripthash: str | Any) -> Sequence[dict[str, Any]]: '''Return the list of UTXOs of a scripthash.''' hashX = scripthash_to_hashX(scripthash) return await self.hashX_listunspent(hashX) - async def scripthash_subscribe(self, scripthash): + async def scripthash_subscribe(self, scripthash: str | Any) -> Optional[str]: '''Subscribe to a script hash. scripthash: the SHA256 hash of the script to subscribe to''' hashX = scripthash_to_hashX(scripthash) return await self.hashX_subscribe(hashX, scripthash) - async def scripthash_unsubscribe(self, scripthash): + async def scripthash_unsubscribe(self, scripthash: str | Any): '''Unsubscribe from a script hash.''' self.bump_cost(0.1) hashX = scripthash_to_hashX(scripthash) @@ -1566,47 +1581,47 @@ def scriptpubkey_unsubscribe(self, spk: str) -> collections.abc.Awaitable[bool]: scripthash = spk_to_scripthash(spk) return self.scripthash_unsubscribe(scripthash) - async def txoutpoint_get_status(self, tx_hash, txout_idx, spk_hint=None): + async def txoutpoint_get_status(self, tx_hash: str | Any, txout_idx: int | Any, spk_hint=None) -> dict[str, Any]: '''Return the status of an outpoint, without subscribing. spk_hint: scriptPubKey corresponding to the outpoint. Might be used by other servers, but we don't need and hence ignore it. ''' - tx_hash = assert_tx_hash(tx_hash) + txid_rev = assert_txid_hum(tx_hash) txout_idx = non_negative_integer(txout_idx) if spk_hint is not None: assert_hex_str(spk_hint) # calc status (but do not side-effect client-last-seen status) - spend_status = await self._calc_txoutpoint_status(tx_hash, txout_idx) + spend_status = await self._calc_txoutpoint_status(txid_rev, txout_idx) return spend_status - async def txoutpoint_subscribe(self, tx_hash, txout_idx, spk_hint=None): + async def txoutpoint_subscribe(self, tx_hash: str | Any, txout_idx: int | Any, spk_hint=None) -> dict[str, Any]: '''Subscribe to an outpoint. spk_hint: scriptPubKey corresponding to the outpoint. Might be used by other servers, but we don't need and hence ignore it. ''' - tx_hash = assert_tx_hash(tx_hash) + txid_rev = assert_txid_hum(tx_hash) txout_idx = non_negative_integer(txout_idx) if spk_hint is not None: assert_hex_str(spk_hint) # calc status, update client-last-seen status, and sub to outpoint - spend_status = await self.txoutpoint_status_for_notif(tx_hash, txout_idx) - self.txoutpoint_subs.add((tx_hash, txout_idx)) + spend_status = await self.txoutpoint_status_for_notif(txid_rev, txout_idx) + self.txoutpoint_subs.add((txid_rev, txout_idx)) return spend_status - async def txoutpoint_unsubscribe(self, tx_hash, txout_idx): + async def txoutpoint_unsubscribe(self, tx_hash: str | Any, txout_idx: int | Any) -> bool: '''Unsubscribe from an outpoint.''' - tx_hash = assert_tx_hash(tx_hash) + txid_rev = assert_txid_hum(tx_hash) txout_idx = non_negative_integer(txout_idx) self.bump_cost(0.1) - prevout = (tx_hash, txout_idx) + prevout = (txid_rev, txout_idx) was_subscribed = prevout in self.txoutpoint_subs self.txoutpoint_subs.discard(prevout) self.mempool_txoutpoint_statuses.pop(prevout, None) return was_subscribed - async def _merkle_proof(self, cp_height, height): + async def _merkle_proof(self, cp_height: int, height: int) -> dict[str, Any]: max_height = self.db.db_height if not height <= cp_height <= max_height: raise RPCError(BAD_REQUEST, @@ -1620,7 +1635,7 @@ async def _merkle_proof(self, cp_height, height): 'root': hash_to_hex_str(root), } - async def block_header(self, height, cp_height=0): + async def block_header(self, height: int, cp_height: int = 0) -> dict[str, Any]: '''Return a raw block header as a hexadecimal string, or as a dictionary with a merkle proof.''' height = non_negative_integer(height) @@ -1633,7 +1648,7 @@ async def block_header(self, height, cp_height=0): result.update(await self._merkle_proof(cp_height, height)) return result - async def block_headers(self, start_height, count, cp_height=0): + async def block_headers(self, start_height: int, count: int, cp_height: int = 0) -> dict[str, Any]: '''Return count concatenated block headers as hex for the main chain; starting at start_height. @@ -1658,7 +1673,7 @@ async def block_headers(self, start_height, count, cp_height=0): self.bump_cost(cost) return result - async def block_headers_array(self, start_height, count, cp_height=0): + async def block_headers_array(self, start_height: int, count: int, cp_height: int = 0) -> dict[str, Any]: '''Return block headers in an array for the main chain; starting at start_height. start_height and count must be non-negative integers. At most @@ -1690,7 +1705,7 @@ async def block_headers_array(self, start_height, count, cp_height=0): self.bump_cost(cost) return result - def is_tor(self): + def is_tor(self) -> bool: '''Try to detect if the connection is to a tor hidden service we are running.''' proxy_address = self.peer_mgr.proxy_address() @@ -1701,7 +1716,7 @@ def is_tor(self): return False return remote_addr.host == proxy_address.host - async def replaced_banner(self, banner): + async def replaced_banner(self, banner: str) -> str: network_info = await self.daemon_request('getnetworkinfo') ni_version = network_info['version'] # e.g. 290100 (for /Satoshi:29.1.0/) major = ni_version // 10_000 @@ -1718,12 +1733,12 @@ async def replaced_banner(self, banner): banner = banner.replace(*pair) return banner - async def donation_address(self): + async def donation_address(self) -> str: '''Return the donation address as a string, empty if there is none.''' self.bump_cost(0.1) return self.env.donation_address - async def banner(self): + async def banner(self) -> str: '''Return the server banner text.''' banner = f'You are connected to an {electrumx.version} server.' self.bump_cost(0.5) @@ -1769,12 +1784,12 @@ async def mempool_recent(self) -> list[dict[str, Any]]: self.bump_cost(1.0) recent_txs = await self.mempool.get_recently_added_txs(count=10) return [{ - "txid": hash_to_hex_str(tx.hash), + "txid": hash_to_hex_str(tx.txid_rev), "fee": tx.fee, "vsize": tx.vsize, } for tx in recent_txs] - async def estimatefee(self, number, mode=None): + async def estimatefee(self, number: int | Any, mode=None): '''The estimated transaction fee per kilobyte to be paid for a transaction to be included within a certain number of blocks. @@ -1884,7 +1899,7 @@ async def server_version( self.sv_negotiated.set() return electrumx.version, self.protocol_version_string() - async def transaction_broadcast(self, raw_tx): + async def transaction_broadcast(self, raw_tx: str | Any) -> str: '''Broadcast a raw transaction to the network. raw_tx: the raw transaction as a hexadecimal string''' @@ -1892,7 +1907,7 @@ async def transaction_broadcast(self, raw_tx): self.bump_cost(0.25 + len(raw_tx) / 5000) # This returns errors as JSON RPC errors, as is natural try: - hex_hash = await self.session_mgr.broadcast_transaction(raw_tx) + txid_hum = await self.session_mgr.broadcast_transaction(raw_tx) except DaemonError as e: error, = e.args message = error['message'] @@ -1905,14 +1920,14 @@ async def transaction_broadcast(self, raw_tx): if client_ver != (0, ): msg = self.coin.warn_old_client_on_tx_broadcast(client_ver) if msg: - self.logger.info(f'sent tx: {hex_hash}. and warned user to upgrade their ' + self.logger.info(f'sent tx: {txid_hum}. and warned user to upgrade their ' f'client from {self.client}') return msg - self.logger.info(f'sent tx: {hex_hash}') - return hex_hash + self.logger.info(f'sent tx: {txid_hum}') + return txid_hum - async def package_broadcast(self, tx_package: Sequence[str], verbose: bool = False) -> dict: + async def package_broadcast(self, tx_package: Sequence[str] | Any, verbose: bool = False) -> dict[str, Any]: """Broadcast a package of raw transactions to the network (submitpackage). The package must consist of a child with its parents, and none of the parents may depend on one another. @@ -1984,37 +1999,37 @@ async def transaction_testmempoolaccept(self, raw_txs: Sequence[str]) -> Sequenc response.append(new_item) return response - async def transaction_get(self, tx_hash, verbose=False): + async def transaction_get(self, tx_hash: str | Any, verbose=False): '''Return the serialized raw transaction given its hash tx_hash: the transaction hash as a hexadecimal string verbose: passed on to the daemon ''' - assert_tx_hash(tx_hash) + assert_txid_hum(tx_hash) if verbose not in (True, False): raise RPCError(BAD_REQUEST, '"verbose" must be a boolean') self.bump_cost(1.0) return await self.daemon_request('getrawtransaction', tx_hash, verbose) - async def transaction_merkle(self, tx_hash, height): + async def transaction_merkle(self, tx_hash: str | Any, height: int | Any) -> dict[str, Any]: '''Return the merkle branch to a confirmed transaction given its hash and height. tx_hash: the transaction hash as a hexadecimal string height: the height of the block it is in ''' - tx_hash = assert_tx_hash(tx_hash) + txid_rev = assert_txid_hum(tx_hash) height = non_negative_integer(height) - branch, tx_pos, block_header, cost = await self.session_mgr.merkle_branch_for_tx_hash( - tx_hash=tx_hash, height=height) + branch, tx_pos, block_header, cost = await self.session_mgr.merkle_branch_for_txid( + txid_rev=txid_rev, height=height) self.bump_cost(cost) - blockhash = hash_to_hex_str(self.coin.header_hash(block_header)) + blockhash_hum = hash_to_hex_str(self.coin.header_hash_rev(block_header)) return { "block_height": height, - "block_hash": blockhash, + "block_hash": blockhash_hum, "merkle": branch, "pos": tx_pos, } @@ -2029,21 +2044,22 @@ async def transaction_id_from_pos(self, height, tx_pos, merkle=False): raise RPCError(BAD_REQUEST, '"merkle" must be a boolean') if merkle: - branch, tx_hash, cost = await self.session_mgr.merkle_branch_for_tx_pos( + branch, txid_hum, cost = await self.session_mgr.merkle_branch_for_tx_pos( height, tx_pos) self.bump_cost(cost) - return {"tx_hash": tx_hash, "merkle": branch} + return {"tx_hash": txid_hum, "merkle": branch} else: - tx_hashes, cost = await self.session_mgr.tx_hashes_at_blockheight(height) + txids_rev, cost = await self.session_mgr.txids_rev_at_blockheight(height) try: - tx_hash = tx_hashes[tx_pos] + txid_rev = txids_rev[tx_pos] except IndexError: raise RPCError(BAD_REQUEST, f'no tx at position {tx_pos:,d} in block at height {height:,d}') self.bump_cost(cost) - return hash_to_hex_str(tx_hash) + txid_hum = hash_to_hex_str(txid_rev) + return txid_hum - async def compact_fee_histogram(self): + async def compact_fee_histogram(self) -> Sequence[tuple[float, int]]: self.bump_cost(1.0) return await self.mempool.compact_fee_histogram() diff --git a/tests/lib/test_tx_zcoin.py b/tests/lib/test_tx_zcoin.py index 24a0cd26d..8ba4afaa0 100644 --- a/tests/lib/test_tx_zcoin.py +++ b/tests/lib/test_tx_zcoin.py @@ -49,6 +49,6 @@ def test_tx_serialiazation(): test = bytes.fromhex(test) deser_xzc = tx_lib.DeserializerZcoin(test) tx = deser_xzc.read_tx() - assert tx.inputs[0].prev_hash == tx_lib.ZERO + assert tx.inputs[0].prev_txid_rev == tx_lib.ZERO assert tx.inputs[0].prev_idx == tx_lib.MINUS_1 diff --git a/tests/server/test_daemon.py b/tests/server/test_daemon.py index 00e5ac69c..0afdbaada 100644 --- a/tests/server/test_daemon.py +++ b/tests/server/test_daemon.py @@ -271,10 +271,10 @@ async def test_mempool_info(daemon): @pytest.mark.asyncio -async def test_mempool_hashes(daemon): +async def test_mempool_txids_hum(daemon): hashes = ['hex_hash1', 'hex_hash2'] daemon.session = ClientSessionGood(('getrawmempool', [], hashes)) - assert await daemon.mempool_hashes() == hashes + assert await daemon.mempool_txids_hum() == hashes @pytest.mark.asyncio diff --git a/tests/server/test_mempool.py b/tests/server/test_mempool.py index 396c42c24..cb3ab9039 100644 --- a/tests/server/test_mempool.py +++ b/tests/server/test_mempool.py @@ -5,6 +5,7 @@ from collections import defaultdict from functools import partial from random import randrange, choice, seed +from typing import Iterable, Sequence import pytest from aiorpcx import Event, sleep, ignore_after @@ -34,7 +35,7 @@ def random_tx(hash160s, utxos): prevout = choice(list(utxos)) hashX, value = utxos.pop(prevout) inputs.append(TxInput( - prev_hash=prevout[0], + prev_txid_rev=prevout[0], prev_idx=prevout[1], script=b'', sequence=4294967295, @@ -45,7 +46,7 @@ def random_tx(hash160s, utxos): # in some coins if randrange(0, 10) == 0: inputs.append(TxInput( - prev_hash=bytes(32), + prev_txid_rev=bytes(32), prev_idx=4294967295, script=b'', sequence=4294967295, @@ -61,10 +62,10 @@ def random_tx(hash160s, utxos): pk_script = coin.hash160_to_P2PKH_script(choice(hash160s)) outputs.append(TxOutput(value=value, pk_script=pk_script)) - tx = Tx(version=2, inputs=inputs, outputs=outputs, locktime=0, txid=None, wtxid=None) + tx = Tx(version=2, inputs=inputs, outputs=outputs, locktime=0, txid_rev=None, wtxid_rev=None) tx_bytes = tx.serialize() tx_hash = tx_hash_fn(tx_bytes) - tx.txid = tx.wtxid = tx_hash + tx.txid_rev = tx.wtxid_rev = tx_hash for n, output in enumerate(tx.outputs): utxos[(tx_hash, n)] = (coin.hashX_from_script(output.pk_script), output.value) @@ -115,7 +116,7 @@ def mempool_utxos(self): return utxos def mempool_spends(self): - return [(input.prev_hash, input.prev_idx) + return [(input.prev_txid_rev, input.prev_idx) for tx in self.txs.values() for input in tx.inputs if not input.is_generation()] @@ -127,7 +128,7 @@ def balance_deltas(self): for n, input in enumerate(tx.inputs): if input.is_generation(): continue - prevout = (input.prev_hash, input.prev_idx) + prevout = (input.prev_txid_rev, input.prev_idx) if prevout in utxos: utxos.pop(prevout) else: @@ -145,7 +146,7 @@ def spends(self): for n, input in enumerate(tx.inputs): if input.is_generation(): continue - prevout = (input.prev_hash, input.prev_idx) + prevout = (input.prev_txid_rev, input.prev_idx) if prevout in utxos: hashX, value = utxos.pop(prevout) else: @@ -164,8 +165,8 @@ def summaries(self): for n, input in enumerate(tx.inputs): if input.is_generation(): continue - has_ui = has_ui or (input.prev_hash in self.txs) - prevout = (input.prev_hash, input.prev_idx) + has_ui = has_ui or (input.prev_txid_rev in self.txs) + prevout = (input.prev_txid_rev, input.prev_idx) if prevout in utxos: hashX, value = utxos[prevout] else: @@ -190,7 +191,7 @@ def touched(self, tx_hashes): for n, input in enumerate(tx.inputs): if input.is_generation(): continue - prevout = (input.prev_hash, input.prev_idx) + prevout = (input.prev_txid_rev, input.prev_idx) if prevout in utxos: hashX, value = utxos[prevout] else: @@ -221,14 +222,14 @@ def db_height(self): def cached_height(self): return self._cached_height - async def mempool_hashes(self): + async def mempool_txids_hum(self): await sleep(0) return [hash_to_hex_str(hash) for hash in self.txs] - async def raw_transactions(self, hex_hashes): + async def raw_transactions(self, txids_hum: Iterable[str]) -> Sequence[bytes | None]: await sleep(0) - hashes = [hex_str_to_hash(hex_hash) for hex_hash in hex_hashes] - return [self.raw_txs.get(hash) for hash in hashes] + txids_rev = [hex_str_to_hash(hex_hash) for hex_hash in txids_hum] + return [self.raw_txs.get(txid_rev) for txid_rev in txids_rev] async def lookup_utxos(self, prevouts): await sleep(0) @@ -246,13 +247,13 @@ def __init__(self, drop_count): self.drop_count = drop_count self.dropped = False - async def raw_transactions(self, hex_hashes): + async def raw_transactions(self, txids_hum: Iterable[str]) -> Sequence[bytes | None]: if not self.dropped: self.dropped = True for hash in self.ordered_adds[-self.drop_count:]: del self.raw_txs[hash] del self.txs[hash] - return await super().raw_transactions(hex_hashes) + return await super().raw_transactions(txids_hum) def in_caplog(caplog, message): @@ -383,7 +384,7 @@ async def _test_summaries(mempool, api): summaries = api.summaries() for hashX in api.hashXs: mempool_result = await mempool.transaction_summaries(hashX) - mempool_result = [(item.hash, item.fee, item.has_unconfirmed_inputs) + mempool_result = [(item.txid_rev, item.fee, item.has_unconfirmed_inputs) for item in mempool_result] our_result = summaries.get(hashX, []) assert set(our_result) == set(mempool_result) @@ -572,7 +573,7 @@ async def test_get_recently_added_txs(): recent_txs = await mempool.get_recently_added_txs(count=10) start_idx = max(0, cur_size-10) recent_adds = api.ordered_adds[start_idx:cur_size] - assert [tx.hash for tx in recent_txs][::-1] == recent_adds + assert [tx.txid_rev for tx in recent_txs][::-1] == recent_adds await group.cancel_remaining() diff --git a/tests/test_blocks.py b/tests/test_blocks.py index 3a56ba064..d51abd34e 100644 --- a/tests/test_blocks.py +++ b/tests/test_blocks.py @@ -95,15 +95,15 @@ def test_block(block_details): block = coin.block(raw_block, block_info['height']) try: - assert coin.header_hash( + assert coin.header_hash_rev( block.header) == hex_str_to_hash(block_info['hash']) except ImportError as e: pytest.skip(str(e)) - assert (coin.header_prevhash(block.header) + assert (coin.header_prevhash_rev(block.header) == hex_str_to_hash(block_info['previousblockhash'])) assert len(block_info['tx']) == len(block.transactions) for n, tx in enumerate(block.transactions): - assert tx.txid == hex_str_to_hash(block_info['tx'][n]) + assert tx.txid_rev == hex_str_to_hash(block_info['tx'][n]) def test_all_coins_are_covered(): diff --git a/tests/test_transactions.py b/tests/test_transactions.py index d25261fa5..2adf9e902 100644 --- a/tests/test_transactions.py +++ b/tests/test_transactions.py @@ -44,16 +44,16 @@ def test_transaction_bitcoin(transaction_details_bitcoin): raw_tx = unhexlify(tx_info['hex']) tx, vsize = coin.DESERIALIZER(raw_tx, 0).read_tx_and_vsize() - tx_hash = tx.txid + tx_hash = tx.txid_rev assert tx_info['txid'] == hash_to_hex_str(tx_hash) - assert tx_info['hash'] == hash_to_hex_str(tx.wtxid) + assert tx_info['hash'] == hash_to_hex_str(tx.wtxid_rev) assert tx_info['version'] == tx.version assert tx_info['vsize'] == vsize vin = tx_info['vin'] for i in range(len(vin)): - assert vin[i]['txid'] == hash_to_hex_str(tx.inputs[i].prev_hash) + assert vin[i]['txid'] == hash_to_hex_str(tx.inputs[i].prev_txid_rev) assert vin[i]['vout'] == tx.inputs[i].prev_idx if "txinwitness" in vin[i] or (hasattr(tx, "witness") and tx.witness[i]): assert vin[i]["txinwitness"] == [x.hex() for x in tx.witness[i]] @@ -80,14 +80,14 @@ def test_transaction_alts(transaction_details_altcoin): raw_tx = unhexlify(tx_info['hex']) tx = coin.DESERIALIZER(raw_tx, 0).read_tx() - tx_hash = tx.txid + tx_hash = tx.txid_rev assert tx_info['txid'] == hash_to_hex_str(tx_hash) if expected_wtxid := tx_info.get('hash'): - assert expected_wtxid == hash_to_hex_str(tx.wtxid) + assert expected_wtxid == hash_to_hex_str(tx.wtxid_rev) vin = tx_info['vin'] for i in range(len(vin)): - assert vin[i]['txid'] == hash_to_hex_str(tx.inputs[i].prev_hash) + assert vin[i]['txid'] == hash_to_hex_str(tx.inputs[i].prev_txid_rev) assert vin[i]['vout'] == tx.inputs[i].prev_idx vout = tx_info['vout'] From 561dcedbe0b77ffed73220437792bbe06bc5a5b1 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Tue, 12 May 2026 15:01:14 +0000 Subject: [PATCH 18/27] session: prefix protocol-handler method names with "phandle_" e.g. to make it clear that these are methods that really need to do input sanitization --- src/electrumx/server/session.py | 211 ++++++++++++++++---------------- 1 file changed, 105 insertions(+), 106 deletions(-) diff --git a/src/electrumx/server/session.py b/src/electrumx/server/session.py index 242fd8749..cced48032 100644 --- a/src/electrumx/server/session.py +++ b/src/electrumx/server/session.py @@ -1190,7 +1190,7 @@ def server_features(cls, env: 'Env') -> dict[str, Any]: 'services': [str(service) for service in env.report_services], } - async def server_features_async(self) -> dict[str, Any]: + async def phandle_server_features_async(self) -> dict[str, Any]: self.bump_cost(0.2) return self.server_features(self.env) @@ -1321,19 +1321,19 @@ async def subscribe_headers_result(self): '''The result of a header subscription or notification.''' return self.session_mgr.hsub_results - async def headers_subscribe(self): + async def phandle_headers_subscribe(self): '''Subscribe to get raw headers of new blocks.''' self.subscribe_headers = True self.bump_cost(0.25) return await self.subscribe_headers_result() - async def add_peer(self, features: dict[str, Any] | Any): + async def phandle_add_peer(self, features: dict[str, Any] | Any): '''Add a peer (but only if the peer resolves to the source).''' self.is_peer = True self.bump_cost(100.0) return await self.peer_mgr.on_add_peer(features, self.remote_address()) - async def peers_subscribe(self): + async def phandle_peers_subscribe(self): '''Return the server peers as a list of (ip, host, details) tuples.''' self.bump_cost(1.0) return self.peer_mgr.on_peers_subscribe(self.is_tor()) @@ -1506,7 +1506,7 @@ async def get_balance(self, hashX: bytes) -> dict[str, Any]: self.bump_cost(1.0 + len(utxos) / 50) return {'confirmed': confirmed, 'unconfirmed': unconfirmed} - async def scripthash_get_balance(self, scripthash: str | Any) -> dict[str, Any]: + async def phandle_scripthash_get_balance(self, scripthash: str | Any) -> dict[str, Any]: '''Return the confirmed and unconfirmed balance of a scripthash.''' hashX = scripthash_to_hashX(scripthash) return await self.get_balance(hashX) @@ -1529,59 +1529,59 @@ async def confirmed_and_unconfirmed_history(self, hashX: bytes) -> list[dict[str for txid_rev, height in history] return conf + await self.unconfirmed_history(hashX) - async def scripthash_get_history(self, scripthash: str | Any) -> list[dict[str, Any]]: + async def phandle_scripthash_get_history(self, scripthash: str | Any) -> list[dict[str, Any]]: '''Return the confirmed and unconfirmed history of a scripthash.''' hashX = scripthash_to_hashX(scripthash) return await self.confirmed_and_unconfirmed_history(hashX) - async def scripthash_get_mempool(self, scripthash: str | Any) -> list[dict[str, Any]]: + async def phandle_scripthash_get_mempool(self, scripthash: str | Any) -> list[dict[str, Any]]: '''Return the mempool transactions touching a scripthash.''' hashX = scripthash_to_hashX(scripthash) return await self.unconfirmed_history(hashX) - async def scripthash_listunspent(self, scripthash: str | Any) -> Sequence[dict[str, Any]]: + async def phandle_scripthash_listunspent(self, scripthash: str | Any) -> Sequence[dict[str, Any]]: '''Return the list of UTXOs of a scripthash.''' hashX = scripthash_to_hashX(scripthash) return await self.hashX_listunspent(hashX) - async def scripthash_subscribe(self, scripthash: str | Any) -> Optional[str]: + async def phandle_scripthash_subscribe(self, scripthash: str | Any) -> Optional[str]: '''Subscribe to a script hash. scripthash: the SHA256 hash of the script to subscribe to''' hashX = scripthash_to_hashX(scripthash) return await self.hashX_subscribe(hashX, scripthash) - async def scripthash_unsubscribe(self, scripthash: str | Any): + async def phandle_scripthash_unsubscribe(self, scripthash: str | Any): '''Unsubscribe from a script hash.''' self.bump_cost(0.1) hashX = scripthash_to_hashX(scripthash) return self.unsubscribe_hashX(hashX) is not None - def scriptpubkey_get_balance(self, spk: str) -> collections.abc.Awaitable[dict]: + def phandle_scriptpubkey_get_balance(self, spk: str) -> collections.abc.Awaitable[dict]: scripthash = spk_to_scripthash(spk) - return self.scripthash_get_balance(scripthash) + return self.phandle_scripthash_get_balance(scripthash) - def scriptpubkey_get_history(self, spk: str) -> collections.abc.Awaitable[list]: + def phandle_scriptpubkey_get_history(self, spk: str) -> collections.abc.Awaitable[list]: scripthash = spk_to_scripthash(spk) - return self.scripthash_get_history(scripthash) + return self.phandle_scripthash_get_history(scripthash) - def scriptpubkey_get_mempool(self, spk: str) -> collections.abc.Awaitable[list]: + def phandle_scriptpubkey_get_mempool(self, spk: str) -> collections.abc.Awaitable[list]: scripthash = spk_to_scripthash(spk) - return self.scripthash_get_mempool(scripthash) + return self.phandle_scripthash_get_mempool(scripthash) - def scriptpubkey_listunspent(self, spk: str) -> collections.abc.Awaitable[list]: + def phandle_scriptpubkey_listunspent(self, spk: str) -> collections.abc.Awaitable[list]: scripthash = spk_to_scripthash(spk) - return self.scripthash_listunspent(scripthash) + return self.phandle_scripthash_listunspent(scripthash) - def scriptpubkey_subscribe(self, spk: str) -> collections.abc.Awaitable[Optional[str]]: + def phandle_scriptpubkey_subscribe(self, spk: str) -> collections.abc.Awaitable[Optional[str]]: scripthash = spk_to_scripthash(spk) - return self.scripthash_subscribe(scripthash) + return self.phandle_scripthash_subscribe(scripthash) - def scriptpubkey_unsubscribe(self, spk: str) -> collections.abc.Awaitable[bool]: + def phandle_scriptpubkey_unsubscribe(self, spk: str) -> collections.abc.Awaitable[bool]: scripthash = spk_to_scripthash(spk) - return self.scripthash_unsubscribe(scripthash) + return self.phandle_scripthash_unsubscribe(scripthash) - async def txoutpoint_get_status(self, tx_hash: str | Any, txout_idx: int | Any, spk_hint=None) -> dict[str, Any]: + async def phandle_txoutpoint_get_status(self, tx_hash: str | Any, txout_idx: int | Any, spk_hint=None) -> dict[str, Any]: '''Return the status of an outpoint, without subscribing. spk_hint: scriptPubKey corresponding to the outpoint. Might be used by @@ -1595,7 +1595,7 @@ async def txoutpoint_get_status(self, tx_hash: str | Any, txout_idx: int | Any, spend_status = await self._calc_txoutpoint_status(txid_rev, txout_idx) return spend_status - async def txoutpoint_subscribe(self, tx_hash: str | Any, txout_idx: int | Any, spk_hint=None) -> dict[str, Any]: + async def phandle_txoutpoint_subscribe(self, tx_hash: str | Any, txout_idx: int | Any, spk_hint=None) -> dict[str, Any]: '''Subscribe to an outpoint. spk_hint: scriptPubKey corresponding to the outpoint. Might be used by @@ -1610,7 +1610,7 @@ async def txoutpoint_subscribe(self, tx_hash: str | Any, txout_idx: int | Any, s self.txoutpoint_subs.add((txid_rev, txout_idx)) return spend_status - async def txoutpoint_unsubscribe(self, tx_hash: str | Any, txout_idx: int | Any) -> bool: + async def phandle_txoutpoint_unsubscribe(self, tx_hash: str | Any, txout_idx: int | Any) -> bool: '''Unsubscribe from an outpoint.''' txid_rev = assert_txid_hum(tx_hash) txout_idx = non_negative_integer(txout_idx) @@ -1635,7 +1635,7 @@ async def _merkle_proof(self, cp_height: int, height: int) -> dict[str, Any]: 'root': hash_to_hex_str(root), } - async def block_header(self, height: int, cp_height: int = 0) -> dict[str, Any]: + async def phandle_block_header(self, height: int, cp_height: int = 0) -> dict[str, Any]: '''Return a raw block header as a hexadecimal string, or as a dictionary with a merkle proof.''' height = non_negative_integer(height) @@ -1648,7 +1648,7 @@ async def block_header(self, height: int, cp_height: int = 0) -> dict[str, Any]: result.update(await self._merkle_proof(cp_height, height)) return result - async def block_headers(self, start_height: int, count: int, cp_height: int = 0) -> dict[str, Any]: + async def phandle_block_headers(self, start_height: int, count: int, cp_height: int = 0) -> dict[str, Any]: '''Return count concatenated block headers as hex for the main chain; starting at start_height. @@ -1733,12 +1733,12 @@ async def replaced_banner(self, banner: str) -> str: banner = banner.replace(*pair) return banner - async def donation_address(self) -> str: + async def phandle_donation_address(self) -> str: '''Return the donation address as a string, empty if there is none.''' self.bump_cost(0.1) return self.env.donation_address - async def banner(self) -> str: + async def phandle_banner(self) -> str: '''Return the server banner text.''' banner = f'You are connected to an {electrumx.version} server.' self.bump_cost(0.5) @@ -1758,13 +1758,13 @@ async def banner(self) -> str: return banner - async def relayfee(self): + async def phandle_relayfee(self): """The minimum fee required for a transaction to be relayed on by the daemon to the bitcoin network. Doesn't guarantee mempool acceptance.""" self.bump_cost(1.0) return await self.daemon_request('relayfee') - async def mempool_info(self) -> dict[str, float]: + async def phandle_mempool_info(self) -> dict[str, float]: """ mempool.get_info, introduced in protocol 1.6. returns: { @@ -1776,7 +1776,7 @@ async def mempool_info(self) -> dict[str, float]: self.bump_cost(1.0) return await self.daemon_request('mempool_info') - async def mempool_recent(self) -> list[dict[str, Any]]: + async def phandle_mempool_recent(self) -> list[dict[str, Any]]: """ mempool.recent, introduced in protocol 1.6.1. Return a list of the last 10 transactions to enter the mempool. @@ -1789,7 +1789,7 @@ async def mempool_recent(self) -> list[dict[str, Any]]: "vsize": tx.vsize, } for tx in recent_txs] - async def estimatefee(self, number: int | Any, mode=None): + async def phandle_estimatefee(self, number: int | Any, mode=None): '''The estimated transaction fee per kilobyte to be paid for a transaction to be included within a certain number of blocks. @@ -1832,7 +1832,7 @@ async def estimatefee(self, number: int | Any, mode=None): cache[(number, mode)] = (blockhash, feerate, lock) return feerate - async def ping(self, pong_len=0, data=""): + async def phandle_ping(self, pong_len=0, data=""): '''Serves as a connection keep-alive mechanism and for the client to confirm the server is still responding. It can also be used to obfuscate traffic patterns. @@ -1847,12 +1847,12 @@ async def ping(self, pong_len=0, data=""): pong_data = pong_len * "0" return {"data": pong_data} - async def on_ping_notification(self, data=""): + async def phandle_on_ping_notification(self, data=""): self.bump_cost(0.1) # note: the bw cost for receiving 'data' has already been incurred assert_hex_str(data) # nothing to do - async def server_version( + async def phandle_server_version( self, client_name='', protocol_version=None, @@ -1899,7 +1899,7 @@ async def server_version( self.sv_negotiated.set() return electrumx.version, self.protocol_version_string() - async def transaction_broadcast(self, raw_tx: str | Any) -> str: + async def phandle_transaction_broadcast(self, raw_tx: str | Any) -> str: '''Broadcast a raw transaction to the network. raw_tx: the raw transaction as a hexadecimal string''' @@ -1927,7 +1927,7 @@ async def transaction_broadcast(self, raw_tx: str | Any) -> str: self.logger.info(f'sent tx: {txid_hum}') return txid_hum - async def package_broadcast(self, tx_package: Sequence[str] | Any, verbose: bool = False) -> dict[str, Any]: + async def phandle_package_broadcast(self, tx_package: Sequence[str] | Any, verbose: bool = False) -> dict[str, Any]: """Broadcast a package of raw transactions to the network (submitpackage). The package must consist of a child with its parents, and none of the parents may depend on one another. @@ -1968,7 +1968,7 @@ async def package_broadcast(self, tx_package: Sequence[str] | Any, verbose: bool response['errors'] = errors return response - async def transaction_testmempoolaccept(self, raw_txs: Sequence[str]) -> Sequence[dict]: + async def phandle_transaction_testmempoolaccept(self, raw_txs: Sequence[str]) -> Sequence[dict]: """Returns result of mempool acceptance tests indicating if txs would be accepted by mempool. raw_txs: a list of raw transactions as hexadecimal strings @@ -1999,7 +1999,7 @@ async def transaction_testmempoolaccept(self, raw_txs: Sequence[str]) -> Sequenc response.append(new_item) return response - async def transaction_get(self, tx_hash: str | Any, verbose=False): + async def phandle_transaction_get(self, tx_hash: str | Any, verbose=False): '''Return the serialized raw transaction given its hash tx_hash: the transaction hash as a hexadecimal string @@ -2012,7 +2012,7 @@ async def transaction_get(self, tx_hash: str | Any, verbose=False): self.bump_cost(1.0) return await self.daemon_request('getrawtransaction', tx_hash, verbose) - async def transaction_merkle(self, tx_hash: str | Any, height: int | Any) -> dict[str, Any]: + async def phandle_transaction_merkle(self, tx_hash: str | Any, height: int | Any) -> dict[str, Any]: '''Return the merkle branch to a confirmed transaction given its hash and height. @@ -2034,7 +2034,7 @@ async def transaction_merkle(self, tx_hash: str | Any, height: int | Any) -> dic "pos": tx_pos, } - async def transaction_id_from_pos(self, height, tx_pos, merkle=False): + async def phandle_transaction_id_from_pos(self, height, tx_pos, merkle=False): '''Return the txid and optionally a merkle proof, given a block height and position in the block. ''' @@ -2059,7 +2059,7 @@ async def transaction_id_from_pos(self, height, tx_pos, merkle=False): txid_hum = hash_to_hex_str(txid_rev) return txid_hum - async def compact_fee_histogram(self) -> Sequence[tuple[float, int]]: + async def phandle_compact_fee_histogram(self) -> Sequence[tuple[float, int]]: self.bump_cost(1.0) return await self.mempool.compact_fee_histogram() @@ -2067,55 +2067,55 @@ def set_request_handlers(self, ptuple): self.protocol_tuple = ptuple handlers = { - 'blockchain.block.header': self.block_header, - 'blockchain.block.headers': self.block_headers, - 'blockchain.estimatefee': self.estimatefee, - 'blockchain.headers.subscribe': self.headers_subscribe, - 'blockchain.transaction.broadcast': self.transaction_broadcast, - 'blockchain.transaction.get': self.transaction_get, - 'blockchain.transaction.get_merkle': self.transaction_merkle, - 'blockchain.transaction.id_from_pos': self.transaction_id_from_pos, - 'mempool.get_fee_histogram': self.compact_fee_histogram, - 'server.add_peer': self.add_peer, - 'server.banner': self.banner, - 'server.donation_address': self.donation_address, - 'server.features': self.server_features_async, - 'server.peers.subscribe': self.peers_subscribe, - 'server.ping': self.ping, - 'server.version': self.server_version, + 'blockchain.block.header': self.phandle_block_header, + 'blockchain.block.headers': self.phandle_block_headers, + 'blockchain.estimatefee': self.phandle_estimatefee, + 'blockchain.headers.subscribe': self.phandle_headers_subscribe, + 'blockchain.transaction.broadcast': self.phandle_transaction_broadcast, + 'blockchain.transaction.get': self.phandle_transaction_get, + 'blockchain.transaction.get_merkle': self.phandle_transaction_merkle, + 'blockchain.transaction.id_from_pos': self.phandle_transaction_id_from_pos, + 'mempool.get_fee_histogram': self.phandle_compact_fee_histogram, + 'server.add_peer': self.phandle_add_peer, + 'server.banner': self.phandle_banner, + 'server.donation_address': self.phandle_donation_address, + 'server.features': self.phandle_server_features_async, + 'server.peers.subscribe': self.phandle_peers_subscribe, + 'server.ping': self.phandle_ping, + 'server.version': self.phandle_server_version, } notif_handlers = {} if ptuple < (1, 7): - handlers['blockchain.scripthash.get_balance'] = self.scripthash_get_balance - handlers['blockchain.scripthash.get_history'] = self.scripthash_get_history - handlers['blockchain.scripthash.get_mempool'] = self.scripthash_get_mempool - handlers['blockchain.scripthash.listunspent'] = self.scripthash_listunspent - handlers['blockchain.scripthash.subscribe'] = self.scripthash_subscribe + handlers['blockchain.scripthash.get_balance'] = self.phandle_scripthash_get_balance + handlers['blockchain.scripthash.get_history'] = self.phandle_scripthash_get_history + handlers['blockchain.scripthash.get_mempool'] = self.phandle_scripthash_get_mempool + handlers['blockchain.scripthash.listunspent'] = self.phandle_scripthash_listunspent + handlers['blockchain.scripthash.subscribe'] = self.phandle_scripthash_subscribe if (1, 4, 2) <= ptuple < (1, 7): - handlers['blockchain.scripthash.unsubscribe'] = self.scripthash_unsubscribe + handlers['blockchain.scripthash.unsubscribe'] = self.phandle_scripthash_unsubscribe if ptuple >= (1, 6): - handlers['blockchain.transaction.broadcast_package'] = self.package_broadcast - handlers['mempool.get_info'] = self.mempool_info + handlers['blockchain.transaction.broadcast_package'] = self.phandle_package_broadcast + handlers['mempool.get_info'] = self.phandle_mempool_info else: - handlers['blockchain.relayfee'] = self.relayfee # removed in 1.6 + handlers['blockchain.relayfee'] = self.phandle_relayfee # removed in 1.6 # experimental: if ptuple >= (1, 7): - handlers['blockchain.transaction.testmempoolaccept'] = self.transaction_testmempoolaccept - handlers['blockchain.outpoint.subscribe'] = self.txoutpoint_subscribe - handlers['blockchain.outpoint.get_status'] = self.txoutpoint_get_status - handlers['blockchain.outpoint.unsubscribe'] = self.txoutpoint_unsubscribe - handlers['blockchain.scriptpubkey.get_balance'] = self.scriptpubkey_get_balance - handlers['blockchain.scriptpubkey.get_history'] = self.scriptpubkey_get_history - handlers['blockchain.scriptpubkey.get_mempool'] = self.scriptpubkey_get_mempool - handlers['blockchain.scriptpubkey.listunspent'] = self.scriptpubkey_listunspent - handlers['blockchain.scriptpubkey.subscribe'] = self.scriptpubkey_subscribe - handlers['blockchain.scriptpubkey.unsubscribe'] = self.scriptpubkey_unsubscribe - handlers['mempool.recent'] = self.mempool_recent - notif_handlers['server.ping'] = self.on_ping_notification + handlers['blockchain.transaction.testmempoolaccept'] = self.phandle_transaction_testmempoolaccept + handlers['blockchain.outpoint.subscribe'] = self.phandle_txoutpoint_subscribe + handlers['blockchain.outpoint.get_status'] = self.phandle_txoutpoint_get_status + handlers['blockchain.outpoint.unsubscribe'] = self.phandle_txoutpoint_unsubscribe + handlers['blockchain.scriptpubkey.get_balance'] = self.phandle_scriptpubkey_get_balance + handlers['blockchain.scriptpubkey.get_history'] = self.phandle_scriptpubkey_get_history + handlers['blockchain.scriptpubkey.get_mempool'] = self.phandle_scriptpubkey_get_mempool + handlers['blockchain.scriptpubkey.listunspent'] = self.phandle_scriptpubkey_listunspent + handlers['blockchain.scriptpubkey.subscribe'] = self.phandle_scriptpubkey_subscribe + handlers['blockchain.scriptpubkey.unsubscribe'] = self.phandle_scriptpubkey_unsubscribe + handlers['mempool.recent'] = self.phandle_mempool_recent + notif_handlers['server.ping'] = self.phandle_on_ping_notification self.request_handlers = handlers self.notification_handlers = notif_handlers @@ -2156,12 +2156,11 @@ def __init__(self, *args, **kwargs): def set_request_handlers(self, ptuple): super().set_request_handlers(ptuple) self.request_handlers.update({ - 'masternode.announce.broadcast': - self.masternode_announce_broadcast, - 'masternode.subscribe': self.masternode_subscribe, - 'masternode.list': self.masternode_list, - 'protx.diff': self.protx_diff, - 'protx.info': self.protx_info, + 'masternode.announce.broadcast': self.phandle_masternode_announce_broadcast, + 'masternode.subscribe': self.phandle_masternode_subscribe, + 'masternode.list': self.phandle_masternode_list, + 'protx.diff': self.phandle_protx_diff, + 'protx.info': self.phandle_protx_info, }) async def _notify_inner( @@ -2184,7 +2183,7 @@ async def _notify_inner( (mn, status.get(mn))) # Masternode command handlers - async def masternode_announce_broadcast(self, signmnb): + async def phandle_masternode_announce_broadcast(self, signmnb): '''Pass through the masternode announce message to be broadcast by the daemon. @@ -2199,7 +2198,7 @@ async def masternode_announce_broadcast(self, signmnb): raise RPCError(BAD_REQUEST, 'the masternode broadcast was ' f'rejected.\n\n{message}\n[{signmnb}]') - async def masternode_subscribe(self, collateral): + async def phandle_masternode_subscribe(self, collateral): '''Returns the status of masternode. collateral: masternode collateral. @@ -2211,7 +2210,7 @@ async def masternode_subscribe(self, collateral): return result.get(collateral) return None - async def masternode_list(self, payees): + async def phandle_masternode_list(self, payees): ''' Returns the list of masternodes. @@ -2308,7 +2307,7 @@ def get_payment_position(payment_queue, address): else: return cache - async def protx_diff(self, base_height, height): + async def phandle_protx_diff(self, base_height, height): ''' Calculates a diff between two deterministic masternode lists. The result also contains proof data. @@ -2330,7 +2329,7 @@ async def protx_diff(self, base_height, height): return await self.daemon_request('protx', ('diff', base_height, height)) - async def protx_info(self, protx_hash): + async def phandle_protx_info(self, protx_hash): ''' Returns detailed information about a deterministic masternode. @@ -2351,18 +2350,18 @@ class SmartCashElectrumX(DashElectrumX): def set_request_handlers(self, ptuple): super().set_request_handlers(ptuple) self.request_handlers.update({ - 'smartrewards.current': self.smartrewards_current, - 'smartrewards.check': self.smartrewards_check + 'smartrewards.current': self.phandle_smartrewards_current, + 'smartrewards.check': self.phandle_smartrewards_check }) - async def smartrewards_current(self): + async def phandle_smartrewards_current(self): '''Returns the current smartrewards info.''' result = await self.daemon_request('smartrewards', ('current',)) if result is not None: return result return None - async def smartrewards_check(self, addr): + async def phandle_smartrewards_check(self, addr): ''' Returns the status of an address @@ -2375,8 +2374,8 @@ async def smartrewards_check(self, addr): class AuxPoWElectrumX(ElectrumX): - async def block_header(self, height, cp_height=0): - result = await super().block_header(height, cp_height) + async def phandle_block_header(self, height, cp_height=0): + result = await super().phandle_block_header(height, cp_height) # Older protocol versions don't truncate AuxPoW if self.protocol_tuple < (1, 4, 1): @@ -2390,14 +2389,14 @@ async def block_header(self, height, cp_height=0): result['header'] = self.truncate_auxpow_single(result['header']) return result - async def block_headers(self, start_height, count, cp_height=0): + async def phandle_block_headers(self, start_height, count, cp_height=0): # Older protocol versions don't truncate AuxPoW if self.protocol_tuple < (1, 4, 1): - return await super().block_headers(start_height, count, cp_height) + return await super().phandle_block_headers(start_height, count, cp_height) # Not covered by a checkpoint; return full AuxPoW data if cp_height == 0: - return await super().block_headers(start_height, count, cp_height) + return await super().phandle_block_headers(start_height, count, cp_height) result = await super().block_headers_array(start_height, count, cp_height) @@ -2429,10 +2428,10 @@ def set_request_handlers(self, ptuple): super().set_request_handlers(ptuple) if ptuple >= (1, 4, 3): - self.request_handlers['blockchain.name.get_value_proof'] = self.name_get_value_proof + self.request_handlers['blockchain.name.get_value_proof'] = self.phandle_name_get_value_proof - async def name_get_value_proof(self, scripthash, cp_height=0): - history = await self.scripthash_get_history(scripthash) + async def phandle_name_get_value_proof(self, scripthash, cp_height=0): + history = await self.phandle_scripthash_get_history(scripthash) trimmed_history = [] prev_height = None @@ -2446,16 +2445,16 @@ async def name_get_value_proof(self, scripthash, cp_height=0): and height < prev_height - self.coin.NAME_EXPIRATION): break - tx = await self.transaction_get(txid) + tx = await self.phandle_transaction_get(txid) update['tx'] = tx del update['tx_hash'] - tx_merkle = await self.transaction_merkle(txid, height) + tx_merkle = await self.phandle_transaction_merkle(txid, height) del tx_merkle['block_height'] update['tx_merkle'] = tx_merkle if height <= cp_height: - header = await self.block_header(height, cp_height) + header = await self.phandle_block_header(height, cp_height) update['header'] = header trimmed_history.append(update) From 9ecc93eafef58ecf81e03e1c57c6e3b7fe9c0d98 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Tue, 12 May 2026 15:10:45 +0000 Subject: [PATCH 19/27] deps: rm 'attrs', use stdlib dataclasses instead --- pyproject.toml | 1 - src/electrumx/server/db.py | 19 +++++++++---------- src/electrumx/server/mempool.py | 21 ++++++++++----------- src/electrumx/server/session.py | 22 +++++++++++----------- 4 files changed, 30 insertions(+), 33 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 606bb6a2c..b516f3953 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,7 +14,6 @@ license = {'file'="LICENSE"} requires-python = ">=3.10" dependencies = [ "aiorpcx[ws]>=0.25.0,<0.26", - "attrs", # TODO try to use stdlib dataclasses instead "plyvel", "aiohttp>=3.3,<4", ] diff --git a/src/electrumx/server/db.py b/src/electrumx/server/db.py index 864419527..229f0a378 100644 --- a/src/electrumx/server/db.py +++ b/src/electrumx/server/db.py @@ -18,7 +18,6 @@ from glob import glob from typing import Dict, List, Sequence, Tuple, Optional, TYPE_CHECKING, Union -import attr from aiorpcx import run_in_thread, sleep import electrumx.lib.util as util @@ -47,17 +46,17 @@ class UTXO: value: int # in satoshis -@attr.s(slots=True) +@dataclass(slots=True) class FlushData: - height = attr.ib() - tx_count = attr.ib() - headers = attr.ib() - block_txids_rev = attr.ib() # type: List[bytes] + height: int + tx_count: int + headers: list[bytes] + block_txids_rev: list[bytes] # The following are flushed to the UTXO DB if undo_infos is not None - undo_infos = attr.ib() # type: List[Tuple[Sequence[bytes], int]] - adds = attr.ib() # type: Dict[bytes, bytes] # txid+out_idx -> hashX+tx_num+value_sats - deletes = attr.ib() # type: List[bytes] # b'h' db keys, and b'u' db keys - tip = attr.ib() + undo_infos: list[tuple[Sequence[bytes], int]] + adds: dict[bytes, bytes] # txid+out_idx -> hashX+tx_num+value_sats + deletes: list[bytes] # b'h' db keys, and b'u' db keys + tip: bytes COMP_TXID_LEN = 4 diff --git a/src/electrumx/server/mempool.py b/src/electrumx/server/mempool.py index d49a4a582..7cdf7bec9 100644 --- a/src/electrumx/server/mempool.py +++ b/src/electrumx/server/mempool.py @@ -17,7 +17,6 @@ from typing import Sequence, Tuple, TYPE_CHECKING, Type, Dict, Optional, Set, Iterable import math -import attr from aiorpcx import run_in_thread, sleep from electrumx.lib.hash import hash_to_hex_str, hex_str_to_hash @@ -30,21 +29,21 @@ from electrumx.lib.coins import Coin -@attr.s(slots=True) +@dataclass(slots=True) class MemPoolTx: - prevouts = attr.ib() # type: Sequence[Tuple[bytes, int]] # (txid_rev, txout_idx) + prevouts: Sequence[Tuple[bytes, int]] # (txid_rev, txout_idx) # A pair is a (hashX, value) tuple - in_pairs = attr.ib() # type: Optional[Sequence[Tuple[bytes, int]]] # (hashX, value_in_sats) - out_pairs = attr.ib() # type: Sequence[Tuple[bytes, int]] # (hashX, value_in_sats) - fee = attr.ib() # type: int # in sats - size = attr.ib() # type: int # in vbytes + in_pairs: Optional[Sequence[Tuple[bytes, int]]] # (hashX, value_in_sats) + out_pairs: Sequence[Tuple[bytes, int]] # (hashX, value_in_sats) + fee: int # in sats + size: int # in vbytes -@attr.s(slots=True) +@dataclass(slots=True) class MemPoolTxSummary: - txid_rev = attr.ib() # type: bytes - fee = attr.ib() # type: int # in sats - has_unconfirmed_inputs = attr.ib() # type: bool + txid_rev: bytes + fee: int # in sats + has_unconfirmed_inputs: bool @dataclass(slots=True, frozen=True, kw_only=True) diff --git a/src/electrumx/server/session.py b/src/electrumx/server/session.py index cced48032..9ee9f848f 100644 --- a/src/electrumx/server/session.py +++ b/src/electrumx/server/session.py @@ -9,6 +9,7 @@ import asyncio import codecs +from dataclasses import dataclass import datetime import itertools import math @@ -22,7 +23,6 @@ from typing import Iterable, Optional, TYPE_CHECKING, Sequence, Union, Any, Tuple, Set, Dict, Mapping from typing import Callable -import attr import aiorpcx from aiorpcx import (Event, JSONRPCAutoDetect, JSONRPCConnection, ReplyAndDisconnect, Request, RPCError, RPCSession, Service, @@ -118,12 +118,12 @@ def assert_list_or_tuple(value: Any) -> None: raise RPCError(BAD_REQUEST, f'{value} should be a list') -@attr.s(slots=True) +@dataclass(slots=True) class SessionGroup: - name = attr.ib() # type: str - weight = attr.ib() # type: float - sessions = attr.ib() # type: Set[SessionBase] - retained_cost = attr.ib() # type: float + name: str + weight: float + sessions: set['SessionBase'] + retained_cost: float def session_cost(self) -> float: return sum(session.cost for session in self.sessions) @@ -132,13 +132,13 @@ def cost(self) -> float: return self.retained_cost + self.session_cost() -@attr.s(slots=True) +@dataclass(slots=True) class SessionReferences: # All attributes are sets but groups is a list - sessions = attr.ib() # type: set[ElectrumX] - groups = attr.ib() # type: Sequence[SessionGroup] - specials = attr.ib() # type: set[str] # Lower-case strings - unknown = attr.ib() # type: set[str] + sessions: set['SessionBase'] + groups: Sequence['SessionGroup'] + specials: set[str] # Lower-case strings + unknown: set[str] class SessionManager: From f1be95a68cabbe5ed74da4a5e161718f3eea0371 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Fri, 22 May 2026 08:07:55 +0000 Subject: [PATCH 20/27] db: clean-break lack-of-db-upgrade - also, in a recent commit I added prefixes to the db "state" keys: now add missing handling of old unprefixed keys to detect old DBs --- src/electrumx/server/db.py | 18 ++++++++++-------- src/electrumx/server/history.py | 28 +++++++++++++++++++++------- 2 files changed, 31 insertions(+), 15 deletions(-) diff --git a/src/electrumx/server/db.py b/src/electrumx/server/db.py index 229f0a378..b3ec20691 100644 --- a/src/electrumx/server/db.py +++ b/src/electrumx/server/db.py @@ -31,6 +31,7 @@ from electrumx.server.storage import db_class, Storage from electrumx.server.history import ( History, TXNUM_LEN, TXNUM_PADDING, TXOUTIDX_LEN, TXOUTIDX_PADDING, pack_txnum, unpack_txnum, + DBTooOldForMigrations, ) if TYPE_CHECKING: @@ -69,7 +70,7 @@ class DB: it was shutdown uncleanly. ''' - DB_VERSIONS = (6, 7, 8) + DB_VERSIONS = (9, ) utxo_db: Optional['Storage'] @@ -615,6 +616,11 @@ def clear_excess_undo_info(self) -> None: # -- UTXO database def read_utxo_state(self) -> None: + if (oldstate := self.utxo_db.get(b'state')) is not None: + oldstate = ast.literal_eval(oldstate.decode()) + db_version = oldstate['db_version'] + raise DBTooOldForMigrations( + db_name="UTXO", db_version=db_version, supported_versions=self.DB_VERSIONS) state = self.utxo_db.get(b'\0state') if not state: self.db_height = -1 @@ -629,13 +635,9 @@ def read_utxo_state(self) -> None: raise self.DBError('failed reading state from DB') self.db_version = state['db_version'] if self.db_version not in self.DB_VERSIONS: - raise self.DBError(f'your UTXO DB version is {self.db_version} ' - f'but this software only handles versions ' - f'{self.DB_VERSIONS}') - # backwards compat + raise DBTooOldForMigrations( + db_name="UTXO", db_version=self.db_version, supported_versions=self.DB_VERSIONS) genesis_hash = state['genesis'] - if isinstance(genesis_hash, bytes): - genesis_hash = genesis_hash.decode() if genesis_hash != self.coin.GENESIS_HASH: raise self.DBError(f'DB genesis hash {genesis_hash} does not ' f'match coin {self.coin.GENESIS_HASH}') @@ -652,7 +654,7 @@ def read_utxo_state(self) -> None: # Upgrade DB if self.db_version != max(self.DB_VERSIONS): - pass # call future upgrade logic here + raise Exception("missing db upgrade") # call future upgrade logic here # Log some stats self.logger.info(f'UTXO DB version: {self.db_version:d}') diff --git a/src/electrumx/server/history.py b/src/electrumx/server/history.py index db43f48d0..584a33a19 100644 --- a/src/electrumx/server/history.py +++ b/src/electrumx/server/history.py @@ -13,7 +13,7 @@ import time from array import array from collections import defaultdict -from typing import TYPE_CHECKING, Type, Optional +from typing import TYPE_CHECKING, Type, Optional, Sequence import electrumx.lib.util as util from electrumx.lib.hash import HASHX_LEN, hash_to_hex_str @@ -40,9 +40,20 @@ def pack_txnum(tx_num: int) -> bytes: return pack_be_uint64(tx_num)[-TXNUM_LEN:] +class DBTooOldForMigrations(RuntimeError): + def __init__(self, *, db_name: str, db_version: int, supported_versions: Sequence[int]): + cmd = 'rm -rf DB_DIRECTORY/{hist,meta,utxo}' + super().__init__( + f'Your {db_name} DB version is {db_version} but this software only handles versions {supported_versions}. ' + f'Manually delete your database (e.g. `{cmd}`, and start again. ' + f'Then, your DB will be rebuilt from genesis, likely taking several hours. ' + f"If you don't have time for this now, you can temporarily downgrade the software." + ) + + class History: - DB_VERSIONS = (3, ) + DB_VERSIONS = (4, ) db: Optional['Storage'] @@ -76,6 +87,11 @@ def close_db(self): self.db = None def read_state(self): + if (oldstate := self.db.get(b'state\0\0')) is not None: + oldstate = ast.literal_eval(oldstate.decode()) + db_version = oldstate['db_version'] + raise DBTooOldForMigrations( + db_name="history", db_version=db_version, supported_versions=self.DB_VERSIONS) state = self.db.get(b'\0state') if state: state = ast.literal_eval(state.decode()) @@ -87,12 +103,10 @@ def read_state(self): self.hist_db_tx_count_next = self.hist_db_tx_count if self.db_version not in self.DB_VERSIONS: - msg = (f'your history DB version is {self.db_version} but ' - f'this software only handles DB versions {self.DB_VERSIONS}') - self.logger.error(msg) - raise RuntimeError(msg) + raise DBTooOldForMigrations( + db_name="history", db_version=self.db_version, supported_versions=self.DB_VERSIONS) if self.db_version != max(self.DB_VERSIONS): - pass # call future upgrade logic here + raise Exception("missing db upgrade") # call future upgrade logic here self.logger.info(f'history DB version: {self.db_version}') def clear_excess(self, utxo_db_tx_count: int) -> None: From d117bf0f9eba0e3e5942f8e810e359a58bbc9ae6 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Fri, 22 May 2026 14:15:05 +0000 Subject: [PATCH 21/27] session: fixes for "server.ping", and use it to send some naive noise --- src/electrumx/lib/util.py | 4 +++- src/electrumx/server/session.py | 38 +++++++++++++++++++++++++++------ 2 files changed, 35 insertions(+), 7 deletions(-) diff --git a/src/electrumx/lib/util.py b/src/electrumx/lib/util.py index 4038cb640..74ff1b6eb 100644 --- a/src/electrumx/lib/util.py +++ b/src/electrumx/lib/util.py @@ -313,9 +313,11 @@ def protocol_version(client_req, min_tuple, max_tuple): return result, client_min -def is_hex_str(text: Any) -> bool: +def is_hex_str(text: Any, *, allow_odd_len: bool = False) -> bool: if not isinstance(text, str): return False + if allow_odd_len and len(text) % 2 == 1: + return is_hex_str("0" + text, allow_odd_len=False) try: b = bytes.fromhex(text) except Exception: diff --git a/src/electrumx/server/session.py b/src/electrumx/server/session.py index 9ee9f848f..17e1425be 100644 --- a/src/electrumx/server/session.py +++ b/src/electrumx/server/session.py @@ -22,6 +22,7 @@ from ipaddress import IPv4Address, IPv6Address, IPv4Network, IPv6Network from typing import Iterable, Optional, TYPE_CHECKING, Sequence, Union, Any, Tuple, Set, Dict, Mapping from typing import Callable +import random import aiorpcx from aiorpcx import (Event, JSONRPCAutoDetect, JSONRPCConnection, @@ -108,8 +109,8 @@ def assert_txid_hum(value: Any | str) -> bytes: raise RPCError(BAD_REQUEST, f'{value} should be a transaction hash') -def assert_hex_str(value: Any | str) -> None: - if not is_hex_str(value): +def assert_hex_str(value: Any | str, *, allow_odd_len: bool = False) -> None: + if not is_hex_str(value, allow_odd_len=allow_odd_len): raise RPCError(BAD_REQUEST, f'{value} should be a hex string') @@ -1253,10 +1254,12 @@ async def _notify_inner( '''Notify the client about changes to touched addresses (from mempool updates or new blocks) and height. ''' + cnt_sent = 0 # block headers if height_changed and self.subscribe_headers: args = (await self.subscribe_headers_result(), ) await self.send_notification('blockchain.headers.subscribe', args) + cnt_sent += 1 # hashXs num_hashx_notifs_sent = 0 @@ -1286,6 +1289,7 @@ async def _notify_inner( method = 'blockchain.scripthash.subscribe' for alias, status in changed.items(): await self.send_notification(method, (alias, status)) + cnt_sent += 1 num_hashx_notifs_sent = len(changed) # tx outpoints @@ -1309,6 +1313,7 @@ async def _notify_inner( spend_status = txo_to_status[(tx_hash, txout_idx)] tx_hash_hex = hash_to_hex_str(tx_hash) await self.send_notification(method, (tx_hash_hex, txout_idx, spend_status)) + cnt_sent += 1 num_txo_notifs_sent = len(touched_outpoints) if num_hashx_notifs_sent + num_txo_notifs_sent > 0: @@ -1317,6 +1322,18 @@ async def _notify_inner( self.logger.info(f'notified of {num_hashx_notifs_sent:,d} address{es1} and ' f'{num_txo_notifs_sent:,d} outpoint{s2}') + # maybe send some noise + if self.protocol_tuple >= (1, 7): + if height_changed: # on block + if cnt_sent < 2: + await self.send_ping_notification_to_client(data_len=128) # similar len to bc.spk.sub + while random.random() < 0.1: + await self.send_ping_notification_to_client(data_len=128) + else: # on mempool + once_per_10_minutes = self.mempool.refresh_secs / 600 + if random.random() < once_per_10_minutes: + await self.send_ping_notification_to_client(data_len=128) + async def subscribe_headers_result(self): '''The result of a header subscription or notification.''' return self.session_mgr.hsub_results @@ -1840,17 +1857,26 @@ async def phandle_ping(self, pong_len=0, data=""): self.bump_cost(0.1) if self.protocol_tuple < (1, 7): return None - assert_hex_str(data) + assert_hex_str(data, allow_odd_len=True) pong_len = non_negative_integer(pong_len) if pong_len > self.env.max_send: raise RPCError(BAD_REQUEST, f'pong_len value too high') pong_data = pong_len * "0" - return {"data": pong_data} + ret = {"data": pong_data} + return ret async def phandle_on_ping_notification(self, data=""): self.bump_cost(0.1) # note: the bw cost for receiving 'data' has already been incurred - assert_hex_str(data) - # nothing to do + assert_hex_str(data, allow_odd_len=True) + # nothing to do. + # note: we could probabilistically send back a ping notif to the client, as noise, + # but we don't. Leave such logic to the client: if they wanted a response, + # they would have sent "server.ping" as a request instead of a notification. + + async def send_ping_notification_to_client(self, data_len: int) -> None: + assert isinstance(data_len, int) and data_len >= 0, repr(data_len) + data = "0" * data_len + await self.send_notification("server.ping", (data,)) async def phandle_server_version( self, From f01f678b68edae0b37964fd5735b30145388cbe6 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Fri, 22 May 2026 14:10:09 +0000 Subject: [PATCH 22/27] session: use spec-1.7-defined RPC error code for "history too large" --- src/electrumx/server/session.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/electrumx/server/session.py b/src/electrumx/server/session.py index 17e1425be..26d2052c8 100644 --- a/src/electrumx/server/session.py +++ b/src/electrumx/server/session.py @@ -56,6 +56,7 @@ BAD_REQUEST = 1 DAEMON_ERROR = 2 +RPC_ERROR_HISTORY_TOO_LONG = 10_001 def scripthash_to_hashX(scripthash: str) -> bytes: @@ -872,7 +873,7 @@ async def limited_history(self, hashX: bytes) -> tuple[Sequence[tuple[bytes, int result = await self.db.limited_history(hashX, limit=limit) cost += 0.1 + len(result) * 0.001 if len(result) >= limit: - result = RPCError(BAD_REQUEST, f'history too large', cost=cost) + result = RPCError(RPC_ERROR_HISTORY_TOO_LONG, f'history too large', cost=cost) self._history_cache[hashX] = result assert result is not None From a67afdd49bc43aa7746b0bdda1b98b7ef0f959d1 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Fri, 22 May 2026 14:23:48 +0000 Subject: [PATCH 23/27] bump version to 2.0b1 --- docs/changelog.rst | 26 ++++++++++++++++++++++++++ docs/conf.py | 2 +- src/electrumx/__init__.py | 2 +- 3 files changed, 28 insertions(+), 2 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index b09f4e5fd..c6d251987 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,30 @@ ChangeLog =========== +Version 2.0 (not yet released) +============================= + +TODO expand + +* Significant changes to on-disk database. Server operators need to manually delete old DB, + no migration path. New DB will be rebuilt while rescanning again from genesis. + - in return, no more manual "history compaction" and unexpected downtimes every year: + no more "DB::flush_count overflow" (`spesmilo/electrumx#88`_) + - the size of the new DB is comparable to the old one, however: +* We now require Bitcoin Core to have `txospenderindex=1` (added in Bitcoin Core 31) + in addition to `txindex=1`. This is needed to serve the `blockchain.outpoint.subscribe` RPC + added in Electrum Protocol 1.7. + For reference, on Bitcoin mainnet around height=950k, + - the ElectrumX db uses around 122 GiB (roughly same for both of e-x 1.x and 2.0), + - `.bitcoin/blocks/` uses 788 GiB, + - `.bitcoin/chainstate/` uses 12 GiB, + - `.bitcoin/indexes/txindex/` uses 66 GiB, + - `.bitcoin/indexes/txospenderindex/` uses 88 GiB (new!) +* This also means that we now require Bitcoin Core 31.0 or newer (for `COIN=Bitcoin`). +* protocol: + - new: implement electrum protocol version 1.7 (`spesmilo/electrum-protocol#2`_). + The min supported protocol version remains 1.4, the max is now 1.7. + Version 1.19.0 (11 Nov 2025) ============================= @@ -276,6 +300,7 @@ This fork maintained by: .. _#67: https://github.com/spesmilo/electrumx/pull/67 .. _#70: https://github.com/spesmilo/electrumx/pull/70 .. _spesmilo/electrumx#75: https://github.com/spesmilo/electrumx/pull/75 +.. _spesmilo/electrumx#88: https://github.com/spesmilo/electrumx/issues/88 .. _spesmilo/electrumx#122: https://github.com/spesmilo/electrumx/pull/122 .. _spesmilo/electrumx#248: https://github.com/spesmilo/electrumx/pull/248 .. _spesmilo/electrumx#273: https://github.com/spesmilo/electrumx/pull/273 @@ -303,5 +328,6 @@ This fork maintained by: .. _0ba87447: https://github.com/spesmilo/electrumx/commit/0ba87447cb293cfc4a8a26c1c27842b95666875a +.. _spesmilo/electrum-protocol#2: https://github.com/spesmilo/electrum-protocol/pull/2 .. _spesmilo/electrum-protocol#6: https://github.com/spesmilo/electrum-protocol/pull/6 diff --git a/docs/conf.py b/docs/conf.py index 47049627b..c6a8df4bf 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -15,7 +15,7 @@ import os import sys sys.path.insert(0, os.path.abspath('..')) -VERSION="ElectrumX 1.19.0" +VERSION="ElectrumX 2.0" # -- Project information ----------------------------------------------------- diff --git a/src/electrumx/__init__.py b/src/electrumx/__init__.py index 26622e5bf..57b299448 100644 --- a/src/electrumx/__init__.py +++ b/src/electrumx/__init__.py @@ -3,6 +3,6 @@ BRANDING = "spesmilo" -__version__ = "1.19.0" +__version__ = "2.0b1" version = f'ElectrumX {__version__}' version_short = __version__ From 75a5f6e72541a5bc9369fabb87cfeb04ab78b83b Mon Sep 17 00:00:00 2001 From: SomberNight Date: Fri, 22 May 2026 15:57:11 +0000 Subject: [PATCH 24/27] session: bc.tx.get_merkle: don't include "block_hash" in resp postponing this from protocol 1.7 -- the motivation for adding this field was reliant on the "height" arg being optional, which we are also postponing. --- src/electrumx/server/session.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/electrumx/server/session.py b/src/electrumx/server/session.py index 26d2052c8..df7d5f70c 100644 --- a/src/electrumx/server/session.py +++ b/src/electrumx/server/session.py @@ -2056,7 +2056,7 @@ async def phandle_transaction_merkle(self, tx_hash: str | Any, height: int | Any return { "block_height": height, - "block_hash": blockhash_hum, + #"block_hash": blockhash_hum, "merkle": branch, "pos": tx_pos, } From f6100ce17fa4c169c166bc6f8219d08a8824f69f Mon Sep 17 00:00:00 2001 From: SomberNight Date: Fri, 22 May 2026 16:07:06 +0000 Subject: [PATCH 25/27] session: "server.features": omit "hash_function" for proto 1.7 --- src/electrumx/server/session.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/electrumx/server/session.py b/src/electrumx/server/session.py index df7d5f70c..545d82da7 100644 --- a/src/electrumx/server/session.py +++ b/src/electrumx/server/session.py @@ -1188,13 +1188,16 @@ def server_features(cls, env: 'Env') -> dict[str, Any]: 'protocol_min': min_str, 'protocol_max': max_str, 'genesis_hash': env.coin.GENESIS_HASH, - 'hash_function': 'sha256', + 'hash_function': 'sha256', # FIXME should only be present for proto < 1.7 'services': [str(service) for service in env.report_services], } async def phandle_server_features_async(self) -> dict[str, Any]: self.bump_cost(0.2) - return self.server_features(self.env) + features = self.server_features(self.env) + if self.protocol_tuple >= (1, 7): + features.pop('hash_function', None) + return features @classmethod def server_version_args(cls): From 21465a768c926409bdb8f3c339e71aaa118cc893 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Fri, 22 May 2026 16:55:31 +0000 Subject: [PATCH 26/27] calm pycodestyle and raise the max line limit as it is more annoying than useful --- .github/workflows/ci.yml | 2 +- src/electrumx/lib/lrucache.py | 2 ++ src/electrumx/server/block_processor.py | 5 +++-- src/electrumx/server/session.py | 4 ++-- 4 files changed, 8 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 457cc9e39..db3aff31b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -79,7 +79,7 @@ jobs: run: pip install pycodestyle - name: Run pycodestyle - run: pycodestyle --max-line-length=100 src + run: pycodestyle --max-line-length=140 src build_docs: # note: this build task is not related to read-the-docs; it is only here to sanity-check the docs can be built. diff --git a/src/electrumx/lib/lrucache.py b/src/electrumx/lib/lrucache.py index c3eea8029..a70011dc0 100644 --- a/src/electrumx/lib/lrucache.py +++ b/src/electrumx/lib/lrucache.py @@ -45,6 +45,8 @@ def pop(self, _): _KT = TypeVar("_KT") _VT = TypeVar("_VT") + + class Cache(collections.abc.MutableMapping[_KT, _VT]): """Mutable mapping to serve as a simple cache or cache base class.""" diff --git a/src/electrumx/server/block_processor.py b/src/electrumx/server/block_processor.py index 9ef72eee8..15556ba36 100644 --- a/src/electrumx/server/block_processor.py +++ b/src/electrumx/server/block_processor.py @@ -401,8 +401,9 @@ def check_cache_size(self) -> Optional[bool]: db_deletes_size = len(self.db_deletes) * 57 hist_cache_size = self.db.history.unflushed_memsize() # Roughly ntxs * 32 + nblocks * 42 - txids_size = ((self.tx_count - self.db.fs_tx_count) * 32 - + (self.height - self.db.fs_height) * 42) + txids_size = ( + (self.tx_count - self.db.fs_tx_count) * 32 + + (self.height - self.db.fs_height) * 42) utxo_MB = (db_deletes_size + utxo_cache_size) // one_MB hist_MB = (hist_cache_size + txids_size) // one_MB diff --git a/src/electrumx/server/session.py b/src/electrumx/server/session.py index 545d82da7..677cce2be 100644 --- a/src/electrumx/server/session.py +++ b/src/electrumx/server/session.py @@ -178,7 +178,7 @@ def __init__( self._txids_cache = LRUCache(maxsize=1000) # type: LRUCache[int, Sequence[bytes]] # Really a MerkleCache cache self._merkle_txid_cache = LRUCache(maxsize=1000) # type: LRUCache[int, MerkleCache] - self.estimatefee_cache : LRUCache[ + self.estimatefee_cache: LRUCache[ tuple[int, str | None], tuple[bytes | None, float | None, asyncio.Lock] ] = LRUCache(maxsize=1000) @@ -2059,7 +2059,7 @@ async def phandle_transaction_merkle(self, tx_hash: str | Any, height: int | Any return { "block_height": height, - #"block_hash": blockhash_hum, + # "block_hash": blockhash_hum, "merkle": branch, "pos": tx_pos, } From 9c2cb07f533cc3ac9ac64cd02818310421efbadb Mon Sep 17 00:00:00 2001 From: SomberNight Date: Sat, 23 May 2026 18:05:38 +0000 Subject: [PATCH 27/27] session: "bc.outpoint.subscribe" RPC: rename height to funder_height --- src/electrumx/server/session.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/electrumx/server/session.py b/src/electrumx/server/session.py index 677cce2be..7a0d554e1 100644 --- a/src/electrumx/server/session.py +++ b/src/electrumx/server/session.py @@ -1478,7 +1478,7 @@ async def _calc_txoutpoint_status(self, prev_txid_rev: bytes, txout_idx: int) -> # convert to json dict the client expects status = {} if spend_status.prev_height is not None: - status['height'] = spend_status.prev_height + status['funder_height'] = spend_status.prev_height if spend_status.spender_txid_rev is not None: assert spend_status.spender_height is not None status['spender_txhash'] = hash_to_hex_str(spend_status.spender_txid_rev) @@ -1490,7 +1490,7 @@ async def txoutpoint_status_for_notif(self, prev_txid_rev: bytes, txout_idx: int status = await self._calc_txoutpoint_status(prev_txid_rev=prev_txid_rev, txout_idx=txout_idx) # update status last sent to client prevout = (prev_txid_rev, txout_idx) - prev_height = status.get('height') # type: Optional[int] + prev_height = status.get('funder_height') # type: Optional[int] spender_height = status.get('spender_height') # type: Optional[int] if ((prev_height is not None and prev_height <= 0) or (spender_height is not None and spender_height <= 0)):