diff --git a/assets/config/dev.json b/assets/config/dev.json index 43eda84f9a..8fe8d0a221 100644 --- a/assets/config/dev.json +++ b/assets/config/dev.json @@ -1,5 +1,5 @@ { - "configVersion": 2, + "configVersion": 3, "defaultArweaveGatewayUrl": "https://ar-io.dev", "defaultArweaveGatewayForDataRequest": { "label": "AR.IO Testnet", @@ -12,5 +12,6 @@ "allowedDataItemSizeForTurbo": 100000, "stripePublishableKey": "pk_test_51JUAtwC8apPOWkDLh2FPZkQkiKZEkTo6wqgLCtQoClL6S4l2jlbbc5MgOdwOUdU9Tn93NNvqAGbu115lkJChMikG00XUfTmo2z", "uploadThumbnails": true, - "autoSync": false + "autoSync": false, + "maxConcurrentDriveSyncs": 50 } diff --git a/assets/config/prod.json b/assets/config/prod.json index 6c091ca320..c3bf2c4240 100644 --- a/assets/config/prod.json +++ b/assets/config/prod.json @@ -1,6 +1,6 @@ { - "configVersion": 2, - "defaultArweaveGatewayUrl": "https://ardrive.net", + "configVersion": 3, + "defaultArweaveGatewayUrl": "https://turbo-gateway.com", "defaultArweaveGatewayForDataRequest": { "label": "Turbo Gateway", "url": "https://turbo-gateway.com" @@ -12,5 +12,6 @@ "allowedDataItemSizeForTurbo": 100000, "stripePublishableKey": "pk_live_51JUAtwC8apPOWkDLMQqNF9sPpfneNSPnwX8YZ8y1FNDl6v94hZIwzgFSYl27bWE4Oos8CLquunUswKrKcaDhDO6m002Yj9AeKj", "uploadThumbnails": true, - "autoSync": false + "autoSync": false, + "maxConcurrentDriveSyncs": 50 } diff --git a/assets/config/staging.json b/assets/config/staging.json index 6c091ca320..c3bf2c4240 100644 --- a/assets/config/staging.json +++ b/assets/config/staging.json @@ -1,6 +1,6 @@ { - "configVersion": 2, - "defaultArweaveGatewayUrl": "https://ardrive.net", + "configVersion": 3, + "defaultArweaveGatewayUrl": "https://turbo-gateway.com", "defaultArweaveGatewayForDataRequest": { "label": "Turbo Gateway", "url": "https://turbo-gateway.com" @@ -12,5 +12,6 @@ "allowedDataItemSizeForTurbo": 100000, "stripePublishableKey": "pk_live_51JUAtwC8apPOWkDLMQqNF9sPpfneNSPnwX8YZ8y1FNDl6v94hZIwzgFSYl27bWE4Oos8CLquunUswKrKcaDhDO6m002Yj9AeKj", "uploadThumbnails": true, - "autoSync": false + "autoSync": false, + "maxConcurrentDriveSyncs": 50 } diff --git a/lib/authentication/ardrive_auth.dart b/lib/authentication/ardrive_auth.dart index a40eda6b00..c2d9397c3c 100644 --- a/lib/authentication/ardrive_auth.dart +++ b/lib/authentication/ardrive_auth.dart @@ -435,7 +435,17 @@ class ArDriveAuthImpl implements ArDriveAuth { @override Future refreshBalance() async { - _updateBalance(); + // Await the fetch so callers (e.g. ProfileCubit.refreshBalance, which + // re-emits currentUser right after) observe the refreshed value instead + // of the stale one. + try { + final balance = await _userRepository.getBalance(currentUser.wallet); + _currentUser = _currentUser!.copyWith(walletBalance: balance); + _userStreamController.add(_currentUser); + } catch (e) { + logger.e('Error refreshing wallet balance', e); + // Keep the previous value on error. + } } } diff --git a/lib/core/arfs/use_cases/bulk_import_files.dart b/lib/core/arfs/use_cases/bulk_import_files.dart index caafa13848..376146ea31 100644 --- a/lib/core/arfs/use_cases/bulk_import_files.dart +++ b/lib/core/arfs/use_cases/bulk_import_files.dart @@ -419,17 +419,23 @@ class BulkImportFiles { : 5, taskQueue: fileEntries, onWorkerError: (file, error) { + // The WorkerPool passes both the failed TASK and its exception. + // Preserve the exception (and its originalError) so payment + // rejections stay detectable, and record every failed file so + // BulkImportResult stays truthful (imported + failures == total). logger.e('Bulk import worker error', error, StackTrace.current); - // Record the failure so BulkImportResult carries it (with its - // originalError) — the pool otherwise swallows task exceptions. + final manifestFile = + file.dataTxId != null ? fileDataTxIdToFile[file.dataTxId] : null; + final path = manifestFile?.path ?? file.name ?? 'unknown'; failures.add(error is FileImportFailure ? error : FileImportFailure( - path: file.name ?? '', + path: path, dataTxId: file.dataTxId ?? '', error: error.toString(), originalError: error, )); + onFileFailure?.call(path); }, execute: (file) async { if (_isCancelled) { @@ -452,6 +458,10 @@ class BulkImportFiles { originalOwnerAddress: originalOwnerAddress, ); + // WorkerPool discards execute's return value, so successful + // imports must be collected here for BulkImportResult. + importedFiles.add(fileEntry); + onFileUploadSuccess?.call(file.name!); return fileEntry; diff --git a/lib/gar/domain/repositories/gar_repository.dart b/lib/gar/domain/repositories/gar_repository.dart index 021cc1b382..3abf3960b2 100644 --- a/lib/gar/domain/repositories/gar_repository.dart +++ b/lib/gar/domain/repositories/gar_repository.dart @@ -8,6 +8,7 @@ import 'package:collection/collection.dart'; abstract class GarRepository { Future> getGateways(); + Future> refreshGateways(); List searchGateways(String query); Future getSelectedGateway(); Future updateGateway(Gateway gateway); @@ -35,10 +36,21 @@ class GarRepositoryImpl implements GarRepository { final List _gateways = []; + /// Serves the persisted/session gateway list; the network is hit at most + /// once ever (on the very first use). Use [refreshGateways] for an + /// explicit, user-initiated re-fetch. @override Future> getGateways() async { _gateways.clear(); - _gateways.addAll(await arioSDK.getGateways()); + _gateways.addAll(await arweave.gatewayFallback.getGatewaysCached()); + + return _gateways; + } + + @override + Future> refreshGateways() async { + _gateways.clear(); + _gateways.addAll(await arweave.gatewayFallback.refreshGateways()); return _gateways; } diff --git a/lib/gar/presentation/bloc/gar_bloc.dart b/lib/gar/presentation/bloc/gar_bloc.dart index 554a233d85..8a6f5ed325 100644 --- a/lib/gar/presentation/bloc/gar_bloc.dart +++ b/lib/gar/presentation/bloc/gar_bloc.dart @@ -33,6 +33,25 @@ class GarBloc extends Bloc { } }); + on((event, emit) async { + try { + emit(LoadingGateways()); + + final gateways = await garRepository.refreshGateways(); + final currentGateway = await garRepository.getSelectedGateway(); + + emit( + GatewaysLoaded( + gateways: gateways, + currentGateway: currentGateway, + ), + ); + } catch (e) { + logger.e('Failed to refresh gateways from AR.IO', e); + emit(const GatewaysError()); + } + }); + on((event, emit) async { emit(VerifyingGateway()); diff --git a/lib/gar/presentation/bloc/gar_event.dart b/lib/gar/presentation/bloc/gar_event.dart index 3a276dcfdd..37efb27cb4 100644 --- a/lib/gar/presentation/bloc/gar_event.dart +++ b/lib/gar/presentation/bloc/gar_event.dart @@ -9,6 +9,9 @@ abstract class GarEvent extends Equatable { final class GetGateways extends GarEvent {} +/// User-initiated refresh of the gateway list from the network. +final class RefreshGateways extends GarEvent {} + final class SelectGateway extends GarEvent { final Gateway gateway; diff --git a/lib/gar/presentation/widgets/gar_modal.dart b/lib/gar/presentation/widgets/gar_modal.dart index a68153ae9b..5677e60dd7 100644 --- a/lib/gar/presentation/widgets/gar_modal.dart +++ b/lib/gar/presentation/widgets/gar_modal.dart @@ -139,6 +139,12 @@ class _ArIOGatewaySelectorModalContentState extends State<_ArIOGatewaySelectorMo ), ), actions: [ + ModalAction( + action: () { + garBloc.add(RefreshGateways()); + }, + title: 'Refresh list', + ), ModalAction( action: () { Navigator.of(context).pop(); diff --git a/lib/main.dart b/lib/main.dart index 2e2f4f2a32..d37bcd6e43 100644 --- a/lib/main.dart +++ b/lib/main.dart @@ -489,7 +489,6 @@ class AppState extends State { configService: configService, arioSDK: ArioSDKFactory().create(), ), - arnsRepository: _.read(), userPreferencesRepository: _.read(), ), ), diff --git a/lib/services/arweave/arweave_service.dart b/lib/services/arweave/arweave_service.dart index 95ffc1ea1e..9e770321b1 100644 --- a/lib/services/arweave/arweave_service.dart +++ b/lib/services/arweave/arweave_service.dart @@ -152,6 +152,18 @@ class ArweaveService { /// Cache for drive signatures (immutable on-chain, never change). final Map _cachedDriveSignatures = {}; + /// Single [MetadataCache] instance reused across all sync batches. + /// Previously a new cache was rebuilt from SharedPreferences for every + /// parsed batch of every drive during sync. + MetadataCache? _metadataCache; + + Future _getMetadataCache() async { + _metadataCache ??= await MetadataCache.fromCacheStore( + await newSharedPreferencesCacheStore(), + ); + return _metadataCache!; + } + /// Clears the cached result of [getUniqueUserDriveEntityTxs] and entity data. /// Call after creating/updating a drive or after a full sync completes. void clearUserDriveTxsCache() { @@ -545,9 +557,7 @@ class ArweaveService { ); } - final metadataCache = await MetadataCache.fromCacheStore( - await newSharedPreferencesCacheStore(), - ); + final metadataCache = await _getMetadataCache(); final blockHistory = []; @@ -1735,9 +1745,13 @@ class ArweaveService { /// Fetches transaction info for multiple transactions in batches. /// Returns a stream of transaction info batches. + /// + /// The batch size matches the query's `first: 100` page size (the same + /// pattern as [getTransactionConfirmations] and [getLicenseAssertions]), + /// so each batch resolves in a single request. Stream> getInfoOfTxsToBePinned( List transactionIds, { - int batchSize = 5, + int batchSize = 100, }) async* { for (var i = 0; i < transactionIds.length; i += batchSize) { final end = (i + batchSize < transactionIds.length) diff --git a/lib/services/arweave/data_gateway_fallback.dart b/lib/services/arweave/data_gateway_fallback.dart index a51177598b..3bf1122897 100644 --- a/lib/services/arweave/data_gateway_fallback.dart +++ b/lib/services/arweave/data_gateway_fallback.dart @@ -1,7 +1,10 @@ import 'dart:async'; +import 'dart:convert'; import 'package:ardrive/download/download_exceptions.dart'; import 'package:ardrive/services/arweave/arweave_service.dart'; +import 'package:ardrive/utils/key_value_store.dart'; +import 'package:ardrive/utils/local_key_value_store.dart'; import 'package:ardrive/utils/logger.dart'; import 'package:ardrive_http/ardrive_http.dart'; import 'package:ario_sdk/ario_sdk.dart'; @@ -33,9 +36,109 @@ class DataGatewayFallback { /// SnapshotValidationService) to avoid duplicate Solana RPC calls. List? cachedGateways; + static const _garCacheKey = 'gar_gateways_cache_v1'; + + KeyValueStore? _store; + DataGatewayFallback({ required ArioSDK arioSDK, - }) : _arioSDK = arioSDK; + KeyValueStore? store, + }) : _arioSDK = arioSDK, + _store = store; + + Future _getStore() async { + if (_store != null) return _store; + try { + _store = await LocalKeyValueStore.getInstance(); + } catch (e) { + logger.w('GAR cache store unavailable: $e'); + } + return _store; + } + + /// Returns the gateway list while fetching from the network at most once + /// ever: memory cache → persisted cache → single SDK fetch (persisted on + /// success). The gateway registry rarely changes, so we avoid hitting the + /// Solana RPC on every session; explicit refreshes go through + /// [refreshGateways] (e.g. from the gateway settings screen). + Future>? _getGatewaysFuture; + + Future> getGatewaysCached() { + // Memoize the in-flight future: concurrent first callers (e.g. several + // drive syncs validating snapshots at once) must share one fetch. + return _getGatewaysFuture ??= _getGatewaysCachedImpl(); + } + + Future> _getGatewaysCachedImpl() async { + if (cachedGateways != null) return cachedGateways!; + + final persisted = await _loadPersistedGateways(); + if (persisted != null) { + cachedGateways = persisted; + return persisted; + } + + try { + final fetched = await _arioSDK + .getGateways() + .timeout(_garListTimeout, onTimeout: () => []); + cachedGateways = fetched; + if (fetched.isNotEmpty) { + await _persistGateways(fetched); + } + } catch (e) { + // RPC failed — cache empty list in memory so we don't retry every call + logger.w('GAR list unavailable, will not retry this session: $e'); + cachedGateways = []; + } + return cachedGateways!; + } + + /// Force-refreshes the gateway list from the network and persists the + /// result. User-initiated only (refresh action in gateway settings). + /// + /// Unlike [getGatewaysCached], a stalled RPC throws ([TimeoutException]) + /// instead of returning an empty list, so the caller can surface an error + /// state with a retry affordance rather than silently showing no gateways. + /// The existing cache and persisted list are left untouched on failure. + Future> refreshGateways() async { + final fetched = await _arioSDK.getGateways().timeout(_garListTimeout); + cachedGateways = fetched; + _getGatewaysFuture = null; // next cached read observes the refresh + if (fetched.isNotEmpty) { + await _persistGateways(fetched); + } + return fetched; + } + + Future?> _loadPersistedGateways() async { + try { + final store = await _getStore(); + final raw = await store?.getString(_garCacheKey); + if (raw == null) return null; + final decoded = (json.decode(raw) as List) + .map((e) => Gateway.fromJson(e as Map)) + .toList(); + // An empty persisted list carries no value; treat as not cached so the + // next session retries the fetch. + return decoded.isEmpty ? null : decoded; + } catch (e) { + logger.w('Failed to load persisted GAR list, refetching: $e'); + return null; + } + } + + Future _persistGateways(List gateways) async { + try { + final store = await _getStore(); + await store?.putString( + _garCacheKey, + json.encode(gateways.map((g) => g.toJson()).toList()), + ); + } catch (e) { + logger.w('Failed to persist GAR list: $e'); + } + } /// Fetch transaction data with serial gateway fallback. /// @@ -193,20 +296,10 @@ class DataGatewayFallback { final primaryHost = primaryClient.api.gatewayUrl.host; try { - if (cachedGateways == null) { - try { - cachedGateways = await _arioSDK - .getGateways() - .timeout(_garListTimeout, onTimeout: () => []); - } catch (e) { - // Solana RPC failed — cache empty list so we don't retry every call - logger.w('GAR list unavailable for fallback, will not retry: $e'); - cachedGateways = []; - } - } + final gateways = await getGatewaysCached(); var added = 0; - for (final gw in cachedGateways!) { + for (final gw in gateways) { if (added >= _maxGarFallbacks) break; if (gw.settings.fqdn == primaryHost) continue; clients.add(_getOrCreateClient(gw.settings.fqdn)); diff --git a/lib/services/arweave/graphql/queries/InfoOfTransactionsToBePinned.graphql b/lib/services/arweave/graphql/queries/InfoOfTransactionsToBePinned.graphql index 7fe672ddaf..c6400df867 100644 --- a/lib/services/arweave/graphql/queries/InfoOfTransactionsToBePinned.graphql +++ b/lib/services/arweave/graphql/queries/InfoOfTransactionsToBePinned.graphql @@ -1,5 +1,5 @@ query InfoOfTransactionsToBePinned($transactionIds: [ID!]) { - transactions(ids: $transactionIds) { + transactions(first: 100, ids: $transactionIds) { edges { node { id diff --git a/lib/services/config/app_config.dart b/lib/services/config/app_config.dart index e3e95bb6a4..03cae582ce 100644 --- a/lib/services/config/app_config.dart +++ b/lib/services/config/app_config.dart @@ -28,6 +28,10 @@ class AppConfig { final String? solanaAntProgramId; final int maxConcurrentDataFetches; + /// Maximum number of drives synced concurrently during a full sync. + final int maxConcurrentDriveSyncs; + + AppConfig({ this.arweaveGatewayUrl, this.arweaveGatewayForDataRequest = const SelectedGateway( @@ -51,6 +55,7 @@ class AppConfig { this.solanaArnsProgramId, this.solanaAntProgramId, this.maxConcurrentDataFetches = 5, + this.maxConcurrentDriveSyncs = 50, }); AppConfig copyWith({ @@ -73,6 +78,7 @@ class AppConfig { String? solanaArnsProgramId, String? solanaAntProgramId, int? maxConcurrentDataFetches, + int? maxConcurrentDriveSyncs, }) { return AppConfig( arweaveGatewayUrl: @@ -102,6 +108,8 @@ class AppConfig { solanaAntProgramId: solanaAntProgramId ?? this.solanaAntProgramId, maxConcurrentDataFetches: maxConcurrentDataFetches ?? this.maxConcurrentDataFetches, + maxConcurrentDriveSyncs: + maxConcurrentDriveSyncs ?? this.maxConcurrentDriveSyncs, ); } diff --git a/lib/services/config/app_config.g.dart b/lib/services/config/app_config.g.dart index 28ad855f7b..25541bd414 100644 --- a/lib/services/config/app_config.g.dart +++ b/lib/services/config/app_config.g.dart @@ -33,6 +33,7 @@ AppConfig _$AppConfigFromJson(Map json) => AppConfig( solanaArnsProgramId: json['solanaArnsProgramId'] as String?, solanaAntProgramId: json['solanaAntProgramId'] as String?, maxConcurrentDataFetches: json['maxConcurrentDataFetches'] as int? ?? 5, + maxConcurrentDriveSyncs: json['maxConcurrentDriveSyncs'] as int? ?? 50, ); Map _$AppConfigToJson(AppConfig instance) => { @@ -56,4 +57,5 @@ Map _$AppConfigToJson(AppConfig instance) => { 'solanaArnsProgramId': instance.solanaArnsProgramId, 'solanaAntProgramId': instance.solanaAntProgramId, 'maxConcurrentDataFetches': instance.maxConcurrentDataFetches, + 'maxConcurrentDriveSyncs': instance.maxConcurrentDriveSyncs, }; diff --git a/lib/services/config/config_fetcher.dart b/lib/services/config/config_fetcher.dart index f72887a7fd..ea670c2654 100644 --- a/lib/services/config/config_fetcher.dart +++ b/lib/services/config/config_fetcher.dart @@ -62,10 +62,33 @@ class ConfigFetcher { final defaultVersion = defaultConfig.configVersion ?? 1; if (storedVersion < defaultVersion) { - // The stored config is from a previous release. Replace it. - // Any developer-specific customizations will be reset, which is acceptable. - await saveConfigOnDevToolsPrefs(defaultConfig); - return defaultConfig; + // The stored config is from a previous release. Replace it, but + // preserve gateway choices that differ from the PREVIOUS defaults: + // those were set deliberately (settings UI, AR.IO detection, or a + // custom endpoint) and silently resetting them would break users on + // networks where the new default is unreachable. + const previousDefaultGqlGateway = 'https://ardrive.net'; + const previousDefaultDataGatewayUrl = 'https://turbo-gateway.com'; + + var migrated = defaultConfig; + + final storedGqlGateway = storedConfig.arweaveGatewayUrl; + if (storedGqlGateway != null && + storedGqlGateway != previousDefaultGqlGateway && + storedGqlGateway != defaultConfig.arweaveGatewayUrl) { + migrated = migrated.copyWith(arweaveGatewayUrl: storedGqlGateway); + } + + final storedDataGateway = storedConfig.arweaveGatewayForDataRequest; + if (storedDataGateway.url != previousDefaultDataGatewayUrl && + storedDataGateway.url != + defaultConfig.arweaveGatewayForDataRequest.url) { + migrated = + migrated.copyWith(arweaveGatewayForDataRequest: storedDataGateway); + } + + await saveConfigOnDevToolsPrefs(migrated); + return migrated; } // The stored config is up-to-date. diff --git a/lib/sync/constants.dart b/lib/sync/constants.dart index 6b1e3101e9..d5699bfb87 100644 --- a/lib/sync/constants.dart +++ b/lib/sync/constants.dart @@ -1,4 +1,5 @@ const kBlockHeightLookBack = 240; + const kRequiredTxConfirmationPendingThreshold = 60 * 8; const kArConnectSyncTimerDuration = 2; diff --git a/lib/sync/data/snapshot_validation_service.dart b/lib/sync/data/snapshot_validation_service.dart index f404f0a837..00e975b003 100644 --- a/lib/sync/data/snapshot_validation_service.dart +++ b/lib/sync/data/snapshot_validation_service.dart @@ -20,6 +20,10 @@ class SnapshotValidationService { /// services share one cache and avoid duplicate Solana RPC calls. DataGatewayFallback? gatewayFallback; + /// Session-local gateway cache used only when no shared + /// [gatewayFallback] is wired up. + List? _localGateways; + SnapshotValidationService({ required ConfigService configService, required ArioSDK arioSDK, @@ -108,29 +112,22 @@ class SnapshotValidationService { } // 3. Primary had a transient error (timeout, 5xx) — try 1 fallback gateway. - // Read the shared gateway cache from DataGatewayFallback. If unavailable, - // fetch from Solana RPC once and cache the result (empty on failure). + // Read the shared gateway cache from DataGatewayFallback (which loads + // a persisted list and fetches from Solana RPC at most once ever). If + // no shared cache is wired up, fetch once and keep it in memory. try { List gateways; - if (gatewayFallback != null && - gatewayFallback!.cachedGateways != null) { - gateways = gatewayFallback!.cachedGateways!; + if (gatewayFallback != null) { + gateways = await gatewayFallback!.getGatewaysCached(); } else { try { - gateways = await _arioSDK + gateways = _localGateways ??= await _arioSDK .getGateways() .timeout(_garListTimeout, onTimeout: () => []); - // Store in shared cache if available - if (gatewayFallback != null) { - gatewayFallback!.cachedGateways = gateways; - } } catch (e) { // Solana RPC failed — cache empty list so we don't retry every call logger.w('GAR gateway list unavailable, will not retry: $e'); - if (gatewayFallback != null) { - gatewayFallback!.cachedGateways = []; - } - gateways = []; + gateways = _localGateways = []; } } diff --git a/lib/sync/domain/repositories/sync_repository.dart b/lib/sync/domain/repositories/sync_repository.dart index 92d980c094..1431b24a22 100644 --- a/lib/sync/domain/repositories/sync_repository.dart +++ b/lib/sync/domain/repositories/sync_repository.dart @@ -1,7 +1,6 @@ import 'dart:async'; import 'dart:math'; -import 'package:ardrive/arns/domain/arns_repository.dart'; import 'package:ardrive/blocs/constants.dart'; import 'package:ardrive/core/crypto/crypto.dart'; import 'package:ardrive/entities/constants.dart'; @@ -26,6 +25,7 @@ import 'package:ardrive/sync/domain/sync_cancellation_token.dart'; import 'package:ardrive/sync/domain/sync_failure_simulator.dart'; import 'package:ardrive/sync/domain/sync_progress.dart'; import 'package:ardrive/sync/utils/batch_processor.dart'; +import 'package:ardrive/sync/utils/bounded_worker_pool.dart'; import 'package:ardrive/sync/utils/network_transaction_utils.dart'; import 'package:ardrive/user/repositories/user_preferences_repository.dart'; import 'package:ardrive/utils/logger.dart'; @@ -36,7 +36,6 @@ import 'package:ardrive/utils/snapshots/range.dart'; import 'package:ardrive/utils/snapshots/snapshot_drive_history.dart'; import 'package:ardrive/utils/snapshots/snapshot_item.dart'; import 'package:ardrive_utils/ardrive_utils.dart'; -import 'package:ario_sdk/ario_sdk.dart'; import 'package:arweave/arweave.dart'; import 'package:cryptography/cryptography.dart'; import 'package:drift/drift.dart'; @@ -125,7 +124,6 @@ abstract class SyncRepository { required ConfigService configService, required BatchProcessor batchProcessor, required SnapshotValidationService snapshotValidationService, - required ARNSRepository arnsRepository, required UserPreferencesRepository userPreferencesRepository, }) { return _SyncRepository( @@ -134,7 +132,6 @@ abstract class SyncRepository { configService: configService, batchProcessor: batchProcessor, snapshotValidationService: snapshotValidationService, - arnsRepository: arnsRepository, userPreferencesRepository: userPreferencesRepository, ); } @@ -146,7 +143,6 @@ class _SyncRepository implements SyncRepository { final ConfigService _configService; final BatchProcessor _batchProcessor; final SnapshotValidationService _snapshotValidationService; - final ARNSRepository _arnsRepository; final UserPreferencesRepository _userPreferencesRepository; final Map _ghostFolders = {}; @@ -171,15 +167,13 @@ class _SyncRepository implements SyncRepository { required ConfigService configService, required BatchProcessor batchProcessor, required SnapshotValidationService snapshotValidationService, - required ARNSRepository arnsRepository, required UserPreferencesRepository userPreferencesRepository, }) : _arweave = arweave, _driveDao = driveDao, _configService = configService, _snapshotValidationService = snapshotValidationService, _batchProcessor = batchProcessor, - _userPreferencesRepository = userPreferencesRepository, - _arnsRepository = arnsRepository; + _userPreferencesRepository = userPreferencesRepository; @override Stream syncAllDrives({ @@ -206,13 +200,6 @@ class _SyncRepository implements SyncRepository { String? walletAddress; if (wallet != null) { walletAddress = await wallet.getAddress(); - - _arnsRepository - .getAntRecordsForWallet(walletAddress, update: true) - .catchError((e) { - logger.e('Error getting ANT records for wallet. Continuing...', e); - return Future.value([]); - }); } // Sync the contents of each drive attached in the app. @@ -423,10 +410,13 @@ class _SyncRepository implements SyncRepository { // Track if sync was cancelled bool wasCancelled = false; - // Start the async work but don't wait for it yet - // Using Future.wait with eagerError: false to continue even if some drives fail - Future.wait( - drivesToSync.map((drive) async { + // Start the async work but don't wait for it yet. Drives are synced + // through a bounded worker pool (config.maxConcurrentDriveSyncs at a time) so + // large accounts don't fan out one full sync pipeline per drive at once; + // like Future.wait(eagerError: false), all drives are processed even if + // some fail. + final driveSyncTasks = + drivesToSync.map((drive) => () async { try { // Check for cancellation before starting each drive token.checkCancellation(); @@ -441,8 +431,12 @@ class _SyncRepository implements SyncRepository { ? 0 : _calculateSyncLastBlockHeight(drive.lastBlockHeight ?? 0), currentBlockHeight: currentBlockHeight, - transactionParseBatchSize: - 200 ~/ (syncProgress.drivesCount - syncProgress.drivesSynced), + transactionParseBatchSize: calculateTransactionParseBatchSize( + drivesCount: syncProgress.drivesCount, + drivesSynced: syncProgress.drivesSynced, + maxConcurrentDriveSyncs: + _configService.config.maxConcurrentDriveSyncs.clamp(1, 64), + ), ownerAddress: drive.ownerAddress, txFechedCallback: txFechedCallback, cancellationToken: token, @@ -502,8 +496,11 @@ class _SyncRepository implements SyncRepository { ); syncProgressController.add(syncProgress); } - }), - eagerError: false, // Continue processing even if some drives fail + }).toList(); + + runBoundedWorkers( + tasks: driveSyncTasks, + maxConcurrent: _configService.config.maxConcurrentDriveSyncs.clamp(1, 64), ).then((_) async { try { // If sync was cancelled during drive sync, add error to stream @@ -586,9 +583,6 @@ class _SyncRepository implements SyncRepository { } // Clear cached transaction IDs now that we've used them SnapshotItemOnChain.clearAllCachedTransactionIds(); - _arnsRepository - .waitForARNSRecordsToUpdate() - .then((value) => _arnsRepository.saveAllFilesWithAssignedNames()); final hasHiddenItems = await _driveDao.hasHiddenItems().getSingle(); await _userPreferencesRepository.saveUserHasHiddenItem(hasHiddenItems); await _userPreferencesRepository.load(); @@ -2159,9 +2153,22 @@ class _SyncRepository implements SyncRepository { continue; } - newRevisions.add(revision); - latestRevisions[entity.id!] = revision; - latestRevisionsCache[entity.id!] = revision; + // Guard against out-of-order arrival (pagination phase restarts can + // interleave heights): only a strictly newer revision may become the + // latest, mirroring the file-revision logic above. + if (latestRevisions.containsKey(entity.id)) { + final latestRevision = latestRevisions[entity.id]; + if (revision.dateCreated.value + .isAfter(latestRevision!.dateCreated.value)) { + latestRevisions[entity.id!] = revision; + latestRevisionsCache[entity.id!] = revision; + newRevisions.add(revision); + } + } else { + latestRevisions[entity.id!] = revision; + latestRevisionsCache[entity.id!] = revision; + newRevisions.add(revision); + } } final newNetworkTransactions = createNetworkTransactionsCompanionsForFolders( @@ -2187,6 +2194,26 @@ class _SyncRepository implements SyncRepository { const fetchPhaseWeight = 0.1; const parsePhaseWeight = 0.9; +/// Splits the 200-transaction parse budget across the drives that still need +/// syncing, clamped so the result is always at least 1. Without the clamp, +/// wallets with more than 200 drives would compute a batch size of 0 and +/// [BatchProcessor.batchProcess] would throw, failing every drive sync. +int calculateTransactionParseBatchSize({ + required int drivesCount, + required int drivesSynced, + required int maxConcurrentDriveSyncs, +}) { + final remainingDrives = max(1, drivesCount - drivesSynced); + // Only [maxConcurrentDriveSyncs] drives sync at once, so divide the parse + // budget across the drives ACTUALLY running concurrently rather than all + // remaining ones. Dividing by every remaining drive under-shoots the batch + // size once the account exceeds the concurrency bound (e.g. 200 drives -> + // batch 1 instead of ~4), needlessly slowing large-account syncs. + final concurrentDrives = + min(remainingDrives, max(1, maxConcurrentDriveSyncs)); + return max(1, 200 ~/ concurrentDrives); +} + /// Computes the refreshed file entries from the provided revisions and returns them as a map keyed by their ids. Future> _computeRefreshedFileEntriesFromRevisions({ diff --git a/lib/sync/utils/bounded_worker_pool.dart b/lib/sync/utils/bounded_worker_pool.dart new file mode 100644 index 0000000000..e3e82816cf --- /dev/null +++ b/lib/sync/utils/bounded_worker_pool.dart @@ -0,0 +1,50 @@ +import 'dart:math'; + +/// Runs [tasks] with at most [maxConcurrent] executing at any moment. +/// +/// Tasks are started in order as workers free up. All tasks run to completion +/// even if some fail, mirroring `Future.wait(..., eagerError: false)`: when +/// one or more tasks throw, the returned future completes with the first +/// error only after every task has finished. +Future runBoundedWorkers({ + required List Function()> tasks, + required int maxConcurrent, +}) async { + if (maxConcurrent <= 0) { + throw ArgumentError.value( + maxConcurrent, + 'maxConcurrent', + 'must be at least 1', + ); + } + + if (tasks.isEmpty) { + return; + } + + var nextTaskIndex = 0; + Object? firstError; + StackTrace? firstStackTrace; + + Future worker() async { + while (nextTaskIndex < tasks.length) { + final task = tasks[nextTaskIndex++]; + try { + await task(); + } catch (e, stackTrace) { + if (firstError == null) { + firstError = e; + firstStackTrace = stackTrace; + } + } + } + } + + await Future.wait( + List.generate(min(maxConcurrent, tasks.length), (_) => worker()), + ); + + if (firstError != null) { + return Future.error(firstError!, firstStackTrace); + } +} diff --git a/lib/user/repositories/user_repository.dart b/lib/user/repositories/user_repository.dart index 6c0919c9df..3ec01a0084 100644 --- a/lib/user/repositories/user_repository.dart +++ b/lib/user/repositories/user_repository.dart @@ -30,6 +30,8 @@ abstract class UserRepository { } class _UserRepository implements UserRepository { + static const _walletBalanceLoginTimeout = Duration(seconds: 5); + final ProfileDao _profileDao; final ArweaveService _arweave; final ArioSDK _arioSDK; @@ -55,16 +57,15 @@ class _UserRepository implements UserRepository { } final profileDetails = await _profileDao.loadDefaultProfile(password); + final walletAddress = await profileDetails.wallet.getAddress(); final user = User( profileType: ProfileType.values[profileDetails.details.profileType], wallet: profileDetails.wallet, cipherKey: profileDetails.key, password: password, - walletAddress: await profileDetails.wallet.getAddress(), - walletBalance: await _arweave.getWalletBalance( - await profileDetails.wallet.getAddress(), - ), + walletAddress: walletAddress, + walletBalance: await _getWalletBalanceSafely(walletAddress), errorFetchingIOTokens: false, sourceWalletAddress: profileDetails.details.sourceWalletAddress, ); @@ -74,6 +75,22 @@ class _UserRepository implements UserRepository { return user; } + /// Fetching the balance at login is best-effort: unlocking a locally stored + /// profile must not fail or hang because the balance endpoint is + /// unavailable. On error or timeout we fall back to zero and ArDriveAuth + /// refreshes the balance asynchronously right after login. + Future _getWalletBalanceSafely(String walletAddress) async { + try { + return await _arweave + .getWalletBalance(walletAddress) + .timeout(_walletBalanceLoginTimeout); + } catch (e) { + logger + .w('Failed to fetch wallet balance during login, defaulting to 0: $e'); + return BigInt.zero; + } + } + @override Future saveUser( String password, diff --git a/lib/utils/graphql_retry.dart b/lib/utils/graphql_retry.dart index c61802036d..77c2d031e7 100644 --- a/lib/utils/graphql_retry.dart +++ b/lib/utils/graphql_retry.dart @@ -9,15 +9,20 @@ import 'package:retry/retry.dart'; /// Retry every GraphQL query for `ArtemisClient` /// -/// On 429 or 5xx errors, falls back to arweave.net/graphql (Goldsky proxy) -/// since most AR.IO gateways don't index ArDrive L2 data. +/// On 429 or 5xx errors, falls back to the Goldsky search index directly +/// (arweave.net/graphql proxies to it but rate-limits aggressively) since +/// most AR.IO gateways don't index ArDrive L2 data. +/// +/// Note: Goldsky caps page size at 100 and, when asked for more, silently +/// clamps to 100 while reporting hasNextPage: false — so queries sent through +/// this fallback must never request more than 100 items per page. class GraphQLRetry { GraphQLRetry(this._client, {required InternetChecker internetChecker, String? fallbackGraphqlUrl}) : _internetChecker = internetChecker, _fallbackGraphqlUrl = - fallbackGraphqlUrl ?? 'https://arweave.net/graphql'; + fallbackGraphqlUrl ?? 'https://arweave-search.goldsky.com/graphql'; final ArtemisClient _client; final InternetChecker _internetChecker; diff --git a/packages/ardrive_ui/pubspec.yaml b/packages/ardrive_ui/pubspec.yaml index 4b000f3506..33a85953d2 100644 --- a/packages/ardrive_ui/pubspec.yaml +++ b/packages/ardrive_ui/pubspec.yaml @@ -23,7 +23,10 @@ dependencies: percent_indicator: ^4.2.2 flutter_svg_image: provider: ^6.0.5 - equatable: ^2.0.5 + # pinned below 2.1.0: equatable 2.1.0 deprecates EquatableMixin (used by + # data_table.dart) and the main app's lockfile resolves 2.0.7; this package + # has no committed lockfile, so CI floats to the newest allowed version + equatable: '>=2.0.5 <2.1.0' ardrive_utils: path: ../ardrive_utils diff --git a/test/gar/domain/repository/gar_repository_test.dart b/test/gar/domain/repository/gar_repository_test.dart index 3db0aae599..a69ead6253 100644 --- a/test/gar/domain/repository/gar_repository_test.dart +++ b/test/gar/domain/repository/gar_repository_test.dart @@ -1,4 +1,5 @@ import 'package:ardrive/gar/domain/repositories/gar_repository.dart'; +import 'package:ardrive/services/arweave/data_gateway_fallback.dart'; import 'package:ardrive/services/arweave/arweave_service.dart'; import 'package:ardrive/services/config/app_config.dart'; import 'package:ardrive/services/config/config_service.dart'; @@ -14,6 +15,8 @@ class MockConfigService extends Mock implements ConfigService {} class MockArweaveService extends Mock implements ArweaveService {} +class MockDataGatewayFallback extends Mock implements DataGatewayFallback {} + class MockGateway extends Mock implements Gateway {} class MockConfig extends Mock implements AppConfig {} @@ -27,12 +30,15 @@ void main() { late MockArioSDK arioSDK; late MockConfigService configService; late MockArweaveService arweaveService; + late MockDataGatewayFallback gatewayFallback; late MockArDriveHTTP http; setUp(() { arioSDK = MockArioSDK(); configService = MockConfigService(); arweaveService = MockArweaveService(); + gatewayFallback = MockDataGatewayFallback(); + when(() => arweaveService.gatewayFallback).thenReturn(gatewayFallback); http = MockArDriveHTTP(); repository = GarRepositoryImpl( arioSDK: arioSDK, @@ -56,14 +62,30 @@ void main() { group('GarRepositoryImpl', () { group('getGateways', () { - test('fetches and returns gateways', () async { + test('serves gateways from the shared cache without hitting the SDK', + () async { final gateways = [MockGateway()]; - when(() => arioSDK.getGateways()).thenAnswer((_) async => gateways); + when(() => gatewayFallback.getGatewaysCached()) + .thenAnswer((_) async => gateways); final result = await repository.getGateways(); expect(result, equals(gateways)); - verify(() => arioSDK.getGateways()).called(1); + verify(() => gatewayFallback.getGatewaysCached()).called(1); + verifyNever(() => arioSDK.getGateways()); + }); + }); + + group('refreshGateways', () { + test('forces a network refresh through the shared cache', () async { + final gateways = [MockGateway()]; + when(() => gatewayFallback.refreshGateways()) + .thenAnswer((_) async => gateways); + + final result = await repository.refreshGateways(); + + expect(result, equals(gateways)); + verify(() => gatewayFallback.refreshGateways()).called(1); }); }); @@ -137,7 +159,8 @@ void main() { url: 'https://ardrive.net', ), )); - when(() => arioSDK.getGateways()).thenAnswer((_) async => gateways); + when(() => gatewayFallback.getGatewaysCached()) + .thenAnswer((_) async => gateways); when(() => gateway.settings).thenReturn(settings); when(() => settings.label).thenReturn('New Gateway'); @@ -171,7 +194,8 @@ void main() { url: 'https://not.in.list.com', ), )); - when(() => arioSDK.getGateways()).thenAnswer((_) async => gateways); + when(() => gatewayFallback.getGatewaysCached()) + .thenAnswer((_) async => gateways); when(() => gateway1.settings).thenReturn(settings); when(() => gateway2.settings).thenReturn(settings); when(() => gateway3.settings).thenReturn(settings); @@ -244,7 +268,8 @@ void main() { when(() => settings2.label).thenReturn('second gateway'); when(() => settings3.label).thenReturn('first'); - when(() => arioSDK.getGateways()).thenAnswer((_) async => gateways); + when(() => gatewayFallback.getGatewaysCached()) + .thenAnswer((_) async => gateways); // Manually populate the _gateways list await repository.getGateways(); @@ -286,7 +311,8 @@ void main() { when(() => settings2.label).thenReturn('Beta Gateway'); when(() => settings3.label).thenReturn('Gamma Gateway'); - when(() => arioSDK.getGateways()).thenAnswer((_) async => gateways); + when(() => gatewayFallback.getGatewaysCached()) + .thenAnswer((_) async => gateways); // Manually populate the _gateways list await repository.getGateways(); diff --git a/test/services/arweave/data_gateway_fallback_test.dart b/test/services/arweave/data_gateway_fallback_test.dart new file mode 100644 index 0000000000..b37e9fbf28 --- /dev/null +++ b/test/services/arweave/data_gateway_fallback_test.dart @@ -0,0 +1,227 @@ +import 'dart:async'; +import 'dart:convert'; + +import 'package:ardrive/services/arweave/data_gateway_fallback.dart'; +import 'package:ardrive/utils/key_value_store.dart'; +import 'package:ario_sdk/ario_sdk.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:mocktail/mocktail.dart'; + +class MockArioSDK extends Mock implements ArioSDK {} + +class FakeKeyValueStore implements KeyValueStore { + final Map _values = {}; + + @override + FutureOr getBool(String key) => _values[key] as bool?; + + @override + FutureOr getString(String key) => _values[key] as String?; + + @override + FutureOr> getKeys() => _values.keys.toSet(); + + @override + Future putBool(String key, bool value) async { + _values[key] = value; + return true; + } + + @override + Future putString(String key, String value) async { + _values[key] = value; + return true; + } + + @override + Future remove(String key) async { + _values.remove(key); + return true; + } +} + +Gateway _makeGateway(String fqdn) { + return Gateway( + operatorStake: 1000, + gatewayAddress: 'gateway-address-$fqdn', + observerAddress: 'observer-address-$fqdn', + settings: Settings( + port: 443, + protocol: 'https', + allowDelegatedStaking: true, + fqdn: fqdn, + delegateRewardShareRatio: 10, + properties: '', + note: '', + minDelegatedStake: 100, + label: 'Gateway $fqdn', + autoStake: true, + ), + startTimestamp: 0, + totalDelegatedStake: 0, + stats: Stats( + failedConsecutiveEpochs: 0, + observedEpochCount: 1, + passedConsecutiveEpochs: 1, + totalEpochCount: 1, + prescribedEpochCount: 1, + passedEpochCount: 1, + failedEpochCount: 0, + ), + status: 'joined', + ); +} + +void main() { + const cacheKey = 'gar_gateways_cache_v1'; + + late MockArioSDK sdk; + late FakeKeyValueStore store; + late DataGatewayFallback fallback; + + setUp(() { + sdk = MockArioSDK(); + store = FakeKeyValueStore(); + fallback = DataGatewayFallback(arioSDK: sdk, store: store); + }); + + group('DataGatewayFallback.getGatewaysCached', () { + test('serves the persisted list without calling the SDK', () async { + final persisted = [_makeGateway('persisted.gateway.com')]; + await store.putString( + cacheKey, + json.encode(persisted.map((g) => g.toJson()).toList()), + ); + + final result = await fallback.getGatewaysCached(); + + expect(result, hasLength(1)); + expect(result.first.settings.fqdn, 'persisted.gateway.com'); + verifyNever(() => sdk.getGateways()); + }); + + test('fetches from the SDK once and persists when nothing is stored', + () async { + final fetched = [_makeGateway('fetched.gateway.com')]; + when(() => sdk.getGateways()).thenAnswer((_) async => fetched); + + final first = await fallback.getGatewaysCached(); + final second = await fallback.getGatewaysCached(); + + expect(first, equals(fetched)); + expect(second, equals(fetched)); + verify(() => sdk.getGateways()).called(1); + + final raw = await store.getString(cacheKey); + expect(raw, isNotNull, reason: 'fetched list must be persisted'); + final roundTripped = (json.decode(raw!) as List) + .map((e) => Gateway.fromJson(e as Map)) + .toList(); + expect(roundTripped.single.settings.fqdn, 'fetched.gateway.com'); + }); + + test('a fresh instance reads the persisted list instead of refetching', + () async { + final fetched = [_makeGateway('fetched.gateway.com')]; + when(() => sdk.getGateways()).thenAnswer((_) async => fetched); + await fallback.getGatewaysCached(); + + // Simulate a new app session sharing the same storage. + final newSdk = MockArioSDK(); + final newSession = DataGatewayFallback(arioSDK: newSdk, store: store); + + final result = await newSession.getGatewaysCached(); + + expect(result.single.settings.fqdn, 'fetched.gateway.com'); + verifyNever(() => newSdk.getGateways()); + }); + + test('caches an empty list on SDK failure and does not retry or persist', + () async { + when(() => sdk.getGateways()).thenThrow(Exception('rpc down')); + + final first = await fallback.getGatewaysCached(); + final second = await fallback.getGatewaysCached(); + + expect(first, isEmpty); + expect(second, isEmpty); + verify(() => sdk.getGateways()).called(1); + expect(store.getString(cacheKey), isNull, + reason: 'failures must not be persisted'); + }); + + test('recovers from a corrupt persisted entry by refetching', () async { + await store.putString(cacheKey, 'not-json'); + final fetched = [_makeGateway('fetched.gateway.com')]; + when(() => sdk.getGateways()).thenAnswer((_) async => fetched); + + final result = await fallback.getGatewaysCached(); + + expect(result.single.settings.fqdn, 'fetched.gateway.com'); + verify(() => sdk.getGateways()).called(1); + }); + }); + + group('DataGatewayFallback.refreshGateways', () { + test('always hits the SDK and replaces the persisted list', () async { + final original = [_makeGateway('old.gateway.com')]; + await store.putString( + cacheKey, + json.encode(original.map((g) => g.toJson()).toList()), + ); + await fallback.getGatewaysCached(); + + final refreshed = [_makeGateway('new.gateway.com')]; + when(() => sdk.getGateways()).thenAnswer((_) async => refreshed); + + final result = await fallback.refreshGateways(); + + expect(result.single.settings.fqdn, 'new.gateway.com'); + verify(() => sdk.getGateways()).called(1); + + final raw = await store.getString(cacheKey); + final roundTripped = (json.decode(raw!) as List) + .map((e) => Gateway.fromJson(e as Map)) + .toList(); + expect(roundTripped.single.settings.fqdn, 'new.gateway.com'); + + // Subsequent cached reads serve the refreshed list. + final cached = await fallback.getGatewaysCached(); + expect(cached.single.settings.fqdn, 'new.gateway.com'); + verifyNoMoreInteractions(sdk); + }); + + test('propagates SDK errors and preserves the existing cache', () async { + final original = [_makeGateway('old.gateway.com')]; + await store.putString( + cacheKey, + json.encode(original.map((g) => g.toJson()).toList()), + ); + await fallback.getGatewaysCached(); + + when(() => sdk.getGateways()).thenThrow(Exception('rpc down')); + + await expectLater(fallback.refreshGateways(), throwsException); + + // The in-memory cache and the persisted list keep the previous values. + final cached = await fallback.getGatewaysCached(); + expect(cached.single.settings.fqdn, 'old.gateway.com'); + final raw = await store.getString(cacheKey); + final persisted = (json.decode(raw!) as List) + .map((e) => Gateway.fromJson(e as Map)) + .toList(); + expect(persisted.single.settings.fqdn, 'old.gateway.com'); + }); + + test('throws on a stalled SDK call instead of hanging', () async { + when(() => sdk.getGateways()).thenAnswer( + (_) => Completer>().future, // never completes + ); + + await expectLater( + fallback.refreshGateways(), + throwsA(isA()), + ); + }); + }); +} diff --git a/test/services/config/app_config_defaults_test.dart b/test/services/config/app_config_defaults_test.dart new file mode 100644 index 0000000000..feaedcb3bd --- /dev/null +++ b/test/services/config/app_config_defaults_test.dart @@ -0,0 +1,36 @@ +import 'dart:convert'; + +import 'package:ardrive/services/config/app_config.dart'; +import 'package:flutter_test/flutter_test.dart'; + +void main() { + group('AppConfig sync tuning fields', () { + test('fall back to safe defaults when absent from stored config JSON', + () { + // Simulates a config persisted by an older app version (configVersion 3 + // era) that predates the sync tuning fields. + final config = AppConfig.fromJson(const { + 'allowedDataItemSizeForTurbo': 100000, + 'stripePublishableKey': '', + }); + + expect(config.maxConcurrentDriveSyncs, 50); + }); + + test('round-trips through toJson/fromJson', () { + final config = AppConfig( + allowedDataItemSizeForTurbo: 100000, + stripePublishableKey: '', + maxConcurrentDriveSyncs: 8, + ); + + // Round-trip through a real encode/decode cycle: AppConfig's toJson + // embeds SelectedGateway as an object (no explicitToJson), which only + // becomes a map through jsonEncode — matching how configs are stored. + final restored = AppConfig.fromJson( + json.decode(json.encode(config)) as Map); + + expect(restored.maxConcurrentDriveSyncs, 8); + }); + }); +} diff --git a/test/services/config/config_fetcher_test.dart b/test/services/config/config_fetcher_test.dart index deaf0ab8e0..859711a805 100644 --- a/test/services/config/config_fetcher_test.dart +++ b/test/services/config/config_fetcher_test.dart @@ -119,10 +119,40 @@ void main() { // Act final result = await configFetcher.fetchConfig(Flavor.development); - // Assert + // Assert: config replaced, but gateway choices that differ from the + // previous defaults are deliberate user choices and are preserved. expect(result.configVersion, 2); expect(result.stripePublishableKey, 'new-key'); - verify(() => localStore.putString('config', newConfigString)).called(1); + expect(result.arweaveGatewayUrl, 'old-gateway'); + expect(result.arweaveGatewayForDataRequest.url, 'old'); + verify(() => localStore.putString('config', any())).called(1); + }); + + test( + 'version bump replaces gateways that still match the previous ' + 'defaults', () async { + final oldConfig = AppConfig( + configVersion: 1, + stripePublishableKey: 'old-key', + allowedDataItemSizeForTurbo: 100, + // Previous defaults: the user never customized these. + arweaveGatewayUrl: 'https://ardrive.net', + arweaveGatewayForDataRequest: const SelectedGateway( + label: 'Turbo Gateway', + url: 'https://turbo-gateway.com', + ), + ); + when(() => localStore.getString('config')) + .thenReturn(json.encode(oldConfig.toJson())); + when(() => assetBundle.loadString(any())) + .thenAnswer((_) async => newConfigString); + when(() => localStore.putString('config', any())) + .thenAnswer((_) async => true); + + final result = await configFetcher.fetchConfig(Flavor.development); + + expect(result.arweaveGatewayUrl, 'new-gateway'); + expect(result.arweaveGatewayForDataRequest.url, 'new'); }); test( @@ -150,10 +180,13 @@ void main() { // Act final result = await configFetcher.fetchConfig(Flavor.development); - // Assert + // Assert: replaced, but custom gateway choices are preserved (they + // differ from the previous defaults). expect(result.configVersion, 2); expect(result.stripePublishableKey, 'new-key'); - verify(() => localStore.putString('config', newConfigString)).called(1); + expect(result.arweaveGatewayUrl, 'old-gateway'); + expect(result.arweaveGatewayForDataRequest.url, 'old'); + verify(() => localStore.putString('config', any())).called(1); }); test('replaces local config with asset config if local config is malformed', diff --git a/test/sync/domain/sync_repository_optimization_test.dart b/test/sync/domain/sync_repository_optimization_test.dart index a284b6d360..ebae8bfd0d 100644 --- a/test/sync/domain/sync_repository_optimization_test.dart +++ b/test/sync/domain/sync_repository_optimization_test.dart @@ -1,4 +1,3 @@ -import 'package:ardrive/arns/domain/arns_repository.dart'; import 'package:ardrive/models/database/database.dart'; import 'package:ardrive/services/arweave/data_gateway_fallback.dart'; import 'package:ardrive/services/config/app_config.dart'; @@ -9,7 +8,6 @@ import 'package:ardrive/user/repositories/user_preferences_repository.dart'; import 'package:ardrive/utils/snapshots/gql_drive_history.dart'; import 'package:ardrive/utils/snapshots/height_range.dart'; import 'package:ardrive/utils/snapshots/range.dart'; -import 'package:ario_sdk/ario_sdk.dart'; import 'package:arweave/arweave.dart'; import 'package:cryptography/cryptography.dart'; import 'package:drift/drift.dart'; @@ -24,8 +22,6 @@ class MockBatchProcessor extends Mock implements BatchProcessor {} class MockSnapshotValidationService extends Mock implements SnapshotValidationService {} -class _MockARNSRepository extends Mock implements ARNSRepository {} - class MockUserPreferencesRepository extends Mock implements UserPreferencesRepository {} @@ -81,7 +77,6 @@ void main() { late MockConfigService mockConfigService; late MockBatchProcessor mockBatchProcessor; late MockSnapshotValidationService mockSnapshotValidation; - late _MockARNSRepository mockArnsRepository; late MockUserPreferencesRepository mockUserPrefsRepo; late SyncRepository syncRepository; late _MockWallet mockWallet; @@ -94,7 +89,6 @@ void main() { mockConfigService = MockConfigService(); mockBatchProcessor = MockBatchProcessor(); mockSnapshotValidation = MockSnapshotValidationService(); - mockArnsRepository = _MockARNSRepository(); mockUserPrefsRepo = MockUserPreferencesRepository(); mockWallet = _MockWallet(); @@ -111,12 +105,8 @@ void main() { final mockGatewayFallback = _MockDataGatewayFallback(); when(() => mockArweave.gatewayFallback).thenReturn(mockGatewayFallback); when(() => mockGatewayFallback.cachedGateways).thenReturn([]); - - // Mock ARNS repository - when(() => mockArnsRepository.getAntRecordsForWallet(any(), - update: any(named: 'update'))).thenAnswer( - (_) async => [], - ); + when(() => mockGatewayFallback.getGatewaysCached()) + .thenAnswer((_) async => []); syncRepository = SyncRepository( arweave: mockArweave, @@ -124,7 +114,6 @@ void main() { configService: mockConfigService, batchProcessor: mockBatchProcessor, snapshotValidationService: mockSnapshotValidation, - arnsRepository: mockArnsRepository, userPreferencesRepository: mockUserPrefsRepo, ); }); diff --git a/test/sync/domain/transaction_parse_batch_size_test.dart b/test/sync/domain/transaction_parse_batch_size_test.dart new file mode 100644 index 0000000000..ec103e131b --- /dev/null +++ b/test/sync/domain/transaction_parse_batch_size_test.dart @@ -0,0 +1,118 @@ +import 'package:ardrive/sync/domain/repositories/sync_repository.dart'; +import 'package:flutter_test/flutter_test.dart'; + +void main() { + group('calculateTransactionParseBatchSize', () { + test('gives the full budget to a single drive', () { + expect( + calculateTransactionParseBatchSize( + drivesCount: 1, + drivesSynced: 0, + maxConcurrentDriveSyncs: 50, + ), + 200, + ); + }); + + test('splits the budget across remaining drives (below the bound)', () { + expect( + calculateTransactionParseBatchSize( + drivesCount: 4, + drivesSynced: 0, + maxConcurrentDriveSyncs: 50, + ), + 50, + ); + expect( + calculateTransactionParseBatchSize( + drivesCount: 4, + drivesSynced: 2, + maxConcurrentDriveSyncs: 50, + ), + 100, + ); + }); + + test( + 'divides by the concurrency bound, not total remaining, once the ' + 'account exceeds it (the fix — 200 drives at 50-wide gives 4, not 1)', + () { + expect( + calculateTransactionParseBatchSize( + drivesCount: 200, + drivesSynced: 0, + maxConcurrentDriveSyncs: 50, + ), + 4, + ); + // Even more drives: still bounded by the concurrency, so still 4. + expect( + calculateTransactionParseBatchSize( + drivesCount: 1000, + drivesSynced: 0, + maxConcurrentDriveSyncs: 50, + ), + 4, + ); + }); + + test('uses remaining drives when they are fewer than the bound', () { + // 10 remaining, 50-wide bound -> divide by 10. + expect( + calculateTransactionParseBatchSize( + drivesCount: 10, + drivesSynced: 0, + maxConcurrentDriveSyncs: 50, + ), + 20, + ); + }); + + test('never returns 0 for wallets with 200 or more drives', () { + for (final drivesCount in [200, 201, 250, 1000]) { + expect( + calculateTransactionParseBatchSize( + drivesCount: drivesCount, + drivesSynced: 0, + maxConcurrentDriveSyncs: 50, + ), + greaterThanOrEqualTo(1), + reason: 'batch size must stay positive for $drivesCount drives', + ); + } + }); + + test('stays positive even with a degenerate concurrency value', () { + expect( + calculateTransactionParseBatchSize( + drivesCount: 100, + drivesSynced: 0, + maxConcurrentDriveSyncs: 0, + ), + greaterThanOrEqualTo(1), + ); + }); + + test('never divides by zero when all drives are synced', () { + expect( + calculateTransactionParseBatchSize( + drivesCount: 5, + drivesSynced: 5, + maxConcurrentDriveSyncs: 50, + ), + 200, + ); + }); + + test('never divides by zero when synced exceeds count', () { + expect( + calculateTransactionParseBatchSize( + drivesCount: 5, + drivesSynced: 6, + maxConcurrentDriveSyncs: 50, + ), + 200, + ); + }); + }); +} diff --git a/test/sync/utils/bounded_worker_pool_test.dart b/test/sync/utils/bounded_worker_pool_test.dart new file mode 100644 index 0000000000..f94d1308ad --- /dev/null +++ b/test/sync/utils/bounded_worker_pool_test.dart @@ -0,0 +1,87 @@ +import 'package:ardrive/sync/utils/bounded_worker_pool.dart'; +import 'package:flutter_test/flutter_test.dart'; + +void main() { + group('runBoundedWorkers', () { + test('runs every task exactly once', () async { + final executed = []; + + await runBoundedWorkers( + tasks: List.generate(20, (i) => () async => executed.add(i)), + maxConcurrent: 3, + ); + + expect(executed, hasLength(20)); + expect(executed.toSet(), List.generate(20, (i) => i).toSet()); + }); + + test('never exceeds maxConcurrent in-flight tasks', () async { + const maxConcurrent = 4; + var inFlight = 0; + var maxObserved = 0; + + await runBoundedWorkers( + tasks: List.generate(25, (i) => () async { + inFlight++; + if (inFlight > maxObserved) maxObserved = inFlight; + // Yield a few times so other workers get a chance to overlap. + await Future.delayed(Duration.zero); + await Future.delayed(Duration.zero); + inFlight--; + }), + maxConcurrent: maxConcurrent, + ); + + expect(maxObserved, lessThanOrEqualTo(maxConcurrent)); + expect(maxObserved, greaterThan(1), + reason: 'tasks should actually overlap'); + }); + + test('runs all tasks even when some fail, then reports the first error', + () async { + final executed = []; + + Object? caught; + try { + await runBoundedWorkers( + tasks: List.generate(10, (i) => () async { + executed.add(i); + if (i == 2) throw StateError('task 2 failed'); + }), + maxConcurrent: 2, + ); + } catch (e) { + caught = e; + } + + expect(executed, hasLength(10), + reason: 'a failing task must not stop the remaining tasks'); + expect(caught, isA()); + }); + + test('handles more workers than tasks', () async { + var count = 0; + + await runBoundedWorkers( + tasks: List.generate(2, (_) => () async => count++), + maxConcurrent: 10, + ); + + expect(count, 2); + }); + + test('completes immediately for an empty task list', () async { + await expectLater( + runBoundedWorkers(tasks: [], maxConcurrent: 3), + completes, + ); + }); + + test('throws ArgumentError for a non-positive maxConcurrent', () { + expect( + () => runBoundedWorkers(tasks: [() async {}], maxConcurrent: 0), + throwsArgumentError, + ); + }); + }); +} diff --git a/test/user/repositories/user_repository_test.dart b/test/user/repositories/user_repository_test.dart index 338fece053..f8f39c418d 100644 --- a/test/user/repositories/user_repository_test.dart +++ b/test/user/repositories/user_repository_test.dart @@ -101,6 +101,31 @@ void main() { .called(1); }); + test( + 'should still return a user with zero balance when the balance ' + 'fetch fails', () async { + when(() => mockArweaveService.getWalletBalance(any())) + .thenThrow(Exception('gateway unavailable')); + + final result = await userRepository.getUser(rightPassword); + + expect(result, isNotNull); + expect(result!.walletBalance, BigInt.zero); + expect(result.walletAddress, await wallet.getAddress()); + }); + + test( + 'should still return a user with zero balance when the balance ' + 'fetch rejects asynchronously', () async { + when(() => mockArweaveService.getWalletBalance(any())).thenAnswer( + (_) async => throw Exception('gateway timed out')); + + final result = await userRepository.getUser(rightPassword); + + expect(result, isNotNull); + expect(result!.walletBalance, BigInt.zero); + }); + test('should return null if there is no profile', () async { when(() => mockProfileDao.getDefaultProfile()) .thenAnswer((_) async => Future.value(null));