diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000000..bde5367495 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,187 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +## Development Commands + +### Setup and Build +```bash +# Initial setup - install dependencies and generate code +scr setup + +# Generate code (run after changing .drift files or GraphQL schemas) +flutter pub run build_runner build --delete-conflicting-outputs + +# Watch for changes during development +flutter packages pub run build_runner watch + +# Check Flutter version compliance +scr check-flutter + +# Database schema validation +scr check-db +``` + +### Testing +```bash +# Run all tests (main app + all packages) +scr test + +# Run main app tests only +flutter test + +# Run tests for specific package +cd packages/ardrive_ui && flutter test + +# Run specific test file +flutter test test/blocs/upload_cubit_test.dart +``` + +### Running the App +```bash +# Development environment (web) +flutter run -d chrome --dart-define=environment=development + +# Production environment (web) +flutter run -d chrome --dart-define=environment=production + +# Mobile development +flutter run --flavor=development + +# Mobile production +flutter run --flavor=production +``` + +### Code Quality +```bash +# Analyze code +flutter analyze + +# Format code +dart format . + +# Lint check (via lefthook pre-commit) +lefthook run pre-commit +``` + +## Architecture Overview + +ArDrive is a Flutter web/mobile application for decentralized file storage on Arweave blockchain. + +### Core Architecture Patterns + +**State Management**: BLoC pattern with Cubits +- Complex features (uploads, sync, file operations) use event-driven BLoCs +- Simple UI state uses Cubits +- Dependency injection via Provider + +**Database**: Drift ORM with SQL generation +- Schema versioning with migrations in `drift_schemas/` (current: version 27) +- Core entities: drives, files, folders, licenses, ARNS records, ANT records, network transactions +- DAOs provide repository pattern for data access +- Database resets for schema versions < 24 + +**Upload System**: Multi-strategy architecture +- Direct Arweave uploads vs Turbo bundled uploads +- Payment methods: AR tokens or Turbo credits +- Upload handles abstract different upload patterns +- Real-time progress tracking with cancellation + +### Key Components + +**Authentication**: Multi-wallet support (ArConnect, keyfile, Ethereum) +**Encryption**: End-to-end encryption for private drives +**ARNS Integration**: Decentralized naming system via `ario_sdk` +**GraphQL**: Artemis-generated clients for Arweave gateway queries +**Packages**: Modular local packages in `/packages/` directory: +- `ardrive_ui` - UI Design Library with Storybook +- `ardrive_crypto` - Cryptography utilities +- `ardrive_uploader` - Upload functionality +- `ardrive_utils` - Shared utilities +- `ario_sdk` - ARNS integration +- `arconnect` - Wallet connection +- `pst` - Profit Sharing Token functionality +- `ardrive_logger` - Logging utilities + +### File Structure + +- `lib/blocs/` - State management (BLoCs/Cubits) +- `lib/models/` - Database models and DAOs +- `lib/services/` - External integrations (Arweave, payments, auth) +- `lib/pages/` - UI screens and routing +- `lib/components/` - Reusable UI components +- `packages/` - Local modular packages +- `test/` - Unit and integration tests + +### Database Schema + +When modifying database schema: +1. Update `.drift` files in `lib/models/tables/` or `lib/models/queries/` +2. Run `flutter pub run build_runner build --delete-conflicting-outputs` +3. Update migration logic in `lib/models/database/database.dart` if needed +4. Run `scr check-db` to validate schema changes + +### Adding New Features + +1. Create BLoC/Cubit in `lib/blocs/` +2. Add models/DAOs if database changes needed +3. Create UI components in `lib/components/` or pages in `lib/pages/` +4. Add tests in corresponding `test/` directories +5. Update routing in `lib/pages/app_router_delegate.dart` if needed + +### Commit Message Format + +Use conventional commit prefixes (lowercase): +- `fix:` - Bug fixes +- `feat:` - New features +- `perf:` - Performance improvements +- `docs:` - Documentation changes +- `style:` - Formatting changes +- `refactor:` - Code refactoring +- `test:` - Adding missing tests +- `chore:` - Chore tasks + +Include detailed changes list after summary line if not self-explanatory. + +### Environment Configuration + +The app uses three environments (development, staging, production) with config files in `assets/config/`. Use `--dart-define=environment=` to specify environment when running. + +### Code Generation + +The codebase uses several code generation tools: +- **Artemis**: GraphQL client generation from schema in `lib/services/arweave/graphql/` +- **Drift**: Database schema and DAO generation from `.drift` files +- **JSON Serialization**: Model serialization via `json_annotation` + +Build configuration in `build.yaml` specifies output paths and options. + +### Key Dependencies + +- **Flutter SDK**: 3.19.6 (exact version required) +- **Dart SDK**: >=3.2.0 <4.0.0 +- **State Management**: flutter_bloc ^8.1.1 +- **Database**: drift ^2.12.1 +- **GraphQL**: artemis ^7.0.0-beta.13 +- **Testing**: mocktail, bloc_test, golden tests +- **Mobile**: Firebase integration (Crashlytics, Core) + +### Development Tools + +- **Lefthook**: Git hooks for pre-commit/pre-push validation +- **Script Runner**: Access to custom scripts via `scr` command +- **Flutter Lints**: Code quality enforcement +- **Golden Tests**: UI regression testing support + +### Custom Gateway + +For testing with custom Arweave gateways, set `flutter.arweaveGatewayUrl` in browser localStorage: +```js +localStorage.setItem('flutter.arweaveGatewayUrl', '"https://my.custom.url"'); +``` + +### Release Process + +- **Staging**: All changes to `dev` branch auto-deploy to staging.ardrive.io +- **Production**: Merge `dev` to `master`, create GitHub release with `v*` tag pattern +- **Preview Builds**: PRs to `dev` trigger shareable preview builds \ No newline at end of file diff --git a/lib/blocs/drive_attach/drive_attach_cubit.dart b/lib/blocs/drive_attach/drive_attach_cubit.dart index fc827b9167..36808647dd 100644 --- a/lib/blocs/drive_attach/drive_attach_cubit.dart +++ b/lib/blocs/drive_attach/drive_attach_cubit.dart @@ -101,6 +101,30 @@ class DriveAttachCubit extends Cubit { } } + Future _checkForSnapshots(String driveId) async { + try { + final snapshotsStream = _arweave.getAllSnapshotsOfDrive( + driveId, + null, // No lastBlockHeight filter for checking + ownerAddress: null, // Allow snapshots from any owner + ); + + final snapshots = await snapshotsStream.take(1).toList(); + final hasSnapshots = snapshots.isNotEmpty; + + if (hasSnapshots) { + logger.i('Drive $driveId has snapshots - will enable performance optimization'); + } else { + logger.d('Drive $driveId has no snapshots - will use standard sync'); + } + + return hasSnapshots; + } catch (e) { + logger.w('Error checking for snapshots on drive $driveId: $e'); + return false; + } + } + void submit() async { final driveId = driveIdController.text; final driveName = driveNameController.text; @@ -158,15 +182,37 @@ class DriveAttachCubit extends Cubit { profileKey: _profileKey, ); - emit(DriveAttachSuccess()); - + // Check for snapshots to provide better user feedback + final hasSnapshots = await _checkForSnapshots(driveId); + + // Don't emit success yet - go straight to syncing state + /// Wait for the sync to finish before syncing the newly attached drive. await _syncBloc.waitCurrentSync(); - /// Then, sync and select the newly attached drive. - unawaited(_syncBloc - .startSync() - .then((value) => _drivesBloc.selectDrive(driveId))); + /// Show syncing state while the drive syncs + emit(DriveAttachSyncing(hasSnapshots: hasSnapshots)); + + /// Start the sync in the background + /// Don't await it so the user can close the modal if they want + _syncBloc.syncSingleDrive(driveId).then((_) { + if (!isClosed) { + /// Select the drive after sync completes + _drivesBloc.selectDrive(driveId); + } + }).catchError((err) { + logger.e('Error during background sync of attached drive', err); + }); + + // Give the UI time to show the syncing state before allowing the user to close + await Future.delayed(const Duration(seconds: 2)); + + // Check if still in syncing state (user hasn't closed the modal) + if (!isClosed && state is DriveAttachSyncing) { + /// Emit success to indicate the attach is complete + /// The sync continues in the background + emit(DriveAttachSuccess()); + } PlausibleEventTracker.trackAttachDrive( drivePrivacy: drivePrivacy, diff --git a/lib/blocs/drive_attach/drive_attach_state.dart b/lib/blocs/drive_attach/drive_attach_state.dart index 8d29b42725..7cd30ac159 100644 --- a/lib/blocs/drive_attach/drive_attach_state.dart +++ b/lib/blocs/drive_attach/drive_attach_state.dart @@ -16,6 +16,15 @@ class DriveAttachInProgress extends DriveAttachState {} class DriveAttachSuccess extends DriveAttachState {} +class DriveAttachSyncing extends DriveAttachState { + final bool hasSnapshots; + + DriveAttachSyncing({this.hasSnapshots = false}); + + @override + List get props => [hasSnapshots]; +} + class DriveAttachFailure extends DriveAttachState {} class DriveAttachInvalidDriveKey extends DriveAttachState {} diff --git a/lib/blocs/drives/drives_cubit.dart b/lib/blocs/drives/drives_cubit.dart index dad88a7a95..a156dc1ba0 100644 --- a/lib/blocs/drives/drives_cubit.dart +++ b/lib/blocs/drives/drives_cubit.dart @@ -9,7 +9,6 @@ import 'package:ardrive/models/models.dart'; import 'package:ardrive/user/repositories/user_preferences_repository.dart'; import 'package:ardrive/utils/user_utils.dart'; import 'package:ardrive_utils/ardrive_utils.dart'; -import 'package:drift/drift.dart'; import 'package:equatable/equatable.dart'; import 'package:flutter_bloc/flutter_bloc.dart'; import 'package:rxdart/rxdart.dart'; @@ -50,14 +49,10 @@ class DrivesCubit extends Cubit { _drivesSubscription = Rx.combineLatest3, List, void, List>( - _driveDao.allDrives( - order: (drives) { - return OrderBy([OrderingTerm.asc(drives.name)]); - }, - ).watch(), + _driveDao.allDrives().watch(), _driveDao.ghostFolders().watch(), _profileCubit.stream.startWith(ProfileCheckingAvailability()), - (drives, _, __) => drives, + (drives, _, __) => drives..sort((a, b) => a.name.compareTo(b.name)), ).listen((drives) async { final state = this.state; diff --git a/lib/components/drive_attach_form.dart b/lib/components/drive_attach_form.dart index c7933b9725..c28ad3e28c 100644 --- a/lib/components/drive_attach_form.dart +++ b/lib/components/drive_attach_form.dart @@ -120,6 +120,42 @@ class _DriveAttachFormState extends State { ); } + if (state is DriveAttachSyncing) { + return ArDriveStandardModalNew( + hasCloseButton: true, + title: 'Syncing drive...', + content: SizedBox( + width: kMediumDialogWidth, + child: Column( + mainAxisSize: MainAxisSize.min, + children: [ + const SizedBox(height: 16), + const CircularProgressIndicator(), + const SizedBox(height: 24), + Text( + state.hasSnapshots + ? 'Snapshots detected! Using optimized sync...\n\nThis should be quick!' + : 'Please wait while we sync the drive contents.\n\nThis may take a moment for large drives.', + textAlign: TextAlign.center, + style: ArDriveTypography.body.bodyRegular(), + ), + const SizedBox(height: 16), + Text( + 'You can close this modal and continue using ArDrive.\nThe sync will continue in the background.', + textAlign: TextAlign.center, + style: ArDriveTypography.body.smallRegular( + color: ArDriveTheme.of(context) + .themeData + .colors + .themeFgSubtle, + ), + ), + ], + ), + ), + ); + } + return ArDriveStandardModalNew( title: appLocalizationsOf(context).attachDriveEmphasized, content: SizedBox( diff --git a/lib/components/progress_bar.dart b/lib/components/progress_bar.dart index c18e81a2c9..1c2c74c368 100644 --- a/lib/components/progress_bar.dart +++ b/lib/components/progress_bar.dart @@ -13,23 +13,33 @@ class ProgressBar extends StatefulWidget { class _ProgressBarState extends State { late double _percentage; + double _lastPercentage = 0; + DateTime _lastUpdate = DateTime.now(); @override Widget build(BuildContext context) { return StreamBuilder( stream: widget.percentage, builder: (context, snapshot) { + final now = DateTime.now(); _percentage = snapshot.hasData ? ((snapshot.data!.progress * 100)).roundToDouble() / 100 : 0; + // Disable animation for rapid updates to prevent UI lag + final isRapidUpdate = now.difference(_lastUpdate).inMilliseconds < 200; + final hasSignificantChange = (_percentage - _lastPercentage).abs() > 0.05; + + _lastPercentage = _percentage; + _lastUpdate = now; + return LinearPercentIndicator( - animation: true, + animation: !isRapidUpdate && hasSignificantChange, animateFromLastPercent: true, lineHeight: 10.0, barRadius: const Radius.circular(5), backgroundColor: const Color(0xffFAFAFA), - animationDuration: 1000, + animationDuration: 100, percent: _percentage, progressColor: const Color(0xff3C3C3C), ); diff --git a/lib/models/daos/drive_dao/drive_dao.dart b/lib/models/daos/drive_dao/drive_dao.dart index a12db07f40..543e3ed35a 100644 --- a/lib/models/daos/drive_dao/drive_dao.dart +++ b/lib/models/daos/drive_dao/drive_dao.dart @@ -125,9 +125,23 @@ class DriveDao extends DatabaseAccessor with _$DriveDaoMixin { List revisions, ) async { try { - await db.batch((b) async { - b.insertAllOnConflictUpdate(db.driveRevisions, revisions); - }); + // Process in chunks to avoid memory issues with large datasets + const chunkSize = 100; + for (var i = 0; i < revisions.length; i += chunkSize) { + final end = (i + chunkSize < revisions.length) + ? i + chunkSize + : revisions.length; + final chunk = revisions.sublist(i, end); + + await db.batch((b) async { + b.insertAllOnConflictUpdate(db.driveRevisions, chunk); + }); + + // Yield control back to the event loop to prevent UI blocking + if (i + chunkSize < revisions.length) { + await Future.delayed(Duration.zero); + } + } } catch (e) { throw _handleError('Error inserting new drive revisions', e); } @@ -137,9 +151,23 @@ class DriveDao extends DatabaseAccessor with _$DriveDaoMixin { List revisions, ) async { try { - await db.batch((b) async { - b.insertAllOnConflictUpdate(db.fileRevisions, revisions); - }); + // Process in chunks to avoid memory issues with large datasets + const chunkSize = 100; + for (var i = 0; i < revisions.length; i += chunkSize) { + final end = (i + chunkSize < revisions.length) + ? i + chunkSize + : revisions.length; + final chunk = revisions.sublist(i, end); + + await db.batch((b) async { + b.insertAllOnConflictUpdate(db.fileRevisions, chunk); + }); + + // Yield control back to the event loop to prevent UI blocking + if (i + chunkSize < revisions.length) { + await Future.delayed(Duration.zero); + } + } } catch (e) { throw _handleError('Error inserting new file revisions', e); } @@ -149,9 +177,23 @@ class DriveDao extends DatabaseAccessor with _$DriveDaoMixin { List revisions, ) async { try { - await db.batch((b) async { - b.insertAllOnConflictUpdate(db.folderRevisions, revisions); - }); + // Process in chunks to avoid memory issues with large datasets + const chunkSize = 100; + for (var i = 0; i < revisions.length; i += chunkSize) { + final end = (i + chunkSize < revisions.length) + ? i + chunkSize + : revisions.length; + final chunk = revisions.sublist(i, end); + + await db.batch((b) async { + b.insertAllOnConflictUpdate(db.folderRevisions, chunk); + }); + + // Yield control back to the event loop to prevent UI blocking + if (i + chunkSize < revisions.length) { + await Future.delayed(Duration.zero); + } + } } catch (e) { throw _handleError('Error inserting new folder revisions', e); } @@ -161,9 +203,23 @@ class DriveDao extends DatabaseAccessor with _$DriveDaoMixin { List transactions, ) async { try { - await db.batch((b) async { - b.insertAllOnConflictUpdate(db.networkTransactions, transactions); - }); + // Process in chunks to avoid memory issues with large datasets + const chunkSize = 100; + for (var i = 0; i < transactions.length; i += chunkSize) { + final end = (i + chunkSize < transactions.length) + ? i + chunkSize + : transactions.length; + final chunk = transactions.sublist(i, end); + + await db.batch((b) async { + b.insertAllOnConflictUpdate(db.networkTransactions, chunk); + }); + + // Yield control back to the event loop to prevent UI blocking + if (i + chunkSize < transactions.length) { + await Future.delayed(Duration.zero); + } + } } catch (e) { throw _handleError('Error inserting new network transactions', e); } @@ -453,25 +509,11 @@ class DriveDao extends DatabaseAccessor with _$DriveDaoMixin { final subfolderQuery = foldersInFolder( driveId: driveId, parentFolderId: folderId, - order: (folderEntries) { - return enumToFolderOrderByClause( - folderEntries, - orderBy, - orderingMode, - ); - }, ); final filesQuery = filesInFolderWithLicenseAndRevisionTransactions( driveId: driveId, parentFolderId: folderId, - order: (fileEntries, _, __, ___) { - return enumToFileOrderByClause( - fileEntries, - orderBy, - orderingMode, - ); - }, ); return Rx.combineLatest3( diff --git a/lib/models/queries/drive_queries.drift b/lib/models/queries/drive_queries.drift index a5537fa589..cf9372a3cb 100644 --- a/lib/models/queries/drive_queries.drift +++ b/lib/models/queries/drive_queries.drift @@ -7,9 +7,8 @@ import '../tables/file_revisions.drift'; import '../tables/network_transactions.drift'; import '../tables/licenses.drift'; -allDrives ($order = ''): - SELECT * FROM drives - ORDER BY $order; +allDrives: + SELECT * FROM drives; driveById: SELECT * FROM drives WHERE id = :driveId; oldestDriveRevisionByDriveId: @@ -32,10 +31,9 @@ folderById: SELECT * FROM folder_entries WHERE id = :folderId; -foldersInFolder ($order = ''): +foldersInFolder: SELECT * FROM folder_entries - WHERE driveId = :driveId AND parentFolderId = :parentFolderId - ORDER BY $order; + WHERE driveId = :driveId AND parentFolderId = :parentFolderId; ghostFolders: SELECT * FROM folder_entries @@ -85,10 +83,9 @@ licenseByTxId: SELECT * FROM licenses WHERE licenseTxId = :tx; -filesInFolder ($order = ''): +filesInFolder: SELECT * FROM file_entries - WHERE driveId = :driveId AND parentFolderId = :parentFolderId - ORDER BY $order; + WHERE driveId = :driveId AND parentFolderId = :parentFolderId; filesInFolderWithName: SELECT * FROM file_entries WHERE driveId = :driveId AND parentFolderId = :parentFolderId AND name = :name; @@ -96,7 +93,7 @@ manifestInFolder: SELECT * FROM file_entries WHERE parentFolderId = :parentFolderId AND dataContentType = 'application/x.arweave-manifest+json'; -filesInFolderWithLicenseAndRevisionTransactions ($order = '') AS FileWithLicenseAndLatestRevisionTransactions: +filesInFolderWithLicenseAndRevisionTransactions AS FileWithLicenseAndLatestRevisionTransactions: SELECT file_entries.*, license.**, metadataTx.**, dataTx.** FROM file_entries LEFT JOIN licenses AS license ON license.licenseTxId = ( SELECT licenseTxId FROM file_revisions AS rev @@ -113,9 +110,8 @@ filesInFolderWithLicenseAndRevisionTransactions ($order = '') AS FileWithLicense WHERE driveId = :driveId AND fileId = file_entries.id ORDER BY rev.dateCreated DESC LIMIT 1) - WHERE file_entries.driveId = :driveId AND file_entries.parentFolderId = :parentFolderId - ORDER BY $order; -filesInDriveWithRevisionTransactions ($order = '') AS FileWithLatestRevisionTransactions: + WHERE file_entries.driveId = :driveId AND file_entries.parentFolderId = :parentFolderId; +filesInDriveWithRevisionTransactions AS FileWithLatestRevisionTransactions: SELECT file_entries.*, metadataTx.**, dataTx.** FROM file_entries JOIN network_transactions AS metadataTx ON metadataTx.id = ( SELECT metadataTxId FROM file_revisions AS rev @@ -127,8 +123,7 @@ JOIN network_transactions AS dataTx ON dataTx.id = ( WHERE driveId = :driveId AND fileId = file_entries.id ORDER BY rev.dateCreated DESC LIMIT 1) -WHERE driveId = :driveId -ORDER BY $order; +WHERE driveId = :driveId; oldestFileRevisionsByFileId: SELECT * FROM file_revisions diff --git a/lib/pages/drive_detail/components/drive_detail_data_list.dart b/lib/pages/drive_detail/components/drive_detail_data_list.dart index d7dc392cbc..dc83d37412 100644 --- a/lib/pages/drive_detail/components/drive_detail_data_list.dart +++ b/lib/pages/drive_detail/components/drive_detail_data_list.dart @@ -119,8 +119,8 @@ Widget _buildDataListContent( key: ValueKey( '${folder.id}-${forceRebuildKey.toString()}${columns.length}-${hideState.toString()}'), initialPage: selectedPage, - lockMultiSelect: context.watch().state is SyncInProgress || - !context.watch().isMultiSelectEnabled, + lockMultiSelect: context.read().state is SyncInProgress || + !context.read().isMultiSelectEnabled, rowsPerPageText: appLocalizationsOf(context).rowsPerPage, maxItemsPerPage: 100, pageItemsDivisorFactor: 25, @@ -305,3 +305,4 @@ class ColumnIndexes { static const int dateCreated = 3; static const int licenseType = 4; } + diff --git a/lib/services/arweave/arweave_service.dart b/lib/services/arweave/arweave_service.dart index fea15fb378..0bffae4e04 100644 --- a/lib/services/arweave/arweave_service.dart +++ b/lib/services/arweave/arweave_service.dart @@ -173,7 +173,7 @@ class ArweaveService { Stream getAllSnapshotsOfDrive( String driveId, int? lastBlockHeight, { - required String ownerAddress, + String? ownerAddress, }) async* { String cursor = ''; @@ -186,7 +186,7 @@ class ArweaveService { driveId: driveId, lastBlockHeight: lastBlockHeight, after: cursor, - ownerAddress: ownerAddress, + ownerAddress: ownerAddress != null ? [ownerAddress] : null, ), ), ); @@ -290,11 +290,16 @@ class ArweaveService { /// rate-limited (TODO: check the latter), many requests will be retrying. /// We shall find another way to fail faster. - // MAYBE FIX: set a narrow concurrency limit - - final List entityDatas = await Future.wait( - entityTxs.map( - (model) async { + // FIXED: Limit concurrency to prevent browser overwhelm + // Reduced concurrency limit for better UI responsiveness + const concurrencyLimit = 5; + final List entityDatas = []; + + for (var i = 0; i < entityTxs.length; i += concurrencyLimit) { + final batch = entityTxs.skip(i).take(concurrencyLimit); + final batchResults = await Future.wait( + batch.map( + (model) async { final entity = model.transactionCommonMixin; final tags = HashMap.fromIterable( @@ -323,8 +328,16 @@ class ArweaveService { isPrivate: driveKey != null, ); }, - ), - ); + )); + + entityDatas.addAll(batchResults); + + // Yield control between batches to prevent UI blocking + // Use a longer delay for better UI responsiveness + if (i + concurrencyLimit < entityTxs.length) { + await Future.delayed(const Duration(milliseconds: 10)); + } + } final metadataCache = await MetadataCache.fromCacheStore( await newSharedPreferencesCacheStore(), diff --git a/lib/services/arweave/graphql/graphql.dart b/lib/services/arweave/graphql/graphql.dart index 69cb1f2e38..41703870b8 100644 --- a/lib/services/arweave/graphql/graphql.dart +++ b/lib/services/arweave/graphql/graphql.dart @@ -10,9 +10,10 @@ extension TransactionMixinExtensions on TransactionCommonMixin { tags.firstWhereOrNull((t) => t.name == tagName)?.value; DateTime getCommitTime() { + final unixTimeStr = getTag(EntityTag.unixTime)!; final milliseconds = getTag(EntityTag.arFs) != '0.10' - ? int.parse(getTag(EntityTag.unixTime)!) * 1000 - : int.parse(getTag(EntityTag.unixTime)!); + ? (double.parse(unixTimeStr) * 1000).round() + : double.parse(unixTimeStr).round(); return DateTime.fromMillisecondsSinceEpoch(milliseconds); } diff --git a/lib/services/arweave/graphql/queries/SnapshotEntityHistory.graphql b/lib/services/arweave/graphql/queries/SnapshotEntityHistory.graphql index 0bf769062b..9767a7b0cb 100644 --- a/lib/services/arweave/graphql/queries/SnapshotEntityHistory.graphql +++ b/lib/services/arweave/graphql/queries/SnapshotEntityHistory.graphql @@ -2,10 +2,10 @@ query SnapshotEntityHistory( $driveId: String! $after: String $lastBlockHeight: Int - $ownerAddress: String! + $ownerAddress: [String!] ) { transactions( - owners: [$ownerAddress] + owners: $ownerAddress first: 100 sort: HEIGHT_DESC tags: [ diff --git a/lib/sync/domain/cubit/sync_cubit.dart b/lib/sync/domain/cubit/sync_cubit.dart index cfca8bdece..161a3227a1 100644 --- a/lib/sync/domain/cubit/sync_cubit.dart +++ b/lib/sync/domain/cubit/sync_cubit.dart @@ -171,6 +171,64 @@ class SyncCubit extends Cubit { var ghostFolders = {}; + Future syncSingleDrive(String driveId) async { + logger.i('Starting sync for single drive: $driveId'); + + if (state is SyncInProgress) { + logger.d('Sync state is SyncInProgress, aborting sync...'); + return; + } + + try { + // Don't emit SyncInProgress for single drive sync to avoid modal conflicts + // The drive attach process will handle user feedback + + // Notify prompt to snapshot bloc that sync is starting + _promptToSnapshotBloc.add(const SyncRunning(isRunning: true)); + + await for (var progress in _syncRepository.syncDriveById( + driveId: driveId, + ownerAddress: '', // Will be fetched from the drive in the repository + txFechedCallback: (driveId, txCount) { + _promptToSnapshotBloc.add( + CountSyncedTxs( + driveId: driveId, + txsSyncedWithGqlCount: txCount, + wasDeepSync: false, + ), + ); + }, + )) { + // Progress is a double from 0.0 to 1.0 + logger.d('Sync progress for drive $driveId: ${(progress * 100).toStringAsFixed(2)}%'); + + // Don't emit to syncProgressController to avoid modal conflicts + // Single drive sync runs in background without blocking UI + } + + logger.i('Single drive sync completed for: $driveId'); + + // Refresh balance if user is logged in + final profile = _profileCubit.state; + if (profile is ProfileLoggedIn) { + _profileCubit.refreshBalance(); + } + + // Notify prompt to snapshot bloc that sync is finished + _promptToSnapshotBloc.add(const SyncRunning(isRunning: false)); + + _lastSync = DateTime.now(); + // Don't emit SyncIdle since we never emitted SyncInProgress + } catch (err, stackTrace) { + logger.e('Error syncing single drive', err, stackTrace); + + // Notify prompt to snapshot bloc that sync is finished (even on error) + _promptToSnapshotBloc.add(const SyncRunning(isRunning: false)); + + addError(err); + } + } + Future startSync({bool deepSync = false}) async { logger.i('Starting Sync'); diff --git a/lib/sync/domain/repositories/sync_repository.dart b/lib/sync/domain/repositories/sync_repository.dart index d21389eb47..d1daa6e313 100644 --- a/lib/sync/domain/repositories/sync_repository.dart +++ b/lib/sync/domain/repositories/sync_repository.dart @@ -194,7 +194,7 @@ class _SyncRepository implements SyncRepository { : _calculateSyncLastBlockHeight(drive.lastBlockHeight!), currentBlockHeight: currentBlockHeight, transactionParseBatchSize: - 200 ~/ (syncProgress.drivesCount - syncProgress.drivesSynced), + 50 ~/ max(1, (syncProgress.drivesCount - syncProgress.drivesSynced)), ownerAddress: drive.ownerAddress, txFechedCallback: txFechedCallback, ); @@ -300,14 +300,28 @@ class _SyncRepository implements SyncRepository { required String driveId, required String ownerAddress, Function(String driveId, int txCount)? txFechedCallback, - }) { + }) async* { _lastSync = DateTime.now(); - return _syncDrive( + + // If ownerAddress is empty, fetch it from the drive + String actualOwnerAddress = ownerAddress; + if (ownerAddress.isEmpty) { + final drive = await _driveDao.driveById(driveId: driveId).getSingleOrNull(); + if (drive == null) { + throw Exception('Drive not found: $driveId'); + } + actualOwnerAddress = drive.ownerAddress; + } + + // Get current block height + final currentBlockHeight = await _arweave.getCurrentBlockHeight(); + + yield* _syncDrive( driveId, - ownerAddress: ownerAddress, + ownerAddress: actualOwnerAddress, lastBlockHeight: 0, - currentBlockHeight: 0, - transactionParseBatchSize: 200, + currentBlockHeight: currentBlockHeight, + transactionParseBatchSize: 50, txFechedCallback: txFechedCallback, ); } @@ -571,6 +585,9 @@ class _SyncRepository implements SyncRepository { final fetchPhaseStartDT = DateTime.now(); logger.d('Fetching all transactions for drive ${drive.id}'); + + // Yield initial progress for fetching phase + yield 0.1; final transactions = []; @@ -579,21 +596,46 @@ class _SyncRepository implements SyncRepository { if (_configService.config.enableSyncFromSnapshot) { logger.i('Syncing from snapshot: ${drive.id}'); + // First try to get snapshots without owner restriction (for attached drives) final snapshotsStream = _arweave.getAllSnapshotsOfDrive( driveId, lastBlockHeight, - ownerAddress: ownerAddress, + ownerAddress: null, // Allow snapshots from any owner ); snapshotItems = await SnapshotItem.instantiateAll( snapshotsStream, arweave: _arweave, ).toList(); + + // Yield progress after snapshot discovery + yield 0.2; + + // If no snapshots found and we have an owner address, try owner-specific search + if (snapshotItems.isEmpty && ownerAddress.isNotEmpty) { + logger.d('No general snapshots found, trying owner-specific for ${drive.id}'); + final ownerSnapshotsStream = _arweave.getAllSnapshotsOfDrive( + driveId, + lastBlockHeight, + ownerAddress: ownerAddress, + ); + + snapshotItems = await SnapshotItem.instantiateAll( + ownerSnapshotsStream, + arweave: _arweave, + ).toList(); + } List snapshotsVerified = await _snapshotValidationService.validateSnapshotItems(snapshotItems); snapshotItems = snapshotsVerified; + + if (snapshotItems.isNotEmpty) { + logger.i('Found ${snapshotItems.length} validated snapshots for drive ${drive.id}'); + } else { + logger.d('No validated snapshots found for drive ${drive.id}'); + } } final SnapshotDriveHistory snapshotDriveHistory = SnapshotDriveHistory( @@ -642,10 +684,12 @@ class _SyncRepository implements SyncRepository { /// This percentage is based on block heights. var fetchPhasePercentage = 0.0; + var transactionCount = 0; /// First phase of the sync /// Here we get all transactions from its drive. await for (DriveEntityHistoryTransactionModel t in transactionsStream) { + transactionCount++; double calculatePercentageBasedOnBlockHeights() { final block = t.transactionCommonMixin.block; @@ -689,7 +733,15 @@ class _SyncRepository implements SyncRepository { } final percentage = calculatePercentageBasedOnBlockHeights() * fetchPhaseWeight; - yield percentage; + + // Yield progress more frequently during fetch phase for better UI responsiveness + // Also yield control to prevent UI blocking + if (transactionCount % 25 == 0 || percentage > fetchPhasePercentage + 0.01) { + yield percentage; + fetchPhasePercentage = percentage; + // Allow UI to update + await Future.delayed(const Duration(milliseconds: 5)); + } } } @@ -703,6 +755,9 @@ class _SyncRepository implements SyncRepository { logger.d( 'Duration of fetch phase for ${drive.name}: $fetchPhaseTotalTime ms. Progress by block height: $fetchPhasePercentage%. Starting parse phase'); + // Yield progress at the end of fetch phase / start of parse phase + yield fetchPhaseWeight; + try { yield* _parseDriveTransactionsIntoDatabaseEntities( transactions: transactions, @@ -714,7 +769,7 @@ class _SyncRepository implements SyncRepository { snapshotDriveHistory: snapshotDriveHistory, ownerAddress: ownerAddress, ).map( - (parseProgress) => parseProgress * 0.9, + (parseProgress) => fetchPhaseWeight + (parseProgress * parsePhaseWeight), ); } catch (e) { logger.e('[Sync Drive] Error while parsing transactions', e); @@ -848,6 +903,10 @@ class _SyncRepository implements SyncRepository { list: transactions, batchSize: batchSize, endOfBatchCallback: (items) async* { + // Yield progress at the start of processing this batch + final currentProgress = driveEntityParseProgress(); + yield currentProgress; + final entityHistory = await _arweave.createDriveEntityHistoryFromTransactions( items, @@ -864,7 +923,12 @@ class _SyncRepository implements SyncRepository { numberOfDriveEntitiesParsed += items.length - newEntities.length; - yield driveEntityParseProgress(); + // Yield progress after processing this batch + final updatedProgress = driveEntityParseProgress(); + yield updatedProgress; + + // Yield control to prevent UI blocking during heavy processing + await Future.delayed(const Duration(milliseconds: 10)); // Handle the last page of newEntities, i.e; There's nothing more to sync if (newEntities.length < batchSize) { diff --git a/pubspec.yaml b/pubspec.yaml index 840160e95d..f1c1cc759d 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -164,6 +164,10 @@ dependency_overrides: url: https://github.com/ardriveapp/fetch_api.git ref: master http: ^1.1.0 + intl: ^0.20.2 + test: ^1.24.0 + vm_service: 15.0.0 + analyzer: ^5.0.0 arweave: git: url: https://github.com/ardriveapp/arweave-dart.git diff --git a/test/blocs/drive_attach_cubit_test.dart b/test/blocs/drive_attach_cubit_test.dart index cb530e8d9b..31060783f1 100644 --- a/test/blocs/drive_attach_cubit_test.dart +++ b/test/blocs/drive_attach_cubit_test.dart @@ -20,7 +20,6 @@ import '../test_utils/utils.dart'; void main() { group('DriveAttachCubit', () { - late Database db; late ArweaveService arweave; late DriveDao driveDao; late SyncCubit syncBloc; @@ -42,14 +41,16 @@ void main() { const ownerAddress = 'owner-address'; const validRootFolderId = 'valid-root-folder-id'; const notFoundDriveId = 'not-found-drive-id'; - db = getTestDb(); setUp(() { registerFallbackValue(SyncStateFake()); registerFallbackValue(ProfileStateFake()); registerFallbackValue(DrivesStateFake()); + registerFallbackValue(FakeDriveEntity()); + registerFallbackValue(FakeDriveKey()); + registerFallbackValue(FakeSecretKey()); - driveDao = db.driveDao; + driveDao = MockDriveDao(); arweave = MockArweaveService(); syncBloc = MockSyncBloc(); @@ -95,6 +96,19 @@ void main() { when(() => syncBloc.waitCurrentSync()) .thenAnswer((_) => Future.value(null)); + + when(() => syncBloc.syncSingleDrive(any())) + .thenAnswer((_) => Future.value(null)); + + when(() => arweave.getAllSnapshotsOfDrive(any(), any(), ownerAddress: any(named: 'ownerAddress'))) + .thenAnswer((_) => const Stream.empty()); + + when(() => driveDao.writeDriveEntity( + name: any(named: 'name'), + entity: any(named: 'entity'), + driveKey: any(named: 'driveKey'), + profileKey: any(named: 'profileKey'), + )).thenAnswer((_) => Future.value()); driveAttachCubit = DriveAttachCubit( arweave: arweave, @@ -117,10 +131,12 @@ void main() { }, expect: () => [ DriveAttachInProgress(), + DriveAttachSyncing(hasSnapshots: false), DriveAttachSuccess(), ], + wait: const Duration(seconds: 3), verify: (_) { - verify(() => syncBloc.startSync()).called(1); + verify(() => syncBloc.syncSingleDrive(validDriveId)).called(1); verify(() => drivesBloc.selectDrive(validDriveId)).called(1); }, ); @@ -197,11 +213,12 @@ void main() { expect: () => [ DriveAttachPrivate(), DriveAttachInProgress(), + DriveAttachSyncing(hasSnapshots: false), DriveAttachSuccess(), ], - wait: const Duration(milliseconds: 1200), + wait: const Duration(seconds: 3), verify: (_) async { - verify(() => syncBloc.startSync()).called(1); + verify(() => syncBloc.syncSingleDrive(validPrivateDriveId)).called(1); verify(() => drivesBloc.selectDrive(validPrivateDriveId)).called(1); }, ); diff --git a/test/test_utils/fakes.dart b/test/test_utils/fakes.dart index 5a755b0ede..0e4ecc9727 100644 --- a/test/test_utils/fakes.dart +++ b/test/test_utils/fakes.dart @@ -1,5 +1,8 @@ import 'package:ardrive/blocs/blocs.dart'; +import 'package:ardrive/core/crypto/crypto.dart'; +import 'package:ardrive/entities/entities.dart'; import 'package:ardrive/sync/domain/cubit/sync_cubit.dart'; +import 'package:cryptography/cryptography.dart'; import 'package:mocktail/mocktail.dart'; class SyncStateFake extends Fake implements SyncState {} @@ -7,3 +10,9 @@ class SyncStateFake extends Fake implements SyncState {} class ProfileStateFake extends Fake implements ProfileState {} class DrivesStateFake extends Fake implements DrivesState {} + +class FakeDriveEntity extends Fake implements DriveEntity {} + +class FakeDriveKey extends Fake implements DriveKey {} + +class FakeSecretKey extends Fake implements SecretKey {}