feat: initial hive -> SQLite (#2502)

* feat: initial hive -> SQLite

Includes migration script and initial work done to
remove dependency on hive generators.

Also contains some refactoring to make sync code
async.

Features new dev screen that lists all SQLite tables

Aspects that need testing:
- backups (from this version, and from previous versions as well)
- wallet loading / wallet list
- almost everything else
- renaming
- deletion
- sorting in wallet list
- wallet groups
- wallet seed/keys
- wallet switching
- wallet creation / restore - all methods
- wallet address page
- hardware wallet connection (and prompts from mall screens)
- contacts
- advanced settings in wallet creation
- derivation paths
- some minor wownero fixes
- base, hardware wallets

* fix: build errors

* fix: Bad state: No element in backup restore

* address comments from review

* fix: HardwareWalletType

Author: MrCyjaneK

.github/workflows/pr_test_build_linux.yml

@@ -308,8 +308,8 @@ jobs:
           rm -rf ~/.local/share/com.example.cake_wallet/ ~/Documents/cake_wallet/ ~/cake_wallet
           exec timeout --signal=SIGKILL 900 flutter drive --driver=test_driver/integration_test.dart --target=integration_test/test_suites/restore_wallet_through_seeds_flow_test.dart
       - name: Test [cw_monero]
-        timeout-minutes: 2
-        run: cd cw_monero && flutter test
+        timeout-minutes: 15
+        run: cd cw_monero && flutter test --verbose
       - name: Stop screen recording, encrypt and upload
         if: always()
         run: |

Dockerfile

@@ -59,6 +59,8 @@ RUN set -o xtrace \
     ffmpeg network-manager x11-utils xvfb psmisc \
     # extra linux dependencies so flutter doesn't complain
     mesa-utils \
+    # database
+    libsqlite3-0 libsqlite3-dev \
     # aarch64-linux-gnu dependencies
     g++-aarch64-linux-gnu gcc-aarch64-linux-gnu \
     # x86_64-linux-gnu dependencies

cw_base/lib/base_wallet.dart

@@ -22,6 +22,7 @@ class BaseWallet extends EVMChainWallet {
   BaseWallet({
     required super.walletInfo,
     required super.password,
+    required super.derivationInfo,
     super.mnemonic,
     super.initialBalance,
     super.privateKey,
@@ -150,6 +151,7 @@ class BaseWallet extends EVMChainWallet {
 
     return BaseWallet(
       walletInfo: walletInfo,
+      derivationInfo: await walletInfo.getDerivationInfo(),
       password: password,
       mnemonic: keysData.mnemonic,
       privateKey: keysData.privateKey,

cw_base/lib/base_wallet_service.dart

@@ -11,7 +11,6 @@ import 'package:cw_base/base_mnemonics_exception.dart';
 
 class BaseWalletService extends EVMChainWalletService<BaseWallet> {
   BaseWalletService(
-    super.walletInfoSource,
     super.isDirect, {
     required this.client,
   });
@@ -29,6 +28,7 @@ class BaseWalletService extends EVMChainWalletService<BaseWallet> {
 
     final wallet = BaseWallet(
       walletInfo: credentials.walletInfo!,
+      derivationInfo: await credentials.walletInfo!.getDerivationInfo(),
       mnemonic: mnemonic,
       password: credentials.password!,
       passphrase: credentials.passphrase,
@@ -44,8 +44,10 @@ class BaseWalletService extends EVMChainWalletService<BaseWallet> {
 
   @override
   Future<BaseWallet> openWallet(String name, String password) async {
-    final walletInfo =
-        walletInfoSource.values.firstWhere((info) => info.id == WalletBase.idFor(name, getType()));
+    final walletInfo = await WalletInfo.get(name, getType());
+    if (walletInfo == null) {
+      throw Exception('Wallet not found');
+    }
 
     try {
       final wallet = await BaseWallet.open(
@@ -84,6 +86,7 @@ class BaseWalletService extends EVMChainWalletService<BaseWallet> {
       password: credentials.password!,
       privateKey: credentials.privateKey,
       walletInfo: credentials.walletInfo!,
+      derivationInfo: await credentials.walletInfo!.getDerivationInfo(),
       client: client,
       encryptionFileUtils: encryptionFileUtilsFor(isDirect),
     );
@@ -97,14 +100,17 @@ class BaseWalletService extends EVMChainWalletService<BaseWallet> {
   @override
   Future<BaseWallet> restoreFromHardwareWallet(
       EVMChainRestoreWalletFromHardware credentials) async {
-    credentials.walletInfo!.derivationInfo = DerivationInfo(
-        derivationType: DerivationType.bip39,
-        derivationPath: "m/44'/60'/${credentials.hwAccountData.accountIndex}'/0/0");
+    final derivationInfo = await credentials.walletInfo!.getDerivationInfo();
+    derivationInfo.derivationType = DerivationType.bip39;
+    derivationInfo.derivationPath = "m/44'/60'/${credentials.hwAccountData.accountIndex}'/0/0";
+    await derivationInfo.save();
     credentials.walletInfo!.hardwareWalletType = credentials.hardwareWalletType;
     credentials.walletInfo!.address = credentials.hwAccountData.address;
+    await credentials.walletInfo!.save();
 
     final wallet = BaseWallet(
       walletInfo: credentials.walletInfo!,
+      derivationInfo: derivationInfo,
       password: credentials.password!,
       client: client,
       encryptionFileUtils: encryptionFileUtilsFor(isDirect),
@@ -128,6 +134,7 @@ class BaseWalletService extends EVMChainWalletService<BaseWallet> {
       password: credentials.password!,
       mnemonic: credentials.mnemonic,
       walletInfo: credentials.walletInfo!,
+      derivationInfo: await credentials.walletInfo!.getDerivationInfo(),
       passphrase: credentials.passphrase,
       client: client,
       encryptionFileUtils: encryptionFileUtilsFor(isDirect),
@@ -142,8 +149,10 @@ class BaseWalletService extends EVMChainWalletService<BaseWallet> {
 
   @override
   Future<void> rename(String currentName, String password, String newName) async {
-    final currentWalletInfo = walletInfoSource.values
-        .firstWhere((info) => info.id == WalletBase.idFor(currentName, getType()));
+    final currentWalletInfo = await WalletInfo.get(currentName, getType());
+    if (currentWalletInfo == null) {
+      throw Exception('Wallet not found');
+    }
     final currentWallet = await BaseWallet.open(
       password: password,
       name: currentName,
@@ -158,6 +167,6 @@ class BaseWalletService extends EVMChainWalletService<BaseWallet> {
     newWalletInfo.id = WalletBase.idFor(newName, getType());
     newWalletInfo.name = newName;
 
-    await walletInfoSource.put(currentWalletInfo.key, newWalletInfo);
+    newWalletInfo.save();
   }
 }

cw_bitcoin/lib/bitcoin_wallet.dart

@@ -47,6 +47,7 @@ abstract class BitcoinWalletBase extends ElectrumWallet with Store {
   BitcoinWalletBase({
     required String password,
     required WalletInfo walletInfo,
+    required DerivationInfo derivationInfo,
     required Box<UnspentCoinsInfo> unspentCoinsInfo,
     required Box<PayjoinSession> payjoinBox,
     required EncryptionFileUtils encryptionFileUtils,
@@ -69,6 +70,7 @@ abstract class BitcoinWalletBase extends ElectrumWallet with Store {
           xpub: xpub,
           password: password,
           walletInfo: walletInfo,
+          derivationInfo: derivationInfo,
           unspentCoinsInfo: unspentCoinsInfo,
           network: networkParam == null
               ? BitcoinNetwork.mainnet
@@ -133,7 +135,9 @@ abstract class BitcoinWalletBase extends ElectrumWallet with Store {
   }) async {
     late Uint8List seedBytes;
 
-    switch (walletInfo.derivationInfo?.derivationType) {
+    final derivationInfo = await walletInfo.getDerivationInfo();
+
+    switch (derivationInfo.derivationType) {
       case DerivationType.bip39:
         seedBytes = await bip39.mnemonicToSeed(
           mnemonic,
@@ -152,6 +156,7 @@ abstract class BitcoinWalletBase extends ElectrumWallet with Store {
       passphrase: passphrase ?? "",
       password: password,
       walletInfo: walletInfo,
+      derivationInfo: derivationInfo,
       unspentCoinsInfo: unspentCoinsInfo,
       initialAddresses: initialAddresses,
       initialSilentAddresses: initialSilentAddresses,
@@ -212,20 +217,21 @@ abstract class BitcoinWalletBase extends ElectrumWallet with Store {
       );
     }
 
-    walletInfo.derivationInfo ??= DerivationInfo();
+    final derivationInfo = await walletInfo.getDerivationInfo();
 
     // set the default if not present:
-    walletInfo.derivationInfo!.derivationPath ??=
+    derivationInfo.derivationPath ??=
         snp?.derivationPath ?? electrum_path;
-    walletInfo.derivationInfo!.derivationType ??=
+    derivationInfo.derivationType ??=
         snp?.derivationType ?? DerivationType.electrum;
+    await derivationInfo.save();
 
     Uint8List? seedBytes = null;
     final mnemonic = keysData.mnemonic;
     final passphrase = keysData.passphrase;
 
     if (mnemonic != null) {
-      switch (walletInfo.derivationInfo!.derivationType) {
+      switch (derivationInfo.derivationType) {
         case DerivationType.electrum:
           seedBytes =
               await mnemonicToSeedBytes(mnemonic, passphrase: passphrase ?? "");
@@ -246,6 +252,7 @@ abstract class BitcoinWalletBase extends ElectrumWallet with Store {
         password: password,
         passphrase: passphrase,
         walletInfo: walletInfo,
+        derivationInfo: derivationInfo,
         unspentCoinsInfo: unspentCoinsInfo,
         initialAddresses: snp?.addresses,
         initialSilentAddresses: snp?.silentAddresses,
@@ -484,7 +491,8 @@ abstract class BitcoinWalletBase extends ElectrumWallet with Store {
           : null;
       final index = addressEntry?.index ?? 0;
       final isChange = addressEntry?.isHidden == true ? 1 : 0;
-      final accountPath = walletInfo.derivationInfo?.derivationPath;
+      final derivationInfo = await walletInfo.getDerivationInfo();
+      final accountPath = derivationInfo.derivationPath;
       final derivationPath =
           accountPath != null ? "$accountPath/$isChange/$index" : null;
 

cw_bitcoin/lib/bitcoin_wallet_creation_credentials.dart

@@ -16,6 +16,10 @@ class BitcoinNewWalletCredentials extends WalletCredentials {
           walletInfo: walletInfo,
           password: password,
           passphrase: passphrase,
+          derivationInfo: DerivationInfo(
+            derivationType: derivationType,
+            derivationPath: derivationPath,
+          ),
         );
 
   final String? mnemonic;

cw_bitcoin/lib/bitcoin_wallet_service.dart

@@ -23,10 +23,9 @@ class BitcoinWalletService extends WalletService<
     BitcoinRestoreWalletFromSeedCredentials,
     BitcoinWalletFromKeysCredentials,
     BitcoinRestoreWalletFromHardware> {
-  BitcoinWalletService(this.walletInfoSource, this.unspentCoinsInfoSource,
+  BitcoinWalletService(this.unspentCoinsInfoSource,
       this.payjoinSessionSource, this.isDirect);
 
-  final Box<WalletInfo> walletInfoSource;
   final Box<UnspentCoinsInfo> unspentCoinsInfoSource;
   final Box<PayjoinSession> payjoinSessionSource;
   final bool isDirect;
@@ -40,7 +39,13 @@ class BitcoinWalletService extends WalletService<
     credentials.walletInfo?.network = network.value;
 
     final String mnemonic;
-    switch ( credentials.walletInfo?.derivationInfo?.derivationType) {
+    final derivationInfo = await credentials.walletInfo!.getDerivationInfo();
+    derivationInfo.derivationType = credentials.derivationInfo?.derivationType ?? derivationInfo.derivationType;
+    derivationInfo.derivationPath = credentials.derivationInfo?.derivationPath ?? derivationInfo.derivationPath;
+    derivationInfo.description = credentials.derivationInfo?.description ?? derivationInfo.description;
+    derivationInfo.scriptType = credentials.derivationInfo?.scriptType ?? derivationInfo.scriptType;
+    await derivationInfo.save();
+    switch (derivationInfo.derivationType) {
       case DerivationType.bip39:
         final strength = credentials.seedPhraseLength == 24 ? 256 : 128;
 
@@ -51,6 +56,7 @@ class BitcoinWalletService extends WalletService<
         mnemonic = await generateElectrumMnemonic();
         break;
     }
+    await derivationInfo.save();
 
     final wallet = await BitcoinWalletBase.create(
       mnemonic: mnemonic,
@@ -75,8 +81,10 @@ class BitcoinWalletService extends WalletService<
 
   @override
   Future<BitcoinWallet> openWallet(String name, String password) async {
-    final walletInfo = walletInfoSource.values
-        .firstWhereOrNull((info) => info.id == WalletBase.idFor(name, getType()))!;
+    final walletInfo = await WalletInfo.get(name, getType());
+    if (walletInfo == null) {
+      throw Exception('Wallet not found');
+    }
     try {
       final wallet = await BitcoinWalletBase.open(
         password: password,
@@ -107,9 +115,11 @@ class BitcoinWalletService extends WalletService<
   @override
   Future<void> remove(String wallet) async {
     File(await pathForWalletDir(name: wallet, type: getType())).delete(recursive: true);
-    final walletInfo = walletInfoSource.values
-        .firstWhereOrNull((info) => info.id == WalletBase.idFor(wallet, getType()))!;
-    await walletInfoSource.delete(walletInfo.key);
+    final walletInfo = await WalletInfo.get(wallet, getType());
+    if (walletInfo == null) {
+      throw Exception('Wallet not found');
+    }
+    await WalletInfo.delete(walletInfo);
 
     final unspentCoinsToDelete = unspentCoinsInfoSource.values.where(
           (unspentCoin) => unspentCoin.walletId == walletInfo.id).toList();
@@ -123,8 +133,10 @@ class BitcoinWalletService extends WalletService<
 
   @override
   Future<void> rename(String currentName, String password, String newName) async {
-    final currentWalletInfo = walletInfoSource.values
-        .firstWhereOrNull((info) => info.id == WalletBase.idFor(currentName, getType()))!;
+    final currentWalletInfo = await WalletInfo.get(currentName, getType());
+    if (currentWalletInfo == null) {
+      throw Exception('Wallet not found');
+    }
     final currentWallet = await BitcoinWalletBase.open(
       password: password,
       name: currentName,
@@ -141,23 +153,26 @@ class BitcoinWalletService extends WalletService<
     newWalletInfo.id = WalletBase.idFor(newName, getType());
     newWalletInfo.name = newName;
 
-    await walletInfoSource.put(currentWalletInfo.key, newWalletInfo);
+    await newWalletInfo.save();
   }
 
   @override
   Future<BitcoinWallet> restoreFromHardwareWallet(BitcoinRestoreWalletFromHardware credentials,
       {bool? isTestnet}) async {
     final network = isTestnet == true ? BitcoinNetwork.testnet : BitcoinNetwork.mainnet;
     credentials.walletInfo?.network = network.value;
-    credentials.walletInfo?.derivationInfo?.derivationPath =
+    final derivationInfo = await credentials.walletInfo!.getDerivationInfo();
+    derivationInfo.derivationPath =
         credentials.hwAccountData.derivationPath;
 
     final xpub = convertZpubToXpub(credentials.hwAccountData.xpub!);
 
+    await credentials.walletInfo!.save();
     final wallet = await BitcoinWallet(
       password: credentials.password!,
       xpub: xpub,
       walletInfo: credentials.walletInfo!,
+      derivationInfo: derivationInfo,
       unspentCoinsInfo: unspentCoinsInfoSource,
       networkParam: network,
       encryptionFileUtils: encryptionFileUtilsFor(isDirect),
@@ -180,6 +195,7 @@ class BitcoinWalletService extends WalletService<
       password: credentials.password!,
       xpub: xpub,
       walletInfo: credentials.walletInfo!,
+      derivationInfo: await credentials.walletInfo!.getDerivationInfo(),
       unspentCoinsInfo: unspentCoinsInfoSource,
       networkParam: network,
       encryptionFileUtils: encryptionFileUtilsFor(isDirect),

cw_bitcoin/lib/electrum_wallet.dart

@@ -63,6 +63,7 @@ abstract class ElectrumWalletBase
   ElectrumWalletBase({
     required String password,
     required WalletInfo walletInfo,
+    required DerivationInfo derivationInfo,
     required Box<UnspentCoinsInfo> unspentCoinsInfo,
     required this.network,
     required this.encryptionFileUtils,
@@ -75,9 +76,8 @@ abstract class ElectrumWalletBase
     ElectrumBalance? initialBalance,
     CryptoCurrency? currency,
     bool? alwaysScan,
-  })
-      : accountHD = getAccountHDWallet(currency, network, seedBytes, xpub,
-            walletInfo.derivationInfo, walletInfo.hardwareWalletType),
+  })  : accountHD =
+            getAccountHDWallet(currency, network, seedBytes, xpub, derivationInfo, walletInfo.hardwareWalletType),
         syncStatus = NotConnectedSyncStatus(),
         _password = password,
         _feeRates = <int>[],
@@ -100,9 +100,10 @@ abstract class ElectrumWalletBase
         this.unspentCoinsInfo = unspentCoinsInfo,
         this.isTestnet = !network.isMainnet,
         this._mnemonic = mnemonic,
-        super(walletInfo) {
+        super(walletInfo, derivationInfo) {
     this.electrumClient = electrumClient ?? electrum.ElectrumClient();
     this.walletInfo = walletInfo;
+    this.derivationInfo = derivationInfo;
     transactionHistory = ElectrumTransactionHistory(
       walletInfo: walletInfo,
       password: password,
@@ -771,8 +772,9 @@ abstract class ElectrumWalletBase
         pubKeyHex = hd.childKey(Bip32KeyIndex(utx.bitcoinAddressRecord.index)).publicKey.toHex();
       }
 
+
       final derivationPath =
-          "${_hardenedDerivationPath(walletInfo.derivationInfo?.derivationPath ?? electrum_path)}"
+          "${_hardenedDerivationPath(derivationInfo.derivationPath ?? electrum_path)}"
           "/${utx.bitcoinAddressRecord.isHidden ? "1" : "0"}"
           "/${utx.bitcoinAddressRecord.index}";
       publicKeys[address.pubKeyHash()] = PublicKeyWithDerivationPath(pubKeyHex, derivationPath);
@@ -981,7 +983,7 @@ abstract class ElectrumWalletBase
 
     // Get Derivation path for change Address since it is needed in Litecoin and BitcoinCash hardware Wallets
     final changeDerivationPath =
-        "${_hardenedDerivationPath(walletInfo.derivationInfo?.derivationPath ?? "m/0'")}"
+        "${_hardenedDerivationPath(derivationInfo.derivationPath ?? "m/0'")}"
         "/${changeAddress.isHidden ? "1" : "0"}"
         "/${changeAddress.index}";
     utxoDetails.publicKeys[address.pubKeyHash()] =
@@ -1447,8 +1449,8 @@ abstract class ElectrumWalletBase
             ? SegwitAddresType.p2wpkh.toString()
             : walletInfo.addressPageType.toString(),
         'balance': balance[currency]?.toJSON(),
-        'derivationTypeIndex': walletInfo.derivationInfo?.derivationType?.index,
-        'derivationPath': walletInfo.derivationInfo?.derivationPath,
+        'derivationTypeIndex': derivationInfo.derivationType?.index,
+        'derivationPath': derivationInfo.derivationPath,
         'silent_addresses': walletAddresses.silentAddresses.map((addr) => addr.toJSON()).toList(),
         'silent_address_index': walletAddresses.currentSilentAddressIndex.toString(),
         'mweb_addresses': walletAddresses.mwebAddresses.map((addr) => addr.toJSON()).toList(),