From 26230513c8fad20865785495142005757279bae6 Mon Sep 17 00:00:00 2001 From: "shai.ungar" Date: Mon, 25 Nov 2024 20:37:46 +0200 Subject: [PATCH 1/2] handle oneZero scrape --- .../backend/configManager/configManager.ts | 18 +++++++ .../main/src/backend/import/bankScraper.ts | 49 +++++++++++++------ .../src/backend/import/importTransactions.ts | 34 ++----------- packages/main/src/manual/setupHelpers.ts | 6 --- packages/renderer/src/components/App.tsx | 2 + 5 files changed, 57 insertions(+), 52 deletions(-) diff --git a/packages/main/src/backend/configManager/configManager.ts b/packages/main/src/backend/configManager/configManager.ts index 4e618d5f..8c1cdd71 100644 --- a/packages/main/src/backend/configManager/configManager.ts +++ b/packages/main/src/backend/configManager/configManager.ts @@ -4,6 +4,7 @@ import { decrypt, encrypt } from '@/backend/configManager/encryption/crypto'; import { existsSync, promises as fs } from 'fs'; import configExample from './defaultConfig'; import logger from '/@/logging/logger'; +import type { CompanyTypes } from 'israeli-bank-scrapers-core'; export async function getConfig(configPath: string = configFilePath): Promise { const configFromFile = await getConfigFromFile(configPath); @@ -35,6 +36,23 @@ export async function updateConfig(configPath: string, configToUpdate: Config): await fs.writeFile(configPath, encryptedConfigStr); } +export async function updateOtpLongTermToken( + key: CompanyTypes, + otpLongTermToken: string, + configPath: string = configFilePath, +): Promise { + const config = await getConfig(configPath); + const account = config.scraping.accountsToScrape.find((acc) => acc.key === key) as { + loginFields: { otpLongTermToken: string }; + }; + if (account) { + account.loginFields.otpLongTermToken = otpLongTermToken; + await updateConfig(configPath, config); + } else { + throw new Error(`Account with key ${key} not found`); + } +} + async function getConfigFromFile(configPath: string) { if (existsSync(configPath)) { return fs.readFile(configPath, { diff --git a/packages/main/src/backend/import/bankScraper.ts b/packages/main/src/backend/import/bankScraper.ts index 7bda550d..7e867fe1 100644 --- a/packages/main/src/backend/import/bankScraper.ts +++ b/packages/main/src/backend/import/bankScraper.ts @@ -1,13 +1,12 @@ import { type AccountToScrapeConfig } from '@/backend/commonTypes'; -import { CompanyTypes, createScraper, SCRAPERS, type ScraperOptions } from 'israeli-bank-scrapers-core'; +import { CompanyTypes, createScraper, type ScraperOptions, SCRAPERS } from 'israeli-bank-scrapers-core'; +import { ipcMain } from 'electron'; +import { EventNames, type EventPublisher } from '@/backend/eventEmitters/EventEmitter'; +import { updateOtpLongTermToken } from '@/backend/configManager/configManager'; export const inputVendors = Object.keys(SCRAPERS) - // Deprecated. see https://github.com/eshaham/israeli-bank-scrapers/blob/07ecd3de0c4aa051f119aa943493f0cda943158c/src/definitions.ts#L26-L29 .filter((key) => key !== CompanyTypes.hapoalimBeOnline) - .map((key) => ({ - key, - ...SCRAPERS[key as CompanyTypes], - })); + .map((key) => ({ key, ...SCRAPERS[key as CompanyTypes] })); interface ScrapeParameters { companyId: AccountToScrapeConfig['key']; @@ -23,20 +22,38 @@ export async function scrape( { companyId, credentials, startDate, timeout, showBrowser = false }: ScrapeParameters, emitProgressEvent: EmitProgressEventFunction, chromePath: string, + eventPublisher: EventPublisher, ) { const options: ScraperOptions = { - companyId, // mandatory; one of 'hapoalim', 'discount', 'otsarHahayal', 'leumiCard', 'isracard', 'amex' - startDate, // the date to fetch transactions from (can't be before the minimum allowed time difference for the scraper) - combineInstallments: false, // if set to true, all installment transactions will be combine into the first one - showBrowser, // shows the browser while scraping, good for debugging (default false) - verbose: false, // include more debug info about in the output + companyId, + startDate, + combineInstallments: false, + showBrowser, + verbose: false, executablePath: chromePath, defaultTimeout: timeout, }; + const scraper = createScraper(options); - scraper.onProgress((eventCompanyId: string, payload: { type: string }) => { - emitProgressEvent(companyId, payload.type); - }); - const scrapeResult = await scraper.scrape(credentials); - return scrapeResult; + scraper.onProgress((eventCompanyId, payload) => emitProgressEvent(companyId, payload.type)); + + if (companyId === CompanyTypes.oneZero) { + const creds = credentials as typeof credentials & { otpLongTermToken: string; phoneNumber: string }; + if (!creds.otpLongTermToken) { + await scraper.triggerTwoFactorAuth(creds.phoneNumber); + await eventPublisher.emit(EventNames.GET_OTP); + const otpCode = await new Promise((resolve) => { + ipcMain.once('get-otp-response', (event, input) => resolve(input)); + }); + const result = await scraper.getLongTermTwoFactorToken(otpCode); + if ('longTermTwoFactorAuthToken' in result) { + await updateOtpLongTermToken(companyId, result.longTermTwoFactorAuthToken); + creds.otpLongTermToken = result.longTermTwoFactorAuthToken; + } else { + throw new Error('Failed to get long-term two-factor auth token'); + } + } + } + + return await scraper.scrape(credentials); } diff --git a/packages/main/src/backend/import/importTransactions.ts b/packages/main/src/backend/import/importTransactions.ts index e0b6c5e5..fd7cf1d9 100644 --- a/packages/main/src/backend/import/importTransactions.ts +++ b/packages/main/src/backend/import/importTransactions.ts @@ -1,25 +1,20 @@ -import { configFilePath, userDataPath } from '@/app-globals'; +import { userDataPath } from '@/app-globals'; import { type AccountToScrapeConfig, type Config, type EnrichedTransaction, - type FinancialAccountDetails, type ScraperScrapingResult, } from '@/backend/commonTypes'; -import { getConfig } from '@/backend/configManager/configManager'; import * as bankScraper from '@/backend/import/bankScraper'; import Bottleneck from 'bottleneck'; import { type Transaction } from 'israeli-bank-scrapers-core/lib/transactions'; -import _ from 'lodash'; import moment from 'moment'; -// import * as categoryCalculation from '@/backend/import/categoryCalculationScript'; import { AccountStatus, - BudgetTrackingEventEmitter, DownalodChromeEvent, EventNames, - ImporterEvent, type EventPublisher, + ImporterEvent, } from '../eventEmitters/EventEmitter'; import { calculateTransactionHash } from '../transactions/transactions'; import getChrome from './downloadChromium'; @@ -97,26 +92,6 @@ function emitChromeDownload(eventPublisher: EventPublisher, percent: number) { eventPublisher.emit(EventNames.DOWNLOAD_CHROME, new DownalodChromeEvent(percent)); } -export async function getFinancialAccountDetails(): Promise { - const config = await getConfig(configFilePath); - const eventEmitter = new BudgetTrackingEventEmitter(); - - const startDate = moment().subtract(30, 'days').startOf('day').toDate(); - - const companyIdToTransactions = await scrapeFinancialAccountsAndFetchTransactions( - config.scraping, - startDate, - eventEmitter, - ); - const financialAccountDetails: { name: string; accountNumber: string }[] = []; - Object.keys(companyIdToTransactions).forEach((companyId) => { - let accountNumbers = companyIdToTransactions[companyId].map((transaction) => transaction.accountNumber); - accountNumbers = _.uniq(accountNumbers); - accountNumbers.forEach((accountNumber) => financialAccountDetails.push({ name: companyId, accountNumber })); - }); - return financialAccountDetails; -} - async function fetchTransactions( account: AccountToScrapeConfig, startDate: Date, @@ -142,6 +117,7 @@ async function fetchTransactions( }, emitImporterProgressEvent, chromePath, + eventPublisher, ); if (!scrapeResult.success) { throw new Error(`${scrapeResult.errorType}: ${scrapeResult.errorMessage}`); @@ -193,14 +169,12 @@ async function postProcessTransactions( function enrichTransaction(transaction: Transaction, companyId: string, accountNumber: string): EnrichedTransaction { const hash = calculateTransactionHash(transaction, companyId, accountNumber); - // const category = categoryCalculation.getCategoryNameByTransactionDescription(transaction.description); - const enrichedTransaction: EnrichedTransaction = { + return { ...transaction, accountNumber, // category, hash, }; - return enrichedTransaction; } function transactionsDateComparator(t1: Transaction, t2: Transaction) { diff --git a/packages/main/src/manual/setupHelpers.ts b/packages/main/src/manual/setupHelpers.ts index dc3b8ffe..5873b39c 100644 --- a/packages/main/src/manual/setupHelpers.ts +++ b/packages/main/src/manual/setupHelpers.ts @@ -2,7 +2,6 @@ import { FETCH_YNAB_ACCOUNT_DATA_STATUS, type YnabAccountDataType, type YnabConfig } from '@/backend/commonTypes'; import { getConfig } from '@/backend/configManager/configManager'; import { getYnabAccountDetails, isAccessTokenValid } from '@/backend/export/outputVendors/ynab/ynab'; -// import { getFinancialAccountDetails } from '@/backend/import/importTransactions'; export async function getYnabAccountData(_: unknown, ynabOptions: YnabConfig['options']): Promise { const config = await getConfig(); @@ -13,21 +12,16 @@ export async function getYnabAccountData(_: unknown, ynabOptions: YnabConfig['op status: FETCH_YNAB_ACCOUNT_DATA_STATUS.INVALID_ACCESS_TOKEN, }; } - // const ynabAccountDataPromise = getYnabAccountDetails(config.outputVendors); - // const financialAccountDetailsPromise = getFinancialAccountDetails(); - // const [ynabAccountData, financialAccountDetails] = await Promise.all([ynabAccountDataPromise, financialAccountDetailsPromise]); const ynabAccountData = await getYnabAccountDetails( config.outputVendors, ynabOptions.budgetId, ynabOptions.accessToken, ); - // const financialAccountDetails = []; return { status: ynabAccountData.categories ? FETCH_YNAB_ACCOUNT_DATA_STATUS.SUCCESS : FETCH_YNAB_ACCOUNT_DATA_STATUS.INVALID_BUDGET_ID, ynabAccountData, - // financialAccountDetails }; } diff --git a/packages/renderer/src/components/App.tsx b/packages/renderer/src/components/App.tsx index 45254c70..d5b600dc 100644 --- a/packages/renderer/src/components/App.tsx +++ b/packages/renderer/src/components/App.tsx @@ -1,6 +1,7 @@ import { StoresProvider } from '../store'; import './App.css'; import Body from './Body'; +import GetOtp from './GetOtp'; import TopBar from './topBar/TopBar'; function App() { @@ -9,6 +10,7 @@ function App() {
+
); From 09f6b09b0d11806f3d67fd9331721e1ef4d1a853 Mon Sep 17 00:00:00 2001 From: "shai.ungar" Date: Mon, 25 Nov 2024 20:47:47 +0200 Subject: [PATCH 2/2] fix import --- packages/renderer/src/components/GetOtp.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/renderer/src/components/GetOtp.tsx b/packages/renderer/src/components/GetOtp.tsx index 91d97bda..0cd50f25 100644 --- a/packages/renderer/src/components/GetOtp.tsx +++ b/packages/renderer/src/components/GetOtp.tsx @@ -1,7 +1,7 @@ import React, { useEffect, useState } from 'react'; import { Button, FormControl, Modal } from 'react-bootstrap'; import { observer } from 'mobx-react-lite'; -import { useConfigStore } from '/@/store/ConfigStore'; +import { useConfigStore } from '../store/ConfigStore'; import { sendOTPResponse } from '#preload'; const GetOtp = () => {