From 3386b54c349bbb5269473b9c69f4f817aa83a6d2 Mon Sep 17 00:00:00 2001 From: Foma Date: Thu, 9 May 2024 09:09:43 +0200 Subject: [PATCH] [bunq] initial plugin --- .../converters/accounts/account.test.ts | 30 ++++ .../converters/transactions/outcome.test.ts | 53 ++++++++ src/plugins/bunq/api.ts | 128 ++++++++++++++++++ src/plugins/bunq/converters.ts | 128 ++++++++++++++++++ src/plugins/bunq/fetchApi.ts | 124 +++++++++++++++++ src/plugins/bunq/index.ts | 32 +++++ src/plugins/bunq/models.ts | 127 +++++++++++++++++ src/plugins/bunq/preferences.xml | 24 ++++ 8 files changed, 646 insertions(+) create mode 100644 src/plugins/bunq/__tests__/converters/accounts/account.test.ts create mode 100644 src/plugins/bunq/__tests__/converters/transactions/outcome.test.ts create mode 100644 src/plugins/bunq/api.ts create mode 100644 src/plugins/bunq/converters.ts create mode 100644 src/plugins/bunq/fetchApi.ts create mode 100644 src/plugins/bunq/index.ts create mode 100644 src/plugins/bunq/models.ts create mode 100755 src/plugins/bunq/preferences.xml diff --git a/src/plugins/bunq/__tests__/converters/accounts/account.test.ts b/src/plugins/bunq/__tests__/converters/accounts/account.test.ts new file mode 100644 index 000000000..1734bb86f --- /dev/null +++ b/src/plugins/bunq/__tests__/converters/accounts/account.test.ts @@ -0,0 +1,30 @@ +import { convertAccount } from '../../../converters' +import { AccountData, BunqAccountType } from '../../../models' +import { AccountOrCard, AccountType } from '../../../../../types/zenmoney' + +const bunqAccount: AccountData = { + type: BunqAccountType.Bank, + id: 3782, + description: 'Account name', + currency: 'EUR', + balance: { value: '222.12' }, + alias: [ + { type: 'IBAN', value: '333444555', name: 'Alias name' } + ], + status: 'ACTIVE' +} + +const convertedAccount: AccountOrCard = { + type: AccountType.checking, + title: 'Account name', + instrument: 'EUR', + balance: 222.12, + syncIds: ['3782', '333444555'], + id: '3782' +} + +describe('convertAccounts', () => { + it('converts current account', () => { + expect(convertAccount(bunqAccount)).toEqual(convertedAccount) + }) +}) diff --git a/src/plugins/bunq/__tests__/converters/transactions/outcome.test.ts b/src/plugins/bunq/__tests__/converters/transactions/outcome.test.ts new file mode 100644 index 000000000..29a483f10 --- /dev/null +++ b/src/plugins/bunq/__tests__/converters/transactions/outcome.test.ts @@ -0,0 +1,53 @@ +import { BunqPayment } from '../../../models' +import { Transaction } from '../../../../../types/zenmoney' +import { convertTransactions } from '../../../converters' + +const bunqTransaction: BunqPayment = { + id: 1212, + created: '2024-05-02 16:14:44.108574', + monetary_account_id: 444, + amount: { + currency: 'EUR', + value: '-24.78' + }, + description: 'FORU ROTTERDAM, NL', + type: 'MASTERCARD', + merchant_reference: null, + counterparty_alias: { + iban: null, + display_name: 'ROTTERDAM FORU', + merchant_category_code: '5651' + }, + sub_type: 'PAYMENT' +} + +const formattedTransaction: Transaction = { + date: new Date(Date.UTC(2024, 4, 2, 16, 14, 44, 108)), + movements: [ + { + id: '1212', + sum: -24.78, + account: { + id: '444' + }, + fee: 0, + invoice: null + } + ], + merchant: { + title: 'ROTTERDAM FORU', + mcc: 5651, + category: undefined, + location: null, + city: null, + country: null + }, + comment: 'FORU ROTTERDAM, NL', + hold: false +} + +describe('convertTransaction', () => { + it('should convert transaction', () => { + expect(convertTransactions([bunqTransaction], [])).toEqual([formattedTransaction]) + }) +}) diff --git a/src/plugins/bunq/api.ts b/src/plugins/bunq/api.ts new file mode 100644 index 000000000..0a30138c9 --- /dev/null +++ b/src/plugins/bunq/api.ts @@ -0,0 +1,128 @@ +import { + AccountData, + BunqAccountType, + BunqAccountResponse, + BunqPaymentListResponse, + BunqPayment, + DateTimeString, + InstallationContext, + Preferences, + SessionContext +} from './models' +import { fetchApiInstallation, fetchDeviceSet, fetchMonetaryAccountList, fetchPayments, fetchSessionStart } from './fetchApi' +import { generatePemCerts, limitRequests } from './utils' +import { isString, last } from 'lodash' + +export async function registerDevice (preferences: Preferences): Promise { + const installationContext = await installationApi() + + await deviceSetApi(installationContext, preferences) + + return installationContext +} + +export async function sessionStartApi (installationContext: InstallationContext, preferences: Preferences): Promise { + return await fetchSessionStart(installationContext.installationToken, installationContext.clientPrivateKey, preferences.apiKey).then((data) => ( + { + sessionToken: data.Response[1].Token.token, + userId: data.Response[2].UserPerson.id + })) +} + +export async function getAccountsList (sessionContext: SessionContext): Promise { + const accounts: AccountData[] = [] + const result = await fetchMonetaryAccountList(sessionContext.userId, sessionContext.sessionToken) + const olderIdPointer = 'older_id=' + const timer = limitRequests(3, 3000) + + await timer() + + const mapAccounts = (accounts: BunqAccountResponse): AccountData[] => accounts.map((account) => { + const monetaryAccount = Object.values(account)[0] + const monetaryType = Object.keys(account)[0] + const type = ((): BunqAccountType => { + switch (monetaryType) { + case 'MonetaryAccountInvestment': + return BunqAccountType.Investment + default: + return BunqAccountType.Bank + } + })() + + const accountData: AccountData = { ...monetaryAccount, type } + + return accountData + }).filter((account) => (account.status === 'ACTIVE') && + (account.type !== BunqAccountType.Investment)) // we don't sync investment accounts, because they have no proper balance and transactions + + accounts.push(...mapAccounts(result.Response)) + + let olderUrl = result.Pagination.older_url + while (isString(olderUrl)) { + const olderIdPos = olderUrl.lastIndexOf(olderIdPointer) + const olderId = Number(olderUrl.substring(olderIdPos + olderIdPointer.length)) + + await timer() + const newResult = await fetchMonetaryAccountList(sessionContext.userId, sessionContext.sessionToken, olderId) + olderUrl = newResult.Pagination.older_url + accounts.push(...mapAccounts(newResult.Response)) + } + + return accounts +} + +export async function getTransactionsForAccounts (accounts: AccountData[], sessionContext: SessionContext, dateFrom: DateTimeString): Promise { + const transactions: BunqPayment[] = [] + + for (const account of accounts) { + transactions.push(...await getTransactionsApi(account.id, sessionContext, dateFrom)) + } + + return transactions +} + +async function getTransactionsApi (accountId: number, sessionContext: SessionContext, dateFrom: DateTimeString): Promise { + const transactions: BunqPayment[] = [] + + const result = await fetchPayments(sessionContext.userId, accountId, sessionContext.sessionToken) + const olderIdPointer = 'older_id=' + const timer = limitRequests(3, 3000) + + await timer() + + const mapTransactions = (payments: BunqPaymentListResponse): BunqPayment[] => payments.filter((payment) => { + const transaction: BunqPayment = payment.Payment + + return transaction.created >= dateFrom + }).map((payment) => payment.Payment) + + transactions.push(...mapTransactions(result.Response)) + + let olderUrl = result.Pagination.older_url + let lastReceivedDate = last(transactions)?.created ?? '' + while (isString(olderUrl) && lastReceivedDate >= dateFrom) { + const olderIdPos = olderUrl.lastIndexOf(olderIdPointer) + const olderId = Number(olderUrl.substring(olderIdPos + olderIdPointer.length)) + + await timer() + const newResult = await fetchPayments(sessionContext.userId, accountId, sessionContext.sessionToken, olderId) + olderUrl = newResult.Pagination.older_url + const newTransactions = mapTransactions(newResult.Response) + transactions.push(...newTransactions) + lastReceivedDate = last(newTransactions)?.created ?? '' + } + + return transactions +} + +async function installationApi (): Promise { + const certs = await generatePemCerts() + return await fetchApiInstallation(certs.publicKey).then((data) => ({ + clientPrivateKey: certs.privateKey, + installationToken: data.Response[1].Token.token + })) +} + +async function deviceSetApi (installationContext: InstallationContext, preferences: Preferences): Promise { + return await fetchDeviceSet(installationContext.installationToken, preferences.apiKey) +} diff --git a/src/plugins/bunq/converters.ts b/src/plugins/bunq/converters.ts new file mode 100644 index 000000000..76a0dd8c1 --- /dev/null +++ b/src/plugins/bunq/converters.ts @@ -0,0 +1,128 @@ +import { AccountOrCard, AccountReferenceById, AccountType, Merchant, Movement, Transaction } from '../../types/zenmoney' +import { AccountData, BunqPayment, DateTimeString } from './models' +import { flatten, isNull, isObject, isString } from 'lodash' + +export function convertAccount (apiAccount: AccountData): AccountOrCard { + return { + id: String(apiAccount.id), + balance: Number(apiAccount.balance.value), + syncIds: [String(apiAccount.id), ...apiAccount.alias.map((alias) => alias.value)], + instrument: apiAccount.currency, + title: apiAccount.description, + type: AccountType.checking + } +} + +export function convertTransactions (apiTransactions: BunqPayment[], accounts: AccountData[]): Transaction[] { + const accountAliases: Array<[string, AccountData]> = flatten(accounts.map( + account => account.alias.map<[string, AccountData]>(alias => ([alias.value, account])))) + const accountsMap: Map = new Map(accountAliases) + + const transactions: Transaction[] = apiTransactions.map((transaction) => convertTransaction(transaction, accountsMap)) + + return mergeTransfers(transactions) +} + +export function convertTransaction (apiTransaction: BunqPayment, ownAccounts: Map): Transaction { + const id = String(apiTransaction.id) + const accountId = String(apiTransaction.monetary_account_id) + const description = apiTransaction.description + const mmc = apiTransaction.counterparty_alias.merchant_category_code + const reference = apiTransaction.merchant_reference + const counterIban = apiTransaction.counterparty_alias.iban + const mainMovement: Movement = { + id, + account: { id: accountId }, + invoice: null, + sum: Number(apiTransaction.amount.value), + fee: 0 + } + + const isSepaTransfer = apiTransaction.type === 'EBA_SCT' && apiTransaction.sub_type === 'SCT' + const isMastercardPayment = apiTransaction.type === 'MASTERCARD' && apiTransaction.sub_type === 'PAYMENT' + const isIgnoreTypeName = isSepaTransfer || isMastercardPayment + const merchantName = isIgnoreTypeName ? apiTransaction.counterparty_alias.display_name : `${apiTransaction.type} ${apiTransaction.sub_type}: ${apiTransaction.counterparty_alias.display_name}` + const merchant: Merchant = { + title: merchantName, + mcc: isString(mmc) ? Number(mmc) : null, + category: reference ?? undefined, + location: null, + city: null, + country: null + } + + const ownCounter = !isNull(counterIban) && ownAccounts.get(counterIban) + const isOwnCounter = isObject(ownCounter) + + const alterMovement: Movement = { + id: null, + fee: 0, + sum: -Number(apiTransaction.amount.value), + invoice: null, + account: { id: (isOwnCounter && String(ownCounter.id)) || '' } + } + + const movementsWithAlter: [Movement, Movement] = [mainMovement, alterMovement] + return { + hold: false, + date: formatDate(apiTransaction.created), + movements: isOwnCounter ? movementsWithAlter : [mainMovement], + merchant: isOwnCounter ? null : merchant, + comment: description + } +} + +function mergeTransfers (transactions: Transaction[]): Transaction[] { + const filteredTransactions: Transaction[] = [] + const transfersHash: {[accountId: string]: {[sumValue: string]: {[timeStamp: string]: Movement | undefined}}} = {} + + for (const transaction of transactions) { + const isTransfer = transaction.movements.length === 2 + + if (!isTransfer) { + filteredTransactions.push(transaction) + continue + } + + const mainMovement = transaction.movements[0] + const mainMovementAccountId = (mainMovement.account as AccountReferenceById).id + const mainSumString = String(mainMovement.sum) + const mainDateString = transaction.date.toLocaleDateString() + transaction.date.toLocaleTimeString() + + const isSaved = isObject(transfersHash[mainMovementAccountId]?.[mainSumString]?.[mainDateString]) + + if (isSaved) { + transfersHash[mainMovementAccountId][mainSumString][mainDateString] = undefined + continue + } + + filteredTransactions.push(transaction) + + const altMovement = transaction.movements[1] as Movement + const altAccountId = (altMovement.account as AccountReferenceById).id + const altSumString = String(altMovement.sum) + const altDateString = transaction.date.toLocaleDateString() + transaction.date.toLocaleTimeString() + + transfersHash[altAccountId] = transfersHash[altAccountId] ?? {} + transfersHash[altAccountId][altSumString] = transfersHash[altAccountId][altSumString] ?? {} + transfersHash[altAccountId][altSumString][altDateString] = altMovement + } + + return filteredTransactions +} + +// We need to behave dates as they are given in UTC timestamp +function formatDate (dateString: DateTimeString): Date { + const localDate = new Date(dateString) + const utcDate = Date.UTC( + localDate.getFullYear(), + localDate.getMonth(), + localDate.getDate(), + localDate.getHours(), + localDate.getMinutes(), + localDate.getSeconds(), + localDate.getMilliseconds() + ) + + return new Date(utcDate) +} diff --git a/src/plugins/bunq/fetchApi.ts b/src/plugins/bunq/fetchApi.ts new file mode 100644 index 000000000..6cd6da242 --- /dev/null +++ b/src/plugins/bunq/fetchApi.ts @@ -0,0 +1,124 @@ +import { fetch, FetchOptions, FetchResponse, ParseError } from '../../common/network' +import { BankMessageError, TemporaryUnavailableError } from '../../errors' +import { + ApiError, + ApiListResponse, + ApiResponse, + BunqAccountResponse, + BunqPaymentListResponse, + BunqInstallationResponse, + BunqSessionResponse +} from './models' +import { isNumber, isObject } from 'lodash' +import { APP_VERSION, OS_VERSION } from '../tbc-ge/models' +import { signObject } from './utils' + +const baseUrl = 'https://api.bunq.com/v1/' + +async function fetchApi (url: string, options: FetchOptions): Promise { + let response: FetchResponse + if (!options.sanitizeRequestLog) { + options.sanitizeRequestLog = {} + } + + const headers: object = isObject(options.headers) ? options.headers : {} + options.headers = { + ...headers, + 'User-Agent': `Zenmoney Bunq Plugin a${APP_VERSION} (Android; Android ${OS_VERSION}; ANDROID_PHONE)`, + 'Content-Type': 'application/json; charset=UTF-8', + 'Cache-Control': 'no-cache' + } + + options.stringify = JSON.stringify + options.parse = JSON.parse + + try { + response = await fetch(baseUrl + url, options) + } catch (e) { + if (e instanceof ParseError && e.response.status === 502) { + throw new TemporaryUnavailableError() + } + throw e + } + return response +} + +export async function fetchApiInstallation (publicKey: string): Promise> { + const body = { client_public_key: publicKey } + const method = 'POST' + + const response = await fetchApi('installation', { method, body }) + + throwIsNotSuccess(response) + + return response.body as ApiResponse +} + +export async function fetchDeviceSet (installationToken: string, userApiKey: string): Promise { + const body = { + description: 'Zenmoney sync plugin', + secret: userApiKey, + permitted_ips: ['*'] + } + const method = 'POST' + const headers = { + 'X-Bunq-Client-Authentication': installationToken + } + + const response = await fetchApi('device-server', { method, body, headers }) + + throwIsNotSuccess(response) +} + +export async function fetchSessionStart (installationToken: string, clientPrivateKey: string, userApiKey: string): Promise> { + const body = { + secret: userApiKey + } + const method = 'POST' + const signature: string = await signObject(body, clientPrivateKey) + const headers = { + 'X-Bunq-Client-Signature': signature, + 'X-Bunq-Client-Authentication': installationToken + } + + const response = await fetchApi('session-server', { method, body, headers }) + + throwIsNotSuccess(response) + + return response.body as ApiResponse +} + +export async function fetchMonetaryAccountList (userId: number, sessionToken: string, olderId?: number): Promise> { + const method = 'GET' + const headers = { + 'X-Bunq-Client-Authentication': sessionToken + } + + const response = await fetchApi(`user/${userId}/monetary-account${isNumber(olderId) ? `?older_id=${olderId}` : ''}`, { method, headers }) + + throwIsNotSuccess(response) + + return response.body as ApiListResponse +} + +export async function fetchPayments (userId: number, monetaryId: number, sessionToken: string, olderId?: number): Promise> { + const method = 'GET' + const headers = { + 'X-Bunq-Client-Authentication': sessionToken + } + + const response = await fetchApi(`user/${userId}/monetary-account/${monetaryId}/payment/${isNumber(olderId) ? `?older_id=${olderId}` : ''}`, { method, headers }) + + throwIsNotSuccess(response) + + return response.body as ApiListResponse +} + +function throwIsNotSuccess (response: FetchResponse): void { + if (response.status !== 200) { + const error = response.body as ApiError + + const errMessage = error?.Error[0]?.error_description_translated ?? 'Unknown error' + throw new BankMessageError(errMessage) + } +} diff --git a/src/plugins/bunq/index.ts b/src/plugins/bunq/index.ts new file mode 100644 index 000000000..9dcc7386c --- /dev/null +++ b/src/plugins/bunq/index.ts @@ -0,0 +1,32 @@ +import { Account, ScrapeFunc } from '../../types/zenmoney' +import { getAccountsList, getTransactionsForAccounts, registerDevice, sessionStartApi } from './api' +import { InstallationContext, Preferences } from './models' +import { isUndefined } from 'lodash' +import { convertAccount, convertTransactions } from './converters' + +export const scrape: ScrapeFunc = async ({ preferences, fromDate }) => { + let savedData = ZenMoney.getData('installationContext') as InstallationContext | undefined + + if (isUndefined(savedData)) { + savedData = await registerDevice(preferences) + ZenMoney.setData('installationContext', savedData) + ZenMoney.saveData() + } + + const sessionContext = await sessionStartApi(savedData, preferences) + + const bunqAccounts = await getAccountsList(sessionContext) + + const accountsToSync = bunqAccounts.filter( + account => !ZenMoney.isAccountSkipped(String(account.id)) + ) + const accounts: Account[] = accountsToSync.map(bunqAccount => convertAccount(bunqAccount)) + + const bunqTransactions = await getTransactionsForAccounts(accountsToSync, sessionContext, fromDate.toISOString()) + const transactions = convertTransactions(bunqTransactions, bunqAccounts) + + return { + accounts, + transactions + } +} diff --git a/src/plugins/bunq/models.ts b/src/plugins/bunq/models.ts new file mode 100644 index 000000000..9bf09e1d9 --- /dev/null +++ b/src/plugins/bunq/models.ts @@ -0,0 +1,127 @@ + +// UTC date and time standard in format YYYY-MM-DD hh:mm:ss.ssssss +// ISO 8601 format https://en.wikipedia.org/wiki/ISO_8601 +export type DateTimeString = string + +export interface ApiResponse { + Response: T +} + +export interface ApiListResponse extends ApiResponse { + Pagination: { + futureUrl: string + newer_url: string + older_url: string + } +} + +export interface ApiError { + Error: [{ + error_description: string + error_description_translated: string + } + ] +} + +export type BunqInstallationResponse = [ + {Id: {id: number}}, + {Token: { + id: number + created: DateTimeString + updated: DateTimeString + token: string + }}, + {ServerPublicKey: { + server_public_key: string + }} +] + +export type BunqSessionResponse = [ + {Id: {id: number}}, + {Token: { + id: number + token: string + }}, + {UserPerson: {id: number}} +] + +export interface BunqMonetaryAccount { + id: number + balance: { + value: string + } + alias: BunqAccountAlias[] + currency: string + description: string + status: string +} + +export enum BunqAccountType { + Bank, + Investment +} +export interface AccountData extends BunqMonetaryAccount { + type: BunqAccountType +} + +export interface BunqMonetaryAccountBank { + MonetaryAccountBank: BunqMonetaryAccount +} + +export interface BunqMonetaryAccountSavings { + MonetaryAccountSavings: BunqMonetaryAccount +} + +export interface BunqMonetaryAccountJoint { + MonetaryAccountJoint: BunqMonetaryAccount +} + +export interface BunqMonetaryAccountInvestment { + MonetaryAccountInvestment: BunqMonetaryAccount +} + +export type BunqAccountResponse = Array + +export interface BunqAccountAlias { + type: 'IBAN' | string + name: string + value: string +} + +export interface BunqPayment { + amount: { + value: string + currency: string + } + description: string + merchant_reference: string | null + id: number + created: DateTimeString + monetary_account_id: number + type: string + sub_type: string + counterparty_alias: { + display_name: string + iban: string | null + merchant_category_code?: string + } +} + +export type BunqPaymentListResponse = Array<{ + Payment: BunqPayment +}> + +export interface InstallationContext { + clientPrivateKey: string + installationToken: string +} + +export interface SessionContext { + userId: number + sessionToken: string +} + +// Input preferences from schema in preferences.xml +export interface Preferences { + apiKey: string +} diff --git a/src/plugins/bunq/preferences.xml b/src/plugins/bunq/preferences.xml new file mode 100755 index 000000000..7418fcc78 --- /dev/null +++ b/src/plugins/bunq/preferences.xml @@ -0,0 +1,24 @@ + + + + +