fix sync flush looping and batch size, add alchemy call logging

apps/desktop/src/bun/index.ts

@@ -16,6 +16,7 @@ import {
 	getTransactionDetails,
 	getCachedTransactionHistory,
 } from './lib/transactions'
+import { syncTransactionHistory } from './lib/sync'
 import { migrate } from 'drizzle-orm/bun-sqlite/migrator'
 import { eq } from 'drizzle-orm'
 import { db } from './lib/db'
@@ -185,6 +186,10 @@ type RPC = {
 				}
 				response: TxEntry[]
 			}
+			syncTxHistory: {
+				params: { address: string; chainid: string }
+				response: void
+			}
 			fetchTokenBalances: {
 				params: {
 					address: string
@@ -573,6 +578,13 @@ const rpc = BrowserView.defineRPC<RPC>({
 				const transfers = getCachedTransactionHistory(chainid, address)
 				return transfers.map(mapTransfer)
 			},
+			syncTxHistory: async ({ address, chainid }) => {
+				const key = await Bun.secrets.get({ service: 'koins', name: 'alchemy_key' })
+				if (!key) return
+				syncTransactionHistory(key, chainid, address).catch((e) =>
+					console.error('[sync] error:', e),
+				)
+			},
 			fetchTokenBalances: async ({ address, chainid }) => {
 				const key = await Bun.secrets.get({
 					service: 'koins',

apps/desktop/src/bun/lib/alchemy/get-transaction-history.ts

@@ -2,63 +2,102 @@ import { post } from './call'
 import { getChainID, getURL } from './network'
 import type { Transaction } from './types'
 import { numberToHex } from 'viem'
-import { inspect } from 'util'
 
-export const getTransactionHistory = async (
+type PageResult = {
-	key: string,
+	transfers: Transaction[]
-	chainid: string,
+	pageKey?: string
-	address: string,
+}
-	{ maxCount = 20 }: { maxCount: number },
-) => {
-	const chainId = getChainID(chainid)
-	if (!chainId) return
-
-	const url = getURL({
-		chainId,
-		key,
-	})
 
+async function fetchPage(
+	url: string,
+	fromAddr: string | undefined,
+	toAddr: string | undefined,
+	fromBlock: string | undefined,
+	chainId: string,
+): Promise<{ result?: PageResult }> {
 	const category = ['external', 'erc20']
-
 	if (chainId !== '56') {
 		category.push('internal')
 	}
 
-	const body = (fromAddress?: string, toAddress?: string) => ({
+	const body = {
 		jsonrpc: '2.0',
 		id: 0,
 		method: 'alchemy_getAssetTransfers',
 		params: [
 			{
-				...(fromAddress ? { fromAddress } : {}),
+				...(fromAddr ? { fromAddress: fromAddr } : {}),
-				...(toAddress ? { toAddress } : {}),
+				...(toAddr ? { toAddress: toAddr } : {}),
-				category: category,
+				...(fromBlock ? { fromBlock } : {}),
+				category,
 				withMetadata: true,
-				maxCount: numberToHex(maxCount),
+				maxCount: numberToHex(1000),
+				order: 'asc' as const,
 			},
 		],
-	})
+	}
 
-	const outRequest = post<{
+	return post<PageResult>(url, body)
-		transfers: Transaction[]
+}
-		pageKey: string
-	}>(url, body(address, undefined))
 
-	const inRequest = post<{
+export const getTransactionHistory = async (
-		transfers: Transaction[]
+	key: string,
-	}>(url, body(undefined, address))
+	chainid: string,
+	address: string,
+	opts: { maxCount: number; fromBlock?: string },
+) => {
+	const chainId = getChainID(chainid)
+	if (!chainId) return
 
-	const outRes = await outRequest
+	const url = getURL({ chainId, key })
-	const inRes = await inRequest
+	const seen = new Set<string>()
+	const all: Transaction[] = []
+
+	for (const direction of ['out', 'in'] as const) {
+		const fromAddr = direction === 'out' ? address : undefined
+		const toAddr = direction === 'in' ? address : undefined
+		let cursorBlock = opts.fromBlock
 
-	const outgoing = outRes.result?.transfers ?? []
+		for (let page = 0; page < 100; page++) {
-	const incoming = inRes.result?.transfers ?? []
+			const res = await fetchPage(url, fromAddr, toAddr, cursorBlock, chainId)
+			const transfers = res.result?.transfers ?? []
+			if (transfers.length === 0) break
 
-	const seen = new Set<string>()
+			let pageMaxBlock = 0
-	return [...outgoing, ...incoming].filter((t) => {
+			let firstTimestamp = ''
-		if (!t.hash) return false
+			let lastTimestamp = ''
-		if (seen.has(t.hash)) return false
+			for (const t of transfers) {
+				if (!t.hash) continue
+				if (seen.has(t.hash)) continue
 				seen.add(t.hash)
-		return true
+				all.push(t)
-	})
+
+				if (t.blockNum) {
+					const blockNum = parseInt(t.blockNum, 16)
+					if (blockNum > pageMaxBlock) {
+						pageMaxBlock = blockNum
+					}
+				}
+				if (t.metadata?.blockTimestamp) {
+					if (!firstTimestamp) firstTimestamp = t.metadata.blockTimestamp
+					lastTimestamp = t.metadata.blockTimestamp
+				}
+			}
+
+			console.log(
+				`[alchemy] ${direction} page ${page + 1}: ${transfers.length} transfers, ` +
+				`block ${pageMaxBlock}, ` +
+				`${new Date(firstTimestamp).toISOString()} → ${new Date(lastTimestamp).toISOString()}`,
+			)
+
+			if (pageMaxBlock === 0) break
+			cursorBlock = numberToHex(pageMaxBlock + 1)
+		}
+	}
+
+	console.log(
+		`[alchemy] total: ${all.length} transfers for ${chainid}/${address.slice(0, 10)}... ` +
+		`from ${opts.fromBlock ?? 'genesis'}`,
+	)
+	return all
 }

apps/desktop/src/bun/lib/db/schema.ts

@@ -80,6 +80,15 @@ export const evmWallets = sqliteTable('evm_wallets', {
 	createdAt: text('created_at').notNull(),
 })
 
+export const txSyncStatus = sqliteTable('tx_sync_status', {
+	chainId: text('chain_id').notNull(),
+	address: text('address').notNull(),
+	lastSyncBlock: text('last_sync_block').notNull(),
+	syncedAt: text('synced_at').notNull(),
+}, (table) => ({
+	syncUnique: uniqueIndex('sync_unique').on(table.chainId, table.address),
+}))
+
 export const txHistory = sqliteTable(
 	'tx_history',
 	{

apps/desktop/src/bun/lib/sync/index.ts

@@ -0,0 +1 @@
+export { syncTransactionHistory } from './sync-tx-history'

apps/desktop/src/bun/lib/sync/sync-tx-history.ts

@@ -0,0 +1,106 @@
+import { eq, and } from 'drizzle-orm'
+import { numberToHex } from 'viem'
+import { db } from '../db'
+import { txHistory, txSyncStatus } from '../db/schema'
+import { getTransactionHistory } from '../alchemy/get-transaction-history'
+
+const PAGE_SIZE = 76
+
+export const syncTransactionHistory = async (
+	key: string,
+	chainid: string,
+	address: string,
+) => {
+	const addrLower = address.toLowerCase()
+
+	const existing = db
+		.select()
+		.from(txSyncStatus)
+		.where(
+			and(
+				eq(txSyncStatus.chainId, chainid),
+				eq(txSyncStatus.address, addrLower),
+			),
+		)
+		.get()
+
+	const fromBlock = existing?.lastSyncBlock
+		? numberToHex(Number(existing.lastSyncBlock) + 1)
+		: undefined
+
+	const transfers = await getTransactionHistory(key, chainid, address, {
+		maxCount: 1000,
+		fromBlock,
+	})
+
+	if (!transfers || transfers.length === 0) return
+
+	const prepared: Array<typeof txHistory.$inferInsert> = []
+	let latestBlock = Number(existing?.lastSyncBlock ?? '0')
+
+	for (const t of transfers) {
+		if (!t.hash) continue
+		if (t.blockNum) {
+			const blockNum = parseInt(t.blockNum, 16)
+			if (blockNum > latestBlock) latestBlock = blockNum
+		}
+
+		prepared.push({
+			chainId: chainid,
+			hash: t.hash,
+			blockNum: t.blockNum ? String(parseInt(t.blockNum, 16)) : null,
+			from: t.from?.toLowerCase() ?? null,
+			to: t.to?.toLowerCase() ?? null,
+			value: t.value?.toString() ?? null,
+			asset: t.asset ?? null,
+			category: t.category ?? null,
+			rawContractAddress: t.rawContract?.address?.toLowerCase() ?? null,
+			rawContractDecimal: t.rawContract?.decimal ?? null,
+			metadataBlockTimestamp: t.metadata?.blockTimestamp ?? null,
+			raw: JSON.stringify(t),
+			syncedAt: new Date().toISOString(),
+		})
+	}
+
+	flush(prepared)
+	db.insert(txSyncStatus)
+		.values({
+			chainId: chainid,
+			address: addrLower,
+			lastSyncBlock: String(latestBlock),
+			syncedAt: new Date().toISOString(),
+		})
+		.onConflictDoUpdate({
+			target: [txSyncStatus.chainId, txSyncStatus.address],
+			set: {
+				lastSyncBlock: String(latestBlock),
+				syncedAt: new Date().toISOString(),
+			},
+		})
+		.run()
+
+	console.log(
+		`[sync] ${chainid}/${addrLower}: synced ${prepared.length} transfers up to block ${latestBlock}`,
+	)
+}
+
+function flush(prepared: Array<typeof txHistory.$inferInsert>) {
+	while (prepared.length > 0) {
+		const batch = prepared.splice(0, PAGE_SIZE)
+		const cols = ['chainId', 'hash', 'blockNum', '"from"', '"to"', 'value', 'asset', 'category', 'rawContractAddress', 'rawContractDecimal', 'metadataBlockTimestamp', 'raw', 'syncedAt']
+		const colKeys = ['chainId', 'hash', 'blockNum', 'from', 'to', 'value', 'asset', 'category', 'rawContractAddress', 'rawContractDecimal', 'metadataBlockTimestamp', 'raw', 'syncedAt']
+		const placeholders = batch.map(() => `(${colKeys.map(() => '?').join(',')})`).join(',')
+		const flat = batch.flatMap((r) => colKeys.map((c) => (r as any)[c] ?? null))
+		const sql = `INSERT OR IGNORE INTO tx_history (${cols.join(',')}) VALUES ${placeholders}`
+
+		const dbClient = db.$client
+		dbClient.exec('BEGIN')
+		try {
+			dbClient.run(sql, ...flat)
+			dbClient.exec('COMMIT')
+		} catch (e) {
+			dbClient.exec('ROLLBACK')
+			console.error('[sync] flush error:', e)
+		}
+	}
+}

apps/desktop/src/lib/components/EvmDashboard.svelte

@@ -23,6 +23,7 @@
 	import Plus from '@lucide/svelte/icons/plus'
 	import WalletIcon from '@lucide/svelte/icons/wallet'
 	import LockKeyhole from '@lucide/svelte/icons/lock-keyhole'
+	import ExternalLink from '@lucide/svelte/icons/external-link'
 	import * as Dialog from '$lib/components/ui/dialog/index.js'
 	import QRCode from 'qrcode'
 	import Loader from './loader.svelte'
@@ -267,6 +268,11 @@
 							<p class="font-mono text-xs break-all flex-1">
 								{w.address}
 							</p>
+							<button
+								onclick={() => electrobun.rpc?.request.openExternal({ url: w.explorerAddressUrl })}
+								class="shrink-0 cursor-pointer rounded-md border p-1.5 text-muted-foreground hover:bg-muted transition-colors">
+								<ExternalLink size={16} />
+							</button>
 							{#if qrDataUrl}
 								<Dialog.Root bind:open={qrDialogOpen}>
 									<Dialog.Trigger>

apps/desktop/src/lib/electrobun.ts

@@ -141,6 +141,10 @@ type RPC = {
 				}
 				response: TxEntry[]
 			}
+			syncTxHistory: {
+				params: { address: string; chainid: string }
+				response: void
+			}
 			fetchTokenBalances: {
 				params: {
 					address: string

apps/desktop/src/lib/states/evm-wallet.svelte.ts

@@ -287,6 +287,8 @@ export const EvmWallet = () => {
 		address = account.address
 		const client = createPublicClient({ transport: http(net().rpc) })
 
+		syncTxHistory()
+
 		;(async () => {
 			const [nativeBal] = await tryCatch(
 				client.getBalance({ address: account.address }),
@@ -307,11 +309,17 @@ export const EvmWallet = () => {
 		})()
 	}
 
+	const syncTxHistory = () => {
+		if (!address || !apiKey || !electrobun.rpc) return
+		electrobun.rpc.request.syncTxHistory({ address, chainid: net().chainid })
+	}
+
 	const saveApiKey = async (key: string) => {
 		await electrobun.rpc?.request.setSecret({
 			service: 'koins', name: 'alchemy_key', value: key,
 		})
 		apiKey = key
+		syncTxHistory()
 		if (address) await fetchTxHistory()
 	}
 
@@ -358,6 +366,7 @@ export const EvmWallet = () => {
 		get networkName() { return net().name },
 		get symbol() { return net().symbol },
 		get explorerUrl() { return net().explorerUrl },
+		get explorerAddressUrl() { return address ? `${net().explorerUrl.replace('/tx/', '/address/')}${address}` : '' },
 		get networks() { return networks },
 		get apiKey() { return apiKey },
 		get transactions() { return transactions },
@@ -391,6 +400,7 @@ export const EvmWallet = () => {
 		unlockWallet,
 		deleteWallet,
 		clearSelection,
+		syncTxHistory,
 	}
 }