diff --git a/dev/tool/src/index.ts b/dev/tool/src/index.ts index d96c6f04a9..faa9a62a3d 100644 --- a/dev/tool/src/index.ts +++ b/dev/tool/src/index.ts @@ -1213,11 +1213,12 @@ export function devTool ( throw new Error('Datalake storage config is required') } + toolCtx.info('using datalake', { datalake: datalakeConfig }) const datalake = createDatalakeClient(datalakeConfig as DatalakeConfig) let workspaces: Workspace[] = [] - const { dbUrl } = prepareTools() - await withDatabase(dbUrl, async (db) => { + const accountUrl = getAccountDBUrl() + await withDatabase(accountUrl, async (db) => { workspaces = await listWorkspacesPure(db) workspaces = workspaces .filter((p) => p.mode !== 'archived') diff --git a/dev/tool/src/storage.ts b/dev/tool/src/storage.ts index 80273dff9c..eb9582b8dd 100644 --- a/dev/tool/src/storage.ts +++ b/dev/tool/src/storage.ts @@ -282,10 +282,19 @@ export async function copyToDatalake ( let time = Date.now() let processedCnt = 0 let skippedCnt = 0 + let failedCnt = 0 function printStats (): void { const duration = Date.now() - time - console.log('...processed', processedCnt, 'skipped', skippedCnt, Math.round(duration / 1000) + 's') + console.log( + '...processed', + processedCnt, + 'skipped', + skippedCnt, + 'failed', + failedCnt, + Math.round(duration / 1000) + 's' + ) time = Date.now() } @@ -319,6 +328,7 @@ export async function copyToDatalake ( ) } catch (err) { console.error('failed to process blob', objectName, err) + failedCnt++ } }) }