wieerwill/platform
1
0
Fork 0
mirror of https://github.com/hcengineering/platform.git synced 2025-04-13 03:40:48 +00:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

chore: update datalake configs (#7033)

Browse Source 
Signed-off-by: Alexander Onnikov <Alexander.Onnikov@xored.com>
This commit is contained in:
Alexander Onnikov 2024-10-24 22:08:00 +07:00 committed by GitHub
parent 54f3376438
commit 723026cc59
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 70 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
server/datalake/src/client.ts
View File

@ -230,6 +230,7 @@ export class Client {
}
})
} catch (err) {
ctx.error('failed to upload via signed url', { workspace, objectName, err })
await this.signObjectDelete(ctx, workspace, objectName)
throw new Error('Failed to upload via signed URL')
}

2
workers/datalake/schema/datalake.sql
View File

@ -7,7 +7,7 @@ DROP TYPE IF EXISTS blob.location;
-- B L O B
CREATE TYPE blob.location AS ENUM ('kv', 'weur', 'eeur', 'wnam', 'enam', 'apac');
CREATE TYPE blob.location AS ENUM ('weur', 'eeur', 'wnam', 'enam', 'apac');
\echo "Creating blob.data..."
CREATE TABLE blob.data (

74
workers/datalake/wrangler.toml
View File

@ -10,16 +10,16 @@ kv_namespaces = [
]
r2_buckets = [
{ binding = "DATALAKE_APAC", bucket_name = "datalake-apac", preview_bucket_name = "dev-datalake-eu-west" },
{ binding = "DATALAKE_EEUR", bucket_name = "datalake-eeur", preview_bucket_name = "dev-datalake-eu-west" },
{ binding = "DATALAKE_WEUR", bucket_name = "datalake-weur", preview_bucket_name = "dev-datalake-eu-west" },
{ binding = "DATALAKE_ENAM", bucket_name = "datalake-enam", preview_bucket_name = "dev-datalake-eu-west" },
{ binding = "DATALAKE_WNAM", bucket_name = "datalake-wnam", preview_bucket_name = "dev-datalake-eu-west" }
{ binding = "DATALAKE_APAC", bucket_name = "datalake-eu-west", preview_bucket_name = "dev-datalake-eu-west" },
{ binding = "DATALAKE_EEUR", bucket_name = "datalake-eu-west", preview_bucket_name = "dev-datalake-eu-west" },
{ binding = "DATALAKE_WEUR", bucket_name = "datalake-eu-west", preview_bucket_name = "dev-datalake-eu-west" },
{ binding = "DATALAKE_ENAM", bucket_name = "datalake-eu-west", preview_bucket_name = "dev-datalake-eu-west" },
{ binding = "DATALAKE_WNAM", bucket_name = "datalake-eu-west", preview_bucket_name = "dev-datalake-eu-west" }
]
[[hyperdrive]]
binding = "HYPERDRIVE"
id = "87259c3ae41e41a7b35e610d4282d85a"
id = "055e968f3067414eaa30467d8a9c5021"
localConnectionString = "postgresql://root:roach@localhost:26257/datalake"
[observability]
@ -46,3 +46,65 @@ DATALAKE_WNAM_BUCKET_NAME = "datalake-wnam"
# STREAMS_ACCOUNT_ID = ""
# STREAMS_AUTH_KEY = ""
# R2_ACCOUNT_ID = ""
[env.staging]
name = "datalake-worker-staging"
kv_namespaces = [
{ binding = "datalake_blobs", id = "d52f8b6a9a1e495a8d6625d14d8f3188" }
]
r2_buckets = [
{ binding = "DATALAKE_APAC", bucket_name = "datalake-staging-wnam" },
{ binding = "DATALAKE_EEUR", bucket_name = "datalake-staging-wnam" },
{ binding = "DATALAKE_WEUR", bucket_name = "datalake-staging-weur" },
{ binding = "DATALAKE_ENAM", bucket_name = "datalake-staging-wnam" },
{ binding = "DATALAKE_WNAM", bucket_name = "datalake-staging-wnam" }
]
hyperdrive = [
{ binding = "HYPERDRIVE", id = "1f713bb2ec1b464cb663983feca4c89e" }
]
[env.staging.vars]
DATALAKE_EEUR_BUCKET_NAME = "datalake-staging-weur"
DATALAKE_WEUR_BUCKET_NAME = "datalake-staging-wnam"
DATALAKE_APAC_BUCKET_NAME = "datalake-staging-wnam"
DATALAKE_ENAM_BUCKET_NAME = "datalake-staging-wnam"
DATALAKE_WNAM_BUCKET_NAME = "datalake-staging-wnam"
[env.staging.observability]
enabled = true
head_sampling_rate = 1
[env.dev]
name = "datalake-worker-dev"
kv_namespaces = [
{ binding = "datalake_blobs", id = "5acdcea18a504ab3a2d73556314b0047" }
]
r2_buckets = [
{ binding = "DATALAKE_APAC", bucket_name = "datalake-dev" },
{ binding = "DATALAKE_EEUR", bucket_name = "datalake-dev" },
{ binding = "DATALAKE_WEUR", bucket_name = "datalake-dev" },
{ binding = "DATALAKE_ENAM", bucket_name = "datalake-dev" },
{ binding = "DATALAKE_WNAM", bucket_name = "datalake-dev" }
]
hyperdrive = [
{ binding = "HYPERDRIVE", id = "055e968f3067414eaa30467d8a9c5021" }
]
[env.dev.vars]
DATALAKE_EEUR_BUCKET_NAME = "datalake-dev"
DATALAKE_WEUR_BUCKET_NAME = "datalake-dev"
DATALAKE_APAC_BUCKET_NAME = "datalake-dev"
DATALAKE_ENAM_BUCKET_NAME = "datalake-dev"
DATALAKE_WNAM_BUCKET_NAME = "datalake-dev"
[env.dev.observability]
enabled = true
head_sampling_rate = 1