{
  "page": {
    "id": "xbqltl005rl8",
    "name": "Weld",
    "url": "https://status.weld.app"
  },
  "status": {
    "indicator": "none",
    "description": "All Systems Operational"
  },
  "components": [{"status":"operational","name":"Weld Sync Management UI","created_at":"2021-07-13T16:14:17.145+02:00","updated_at":"2025-06-12T22:44:19.167+02:00","position":3,"description":"app.weld.so","showcase":true,"start_date":"2021-03-02T01:00:00.000+01:00","id":"s2n4f9hqwltv","group_id":null,"page_id":"xbqltl005rl8"},{"status":"operational","name":"Sync Engine","created_at":"2021-07-13T16:39:53.083+02:00","updated_at":"2026-07-18T13:01:16.745+02:00","position":4,"description":"Backend that handles all the data processing","showcase":true,"start_date":"2021-03-02T01:00:00.000+01:00","id":"0xpz8s94p1k1","group_id":null,"page_id":"xbqltl005rl8"},{"status":"operational","name":"Landing page","created_at":"2021-07-13T16:40:15.018+02:00","updated_at":"2021-07-13T16:40:15.018+02:00","position":5,"description":"weld.app","showcase":true,"start_date":"2021-01-03T01:00:00.000+01:00","id":"lf1mwt2399q3","group_id":null,"page_id":"xbqltl005rl8"},{"status":"operational","name":"Orchestration","created_at":"2024-02-16T15:02:27.444+01:00","updated_at":"2024-02-16T15:02:27.444+01:00","position":6,"description":"This is the orchestration feature that binds together ELT, reverse ELT and materialisation","showcase":true,"start_date":"2024-02-16T00:00:00.000+01:00","id":"xvsc1jbg4sr9","group_id":null,"page_id":"xbqltl005rl8"}],
  "incidents": [{"name":"Elevated job failures due to rate limiting issues with AWS","status":"resolved","created_at":"2026-07-18T08:35:26.915+02:00","updated_at":"2026-07-18T13:01:16.794+02:00","monitoring_at":"2026-07-18T12:34:24.647+02:00","resolved_at":"2026-07-18T13:01:16.778+02:00","impact":"major","shortlink":"https://stspg.io/d4h7jsqn7cfm","scheduled_for":null,"scheduled_until":null,"scheduled_remind_prior":false,"scheduled_reminded_at":null,"impact_override":null,"scheduled_auto_in_progress":false,"scheduled_auto_completed":false,"metadata":{},"started_at":"2026-07-18T08:35:26.906+02:00","reminder_intervals":null,"id":"34jln2scqs0r","page_id":"xbqltl005rl8","incident_updates":[{"status":"resolved","body":"We have implemented multiple fixes to avoid the false positives in our stale monitor service.\nWe have implemented parallel processing and split our job into singular docker containers. To allow us to scale them automatically depending on queue build up.\nAnd we have also guarded against build up which in turn terminates the stale monitor until the build up is resolved (should it happen again).","created_at":"2026-07-18T13:01:16.778+02:00","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2026-07-18T13:01:16.778+02:00","display_at":"2026-07-18T13:01:16.778+02:00","deliver_notifications":true,"tweet_id":null,"id":"zkvcbgpkz58l","incident_id":"34jln2scqs0r","custom_tweet":null,"affected_components":[{"code":"0xpz8s94p1k1","name":"Sync Engine","old_status":"partial_outage","new_status":"operational"}]},{"status":"monitoring","body":"We have deployed a fix to make sure our queues gets drained faster. \nEverything seems to be back to normal but monitoring the situation now.","created_at":"2026-07-18T12:34:24.647+02:00","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2026-07-18T12:34:24.647+02:00","display_at":"2026-07-18T12:34:24.647+02:00","deliver_notifications":true,"tweet_id":null,"id":"3bpclsbr5bsn","incident_id":"34jln2scqs0r","custom_tweet":null,"affected_components":[{"code":"0xpz8s94p1k1","name":"Sync Engine","old_status":"partial_outage","new_status":"partial_outage"}]},{"status":"identified","body":"We have been working on improving speeds of our data pipelines to avoid slow starts.\nThis at midnight caused A LOT of jobs to start at the same time overwhelming our system.\nOur system then wasn't able to process all the finishing jobs fast enough and our stale job monitoring kicked in.\n\nThis in turn reset a whole bunch of jobs that were just waiting to get flagged as finished. And restarted them. This caused a vicious cycle of jobs restarting despite finishing and keeping our queues maxed out.\n\nWe are working on breaking these monitors up into singular service now so we can run them individually and in parallel without worrying about side-effects from running many of all monitors.","created_at":"2026-07-18T11:22:34.087+02:00","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2026-07-18T11:22:34.087+02:00","display_at":"2026-07-18T11:22:34.087+02:00","deliver_notifications":true,"tweet_id":null,"id":"xv5tptq2fm7m","incident_id":"34jln2scqs0r","custom_tweet":null,"affected_components":[{"code":"0xpz8s94p1k1","name":"Sync Engine","old_status":"partial_outage","new_status":"partial_outage"}]},{"status":"investigating","body":"We are currently trying to resolve an issue where 429s from Amazon is causing jobs to be flagged as stale incorrectly","created_at":"2026-07-18T08:35:27.001+02:00","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2026-07-18T08:35:27.001+02:00","display_at":"2026-07-18T08:35:27.001+02:00","deliver_notifications":true,"tweet_id":null,"id":"8tzzh6lfkhpg","incident_id":"34jln2scqs0r","custom_tweet":null,"affected_components":[{"code":"0xpz8s94p1k1","name":"Sync Engine","old_status":"operational","new_status":"partial_outage"}]}],"postmortem_body":null,"postmortem_body_last_updated_at":null,"postmortem_ignored":false,"postmortem_published_at":null,"postmortem_notified_subscribers":false,"postmortem_notified_twitter":false,"components":[{"status":"operational","name":"Sync Engine","created_at":"2021-07-13T16:39:53.083+02:00","updated_at":"2026-07-18T13:01:16.745+02:00","position":4,"description":"Backend that handles all the data processing","showcase":true,"start_date":"2021-03-02T01:00:00.000+01:00","id":"0xpz8s94p1k1","page_id":"xbqltl005rl8","group_id":null}]},{"name":"Partial Outage: Web Application","status":"resolved","created_at":"2026-04-16T14:54:49.666+02:00","updated_at":"2026-04-16T18:53:15.838+02:00","monitoring_at":"2026-04-16T15:21:49.028+02:00","resolved_at":"2026-04-16T18:48:51.599+02:00","impact":"major","shortlink":"https://stspg.io/nwsgbdbnz8jq","scheduled_for":null,"scheduled_until":null,"scheduled_remind_prior":false,"scheduled_reminded_at":null,"impact_override":"major","scheduled_auto_in_progress":false,"scheduled_auto_completed":false,"metadata":{},"started_at":"2026-04-16T14:54:49.660+02:00","reminder_intervals":null,"id":"hthwwc9mb97y","page_id":"xbqltl005rl8","incident_updates":[{"status":"resolved","body":"Everything is back to normal. There was a partial outtage related to AWS ECS and our GraphQL federation server that had communication issues internally. \n\nIt only affected the web app and all syncs kept on running without any issues.","created_at":"2026-04-16T18:48:51.599+02:00","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2026-04-16T18:48:51.599+02:00","display_at":"2026-04-16T18:48:51.599+02:00","deliver_notifications":true,"tweet_id":null,"id":"v7lbllhj64zj","incident_id":"hthwwc9mb97y","custom_tweet":null,"affected_components":[{"code":"s2n4f9hqwltv","name":"Weld Sync Management UI","old_status":"operational","new_status":"operational"}]},{"status":"monitoring","body":"A fix has been implemented and we are monitoring the results.","created_at":"2026-04-16T15:21:49.028+02:00","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2026-04-16T15:21:49.028+02:00","display_at":"2026-04-16T15:21:49.028+02:00","deliver_notifications":true,"tweet_id":null,"id":"430sl1zf1k0k","incident_id":"hthwwc9mb97y","custom_tweet":null,"affected_components":[{"code":"s2n4f9hqwltv","name":"Weld Sync Management UI","old_status":"operational","new_status":"operational"}]},{"status":"investigating","body":"We are currently experiencing issues causing slow or failed loading of the web application.\n\nUsers may encounter timeouts or be unable to access parts of the platform.\n\nData syncs and scheduled processes are not affected and continue to run as expected.\n\nOur engineering team is actively investigating and working to resolve the issue as quickly as possible. Updates will follow shortly.","created_at":"2026-04-16T14:54:49.717+02:00","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2026-04-16T14:54:49.717+02:00","display_at":"2026-04-16T14:54:49.717+02:00","deliver_notifications":true,"tweet_id":null,"id":"7xpkg4m6nflf","incident_id":"hthwwc9mb97y","custom_tweet":null,"affected_components":[{"code":"s2n4f9hqwltv","name":"Weld Sync Management UI","old_status":"operational","new_status":"operational"}]}],"postmortem_body":"**Post-Mortem: Partial Web Application Outage — April 16, 2026**\n\n**Duration:** ~3.5 hours \\(14:54–18:48 CEST\\)\n\n**Severity:** Minor — Web UI only. All data syncs and scheduled processes continued running normally.\n\n**Summary**\n\nThe Weld web application experienced intermittent timeouts and failed page loads for approximately 3.5 hours. The issue was isolated to communication between our AWS ECS-hosted services — specifically, the load balancer was unable to reliably forward requests to our GraphQL federation router, resulting in persistent 504 \\(Gateway Timeout\\) errors. No data syncs, pipelines, or scheduled processes were affected.\n\n**Timeline**\n\n* **14:54** — Engineering began receiving reports of slow or failed loading in the web app. Investigation started immediately.\n* **~15:00–15:20** — Initial debugging pointed to 504 errors between the load balancer and the GraphQL router running on ECS. Service redeployments were attempted, which temporarily restored connectivity before the issue recurred.\n* **15:21** — A redeployment was applied and the team moved to active monitoring.\n* **~15:21–18:48** — The issue persisted intermittently despite redeployments. After approximately 2 hours of stability monitoring with no further intervention, the problem resolved on its own.\n* **18:48** — Full resolution confirmed. All systems operating normally.\n\n**Root Cause**\n\nThe load balancer was consistently unable to route requests to the GraphQL federation router on ECS, returning 504 errors. Redeploying the affected services provided temporary relief but did not permanently resolve the issue. The problem ultimately cleared without any code or configuration change on our side, suggesting the root cause was an underlying AWS infrastructure issue affecting ECS networking or load balancer target health checks. As of this writing, AWS has not published a related incident or advisory.\n\n**Impact**\n\n* Users experienced timeouts or blank pages when accessing the web application.\n* All data syncs and scheduled processes were unaffected and ran as expected throughout the incident.\n\n**What Went Well**\n\n* The issue was detected and communicated to users quickly.\n* Data pipeline infrastructure proved resilient — sync operations were fully decoupled from the web UI path and continued without interruption.\n\n**What Could Be Improved**\n\n* **Observability:** Better alerting on sustained 504 rates between the load balancer and ECS targets would allow faster detection before user reports come in.\n* **Redundancy:** Evaluate whether a fallback routing path or a secondary GraphQL router deployment could reduce single-point-of-failure risk in the request path.\n\n**Action Items**\n\n| Action | Owner | Status |\n| --- | --- | --- |\n| Add alerting on elevated 504 rates at the load balancer | Kristian Barrett | Open |\n| Evaluate architectural options for GraphQL router redundancy | Kristian Barrett | Open |\n| Investigate AWS support case for root cause confirmation | Kristian Barrett | Open |","postmortem_body_last_updated_at":"2026-04-16T18:53:15.822+02:00","postmortem_ignored":false,"postmortem_published_at":"2026-04-16T18:53:15.822+02:00","postmortem_notified_subscribers":true,"postmortem_notified_twitter":false,"components":[{"status":"operational","name":"Weld Sync Management UI","created_at":"2021-07-13T16:14:17.145+02:00","updated_at":"2025-06-12T22:44:19.167+02:00","position":3,"description":"app.weld.so","showcase":true,"start_date":"2021-03-02T01:00:00.000+01:00","id":"s2n4f9hqwltv","page_id":"xbqltl005rl8","group_id":null}]},{"name":"Google cloud is down - it is interferring with all of the connected BigQuery instances.","status":"resolved","created_at":"2025-06-12T20:42:48.294+02:00","updated_at":"2025-06-12T22:44:19.229+02:00","monitoring_at":null,"resolved_at":"2025-06-12T22:44:19.214+02:00","impact":"major","shortlink":"https://stspg.io/mpz3ywmgdsh2","scheduled_for":null,"scheduled_until":null,"scheduled_remind_prior":false,"scheduled_reminded_at":null,"impact_override":null,"scheduled_auto_in_progress":false,"scheduled_auto_completed":false,"metadata":{},"started_at":"2025-06-12T20:42:48.281+02:00","reminder_intervals":null,"id":"yn5qc5rdcynw","page_id":"xbqltl005rl8","incident_updates":[{"status":"resolved","body":"Google Cloud is back online. And we can resume normal operations","created_at":"2025-06-12T22:44:19.214+02:00","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2025-06-12T22:44:19.214+02:00","display_at":"2025-06-12T22:44:19.214+02:00","deliver_notifications":true,"tweet_id":null,"id":"zndpwwvvwljr","incident_id":"yn5qc5rdcynw","custom_tweet":null,"affected_components":[{"code":"s2n4f9hqwltv","name":"Weld Sync Management UI","old_status":"partial_outage","new_status":"operational"},{"code":"0xpz8s94p1k1","name":"Sync Engine","old_status":"partial_outage","new_status":"operational"}]},{"status":"identified","body":"Google Cloud is experiencing a full outage: https://status.cloud.google.com/incidents/ow5i3PPK96RduMcb1SsW\nWe keep working on getting everything back online.","created_at":"2025-06-12T21:28:19.859+02:00","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2025-06-12T21:28:19.859+02:00","display_at":"2025-06-12T21:28:19.859+02:00","deliver_notifications":true,"tweet_id":null,"id":"c8rm8f20bf2n","incident_id":"yn5qc5rdcynw","custom_tweet":null,"affected_components":[{"code":"s2n4f9hqwltv","name":"Weld Sync Management UI","old_status":"partial_outage","new_status":"partial_outage"},{"code":"0xpz8s94p1k1","name":"Sync Engine","old_status":"partial_outage","new_status":"partial_outage"}]},{"status":"investigating","body":"Google cloud is down - it is interferring with all of the connected BigQuery instances.\nOur syncs are also halted for now. There is still no official messages from Google.\nWe are trying to figure out what to do.","created_at":"2025-06-12T20:42:48.451+02:00","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2025-06-12T20:42:48.451+02:00","display_at":"2025-06-12T20:42:48.451+02:00","deliver_notifications":true,"tweet_id":null,"id":"0lfv4f5ftqm4","incident_id":"yn5qc5rdcynw","custom_tweet":null,"affected_components":[{"code":"s2n4f9hqwltv","name":"Weld Sync Management UI","old_status":"operational","new_status":"partial_outage"},{"code":"0xpz8s94p1k1","name":"Sync Engine","old_status":"operational","new_status":"partial_outage"}]}],"postmortem_body":null,"postmortem_body_last_updated_at":null,"postmortem_ignored":false,"postmortem_published_at":null,"postmortem_notified_subscribers":false,"postmortem_notified_twitter":false,"components":[{"status":"operational","name":"Weld Sync Management UI","created_at":"2021-07-13T16:14:17.145+02:00","updated_at":"2025-06-12T22:44:19.167+02:00","position":3,"description":"app.weld.so","showcase":true,"start_date":"2021-03-02T01:00:00.000+01:00","id":"s2n4f9hqwltv","page_id":"xbqltl005rl8","group_id":null},{"status":"operational","name":"Sync Engine","created_at":"2021-07-13T16:39:53.083+02:00","updated_at":"2026-07-18T13:01:16.745+02:00","position":4,"description":"Backend that handles all the data processing","showcase":true,"start_date":"2021-03-02T01:00:00.000+01:00","id":"0xpz8s94p1k1","page_id":"xbqltl005rl8","group_id":null}]},{"name":"Feature flag provider Launchdarkly is having issues affecting some of our customers","status":"resolved","created_at":"2025-05-27T15:55:15.277+02:00","updated_at":"2025-05-27T16:59:35.948+02:00","monitoring_at":"2025-05-27T16:43:18.331+02:00","resolved_at":"2025-05-27T16:59:35.933+02:00","impact":"major","shortlink":"https://stspg.io/27s2p9h5nhc9","scheduled_for":null,"scheduled_until":null,"scheduled_remind_prior":false,"scheduled_reminded_at":null,"impact_override":null,"scheduled_auto_in_progress":false,"scheduled_auto_completed":false,"metadata":{},"started_at":"2025-05-27T15:55:15.271+02:00","reminder_intervals":null,"id":"s3w65zt6l45g","page_id":"xbqltl005rl8","incident_updates":[{"status":"resolved","body":"The issue is resolved on our side and launchdarkly is clearing the last issues on their side.","created_at":"2025-05-27T16:59:35.933+02:00","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2025-05-27T16:59:35.933+02:00","display_at":"2025-05-27T16:59:35.933+02:00","deliver_notifications":true,"tweet_id":null,"id":"8m2h2rb256nb","incident_id":"s3w65zt6l45g","custom_tweet":null,"affected_components":[{"code":"s2n4f9hqwltv","name":"Weld Sync Management UI","old_status":"partial_outage","new_status":"operational"}]},{"status":"monitoring","body":"We have bypassed the launch darkly flags and implemented a fallback mechanism. This issue should be resolved. Make sure to hard-refresh your front-end app if you are still having issues.","created_at":"2025-05-27T16:43:18.331+02:00","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2025-05-27T16:43:18.331+02:00","display_at":"2025-05-27T16:43:18.331+02:00","deliver_notifications":true,"tweet_id":null,"id":"qqkld149t1mf","incident_id":"s3w65zt6l45g","custom_tweet":null,"affected_components":[{"code":"s2n4f9hqwltv","name":"Weld Sync Management UI","old_status":"partial_outage","new_status":"partial_outage"}]},{"status":"identified","body":"Launch Darkly is having issue the status is here:\nhttps://status.launchdarkly.com/","created_at":"2025-05-27T15:58:53.593+02:00","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2025-05-27T15:58:53.593+02:00","display_at":"2025-05-27T15:58:53.593+02:00","deliver_notifications":true,"tweet_id":null,"id":"2s77lx7dp0my","incident_id":"s3w65zt6l45g","custom_tweet":null,"affected_components":[{"code":"s2n4f9hqwltv","name":"Weld Sync Management UI","old_status":"partial_outage","new_status":"partial_outage"}]},{"status":"investigating","body":"We are currently figuring out how to disable this service until they have resolved the issues at hand.","created_at":"2025-05-27T15:55:15.346+02:00","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2025-05-27T15:55:15.346+02:00","display_at":"2025-05-27T15:55:15.346+02:00","deliver_notifications":true,"tweet_id":null,"id":"qk3frvc3xxn5","incident_id":"s3w65zt6l45g","custom_tweet":null,"affected_components":[{"code":"s2n4f9hqwltv","name":"Weld Sync Management UI","old_status":"operational","new_status":"partial_outage"}]}],"postmortem_body":null,"postmortem_body_last_updated_at":null,"postmortem_ignored":false,"postmortem_published_at":null,"postmortem_notified_subscribers":false,"postmortem_notified_twitter":false,"components":[{"status":"operational","name":"Weld Sync Management UI","created_at":"2021-07-13T16:14:17.145+02:00","updated_at":"2025-06-12T22:44:19.167+02:00","position":3,"description":"app.weld.so","showcase":true,"start_date":"2021-03-02T01:00:00.000+01:00","id":"s2n4f9hqwltv","page_id":"xbqltl005rl8","group_id":null}]},{"name":"Data Source syncs failing","status":"resolved","created_at":"2025-04-30T15:36:39.152+02:00","updated_at":"2025-04-30T16:44:53.728+02:00","monitoring_at":null,"resolved_at":"2025-04-30T16:03:42.297+02:00","impact":"major","shortlink":"https://stspg.io/18bsj7k0mkj6","scheduled_for":null,"scheduled_until":null,"scheduled_remind_prior":false,"scheduled_reminded_at":null,"impact_override":null,"scheduled_auto_in_progress":false,"scheduled_auto_completed":false,"metadata":{},"started_at":"2025-04-30T15:36:39.148+02:00","reminder_intervals":null,"id":"hmrfzgmmldqk","page_id":"xbqltl005rl8","incident_updates":[{"status":"resolved","body":"The issue causing Data Source syncs to fail was related to a production environment dependency problem. The issue has been identified and corrected, and all systems are now operating normally. We are improving our deployment processes to prevent similar problems in the future.","created_at":"2025-04-30T16:03:42.297+02:00","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2025-04-30T16:03:42.297+02:00","display_at":"2025-04-30T16:03:42.297+02:00","deliver_notifications":true,"tweet_id":null,"id":"y70jt3p33sdq","incident_id":"hmrfzgmmldqk","custom_tweet":null,"affected_components":[{"code":"0xpz8s94p1k1","name":"Sync Engine","old_status":"partial_outage","new_status":"operational"}]},{"status":"identified","body":"We are continuing to work on a fix for this issue.","created_at":"2025-04-30T15:40:43.502+02:00","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2025-04-30T15:40:43.502+02:00","display_at":"2025-04-30T15:40:43.502+02:00","deliver_notifications":false,"tweet_id":null,"id":"1lslty5n488x","incident_id":"hmrfzgmmldqk","custom_tweet":null,"affected_components":[{"code":"0xpz8s94p1k1","name":"Sync Engine","old_status":"operational","new_status":"partial_outage"}]},{"status":"identified","body":"The issue has been identified and a fix is being implemented","created_at":"2025-04-30T15:36:39.201+02:00","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2025-04-30T15:36:39.201+02:00","display_at":"2025-04-30T15:36:39.201+02:00","deliver_notifications":true,"tweet_id":null,"id":"gwpdsgv04wyp","incident_id":"hmrfzgmmldqk","custom_tweet":null,"affected_components":[{"code":"0xpz8s94p1k1","name":"Sync Engine","old_status":"operational","new_status":"operational"}]}],"postmortem_body":"## Problem\n\nWe had a package resolution that failed in production.\n\nHere is a stackoverflow question explaining the problem with this package:  \n[https://stackoverflow.com/questions/77406363/error-err-require-esm-require-of-es-module-node-modules-wrap-ansi-index-js](https://stackoverflow.com/questions/77406363/error-err-require-esm-require-of-es-module-node-modules-wrap-ansi-index-js)  \n  \nThe resolution also didn’t work for us and unfortunately ended up disrupting our ELT pipeline.\n\n## Timeline\n\nApril 30, 2025 14:37 - Faulty code was rolled out  \nApril 30, 2025 14:56 - Alert rolled out on elevated errors in production  \nApril 30, 2025 15:30 - A solution was rolled out and jobs started succeeding again.\n\n## Resolution\n\n  \nWhen we were notified a lot of jobs had already started failing \\(all failed at the same time\\).   \nWe immediately rolled back the affected code and rolled out a former working version of our pipeline workers.\n\n## Future prevention\n\n* Upgrade to use pnpm or a more safe package manager that doesn’t allow hoisting and piggy-bagging of packages.\n* Upgrade our data pipelines to have a “staging” zone in our production env that all new packages goes through. Once one is released the scheduler will schedule 50 jobs to that one. At least x has to finish before it promotes it to production.\n* More noisy alerts on these types of issues \\(we were notified but not quickly enough\\).","postmortem_body_last_updated_at":"2025-04-30T16:44:53.716+02:00","postmortem_ignored":false,"postmortem_published_at":"2025-04-30T16:44:53.716+02:00","postmortem_notified_subscribers":true,"postmortem_notified_twitter":false,"components":[{"status":"operational","name":"Sync Engine","created_at":"2021-07-13T16:39:53.083+02:00","updated_at":"2026-07-18T13:01:16.745+02:00","position":4,"description":"Backend that handles all the data processing","showcase":true,"start_date":"2021-03-02T01:00:00.000+01:00","id":"0xpz8s94p1k1","page_id":"xbqltl005rl8","group_id":null}]},{"name":"Degraded Performance on Amazon Ads Syncs","status":"resolved","created_at":"2024-10-01T11:40:17.611+02:00","updated_at":"2024-10-08T09:03:10.129+02:00","monitoring_at":null,"resolved_at":"2024-10-08T09:03:10.111+02:00","impact":"minor","shortlink":"https://stspg.io/0fhyl8wxjmf1","scheduled_for":null,"scheduled_until":null,"scheduled_remind_prior":false,"scheduled_reminded_at":null,"impact_override":"minor","scheduled_auto_in_progress":false,"scheduled_auto_completed":false,"metadata":{},"started_at":"2024-10-01T11:40:17.606+02:00","reminder_intervals":null,"id":"yc6bbhmk5xlc","page_id":"xbqltl005rl8","incident_updates":[{"status":"resolved","body":"We are pleased to inform you that we have successfully implemented improvements to address the stability issues related to Amazon Ads data syncs. As a result, the syncs are now completing successfully. Additionally, we have taken measures to minimize the impact of similar issues in the future. We sincerely appreciate your patience and understanding as we worked through this challenge. Thank you for your continued trust in our services.","created_at":"2024-10-08T09:03:10.111+02:00","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2024-10-08T09:03:10.111+02:00","display_at":"2024-10-08T09:03:10.111+02:00","deliver_notifications":true,"tweet_id":null,"id":"6qjh3drd68dg","incident_id":"yc6bbhmk5xlc","custom_tweet":null,"affected_components":[{"code":"0xpz8s94p1k1","name":"Sync Engine","old_status":"operational","new_status":"operational"}]},{"status":"identified","body":"We are currently experiencing issues with Amazon Ads data syncs due to increased rate limiting from the Amazon Ads reporting API. Our team is actively working on a solution to minimize the impact, but some Amazon Ads syncs may fail in the meantime. Thank you for your understanding as we work to resolve this issue.","created_at":"2024-10-01T11:40:17.656+02:00","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2024-10-01T11:40:17.656+02:00","display_at":"2024-10-01T11:40:17.656+02:00","deliver_notifications":true,"tweet_id":null,"id":"qq42fwkqpbd0","incident_id":"yc6bbhmk5xlc","custom_tweet":null,"affected_components":[{"code":"0xpz8s94p1k1","name":"Sync Engine","old_status":"operational","new_status":"operational"}]}],"postmortem_body":null,"postmortem_body_last_updated_at":null,"postmortem_ignored":false,"postmortem_published_at":null,"postmortem_notified_subscribers":false,"postmortem_notified_twitter":false,"components":[{"status":"operational","name":"Sync Engine","created_at":"2021-07-13T16:39:53.083+02:00","updated_at":"2026-07-18T13:01:16.745+02:00","position":4,"description":"Backend that handles all the data processing","showcase":true,"start_date":"2021-03-02T01:00:00.000+01:00","id":"0xpz8s94p1k1","page_id":"xbqltl005rl8","group_id":null}]},{"name":"Faulty deploy took out data pipelines. We have rolled back and are restarting all the affected streams","status":"resolved","created_at":"2024-07-26T13:43:52.992+02:00","updated_at":"2024-07-26T16:22:16.453+02:00","monitoring_at":"2024-07-26T14:34:14.471+02:00","resolved_at":"2024-07-26T16:22:16.438+02:00","impact":"major","shortlink":"https://stspg.io/lq3d9h36gchm","scheduled_for":null,"scheduled_until":null,"scheduled_remind_prior":false,"scheduled_reminded_at":null,"impact_override":"major","scheduled_auto_in_progress":false,"scheduled_auto_completed":false,"metadata":{},"started_at":"2024-07-26T13:43:52.986+02:00","reminder_intervals":null,"id":"tfjdb60n2k5g","page_id":"xbqltl005rl8","incident_updates":[{"status":"resolved","body":"The incident is now resolved. We are planning on Monday how to take steps so this kind of build will not take down our pipelines again.","created_at":"2024-07-26T16:22:16.438+02:00","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2024-07-26T16:22:16.438+02:00","display_at":"2024-07-26T16:22:16.438+02:00","deliver_notifications":true,"tweet_id":null,"id":"yyqwk3x4dsrf","incident_id":"tfjdb60n2k5g","custom_tweet":null,"affected_components":[{"code":"0xpz8s94p1k1","name":"Sync Engine","old_status":"operational","new_status":"operational"}]},{"status":"monitoring","body":"Everything is restarted and are resolving. We keep monitoring the issue to make sure everything is back to normal","created_at":"2024-07-26T14:34:14.471+02:00","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2024-07-26T14:34:14.471+02:00","display_at":"2024-07-26T14:34:14.471+02:00","deliver_notifications":true,"tweet_id":null,"id":"rs513ck9r29v","incident_id":"tfjdb60n2k5g","custom_tweet":null,"affected_components":[{"code":"0xpz8s94p1k1","name":"Sync Engine","old_status":"degraded_performance","new_status":"operational"}]},{"status":"identified","body":"The issue was linked to two libraries:\nnode-prune \ngoogleapis\n\nWhere node-prune faultily removed parts of runtime files needed for the googleapis library to run. Which caused the images to fail on start-up and not on build time. We are taking steps to assure this cannot happen again.","created_at":"2024-07-26T14:12:38.851+02:00","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2024-07-26T14:12:38.851+02:00","display_at":"2024-07-26T14:12:38.851+02:00","deliver_notifications":true,"tweet_id":null,"id":"l9n003sys460","incident_id":"tfjdb60n2k5g","custom_tweet":null,"affected_components":[{"code":"0xpz8s94p1k1","name":"Sync Engine","old_status":"degraded_performance","new_status":"degraded_performance"}]},{"status":"identified","body":"The issue has been identified and a fix is being implemented.","created_at":"2024-07-26T13:43:53.070+02:00","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2024-07-26T13:43:53.070+02:00","display_at":"2024-07-26T13:43:53.070+02:00","deliver_notifications":true,"tweet_id":null,"id":"526mh17bcvz3","incident_id":"tfjdb60n2k5g","custom_tweet":null,"affected_components":[{"code":"0xpz8s94p1k1","name":"Sync Engine","old_status":"operational","new_status":"degraded_performance"}]}],"postmortem_body":null,"postmortem_body_last_updated_at":null,"postmortem_ignored":false,"postmortem_published_at":null,"postmortem_notified_subscribers":false,"postmortem_notified_twitter":false,"components":[{"status":"operational","name":"Sync Engine","created_at":"2021-07-13T16:39:53.083+02:00","updated_at":"2026-07-18T13:01:16.745+02:00","position":4,"description":"Backend that handles all the data processing","showcase":true,"start_date":"2021-03-02T01:00:00.000+01:00","id":"0xpz8s94p1k1","page_id":"xbqltl005rl8","group_id":null}]},{"name":"Degraded performance in SQL Editor","status":"resolved","created_at":"2024-05-31T15:04:15.083+02:00","updated_at":"2024-05-31T20:06:16.168+02:00","monitoring_at":"2024-05-31T19:36:42.352+02:00","resolved_at":"2024-05-31T20:05:56.000+02:00","impact":"minor","shortlink":"https://stspg.io/6pspgsq9bwbl","scheduled_for":null,"scheduled_until":null,"scheduled_remind_prior":false,"scheduled_reminded_at":null,"impact_override":null,"scheduled_auto_in_progress":false,"scheduled_auto_completed":false,"metadata":{},"started_at":"2024-05-31T15:04:15.078+02:00","reminder_intervals":null,"id":"c44y7j77y6y9","page_id":"xbqltl005rl8","incident_updates":[{"status":"resolved","body":"Dear Users,\n\nWe recently experienced a performance issue caused by third-party code from our analytics provider, G2. This led to app unresponsiveness on 31st of May 2024.\n\nImpact:\n- Duration: 1pm to 6pm.\n- Affected Users: All users during this period.\n\nResolution: The issue has been resolved by removing the problematic code, restoring app performance.\n\nWe apologize for the inconvenience. For further assistance, contact support via the in-app chat.\n\nThank you for your understanding.\n\nBest regards,\nThe Weld Team","created_at":"2024-05-31T20:05:56.094+02:00","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2024-05-31T20:06:16.150+02:00","display_at":"2024-05-31T20:05:56.000+02:00","deliver_notifications":true,"tweet_id":null,"id":"5qdjtpfzl9dc","incident_id":"c44y7j77y6y9","custom_tweet":null,"affected_components":[{"code":"s2n4f9hqwltv","name":"Weld Sync Management UI","old_status":"degraded_performance","new_status":"operational"}]},{"status":"monitoring","body":"We are continuing to monitor for any further issues.","created_at":"2024-05-31T20:01:00.548+02:00","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2024-05-31T20:01:00.548+02:00","display_at":"2024-05-31T20:01:00.548+02:00","deliver_notifications":true,"tweet_id":null,"id":"ffh6q08s5nck","incident_id":"c44y7j77y6y9","custom_tweet":null,"affected_components":[{"code":"s2n4f9hqwltv","name":"Weld Sync Management UI","old_status":"degraded_performance","new_status":"degraded_performance"}]},{"status":"monitoring","body":"A fix has been implemented and we are monitoring the results","created_at":"2024-05-31T19:36:42.352+02:00","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2024-05-31T19:36:42.352+02:00","display_at":"2024-05-31T19:36:42.352+02:00","deliver_notifications":true,"tweet_id":null,"id":"9c0ntc3yhnrg","incident_id":"c44y7j77y6y9","custom_tweet":null,"affected_components":[{"code":"s2n4f9hqwltv","name":"Weld Sync Management UI","old_status":"degraded_performance","new_status":"degraded_performance"}]},{"status":"investigating","body":"We are currently experiencing issues in the editor that are causing the app to become unresponsive. Please rest assured that sync processes are not affected by this and are functioning as expected. Our team is actively looking into these problems and working on a solution. We appreciate your patience and understanding.\n\nThank you for your continued support.","created_at":"2024-05-31T15:04:15.154+02:00","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2024-05-31T15:04:15.154+02:00","display_at":"2024-05-31T15:04:15.154+02:00","deliver_notifications":true,"tweet_id":null,"id":"ng3q99n42hw2","incident_id":"c44y7j77y6y9","custom_tweet":null,"affected_components":[{"code":"s2n4f9hqwltv","name":"Weld Sync Management UI","old_status":"operational","new_status":"degraded_performance"}]}],"postmortem_body":null,"postmortem_body_last_updated_at":null,"postmortem_ignored":false,"postmortem_published_at":null,"postmortem_notified_subscribers":false,"postmortem_notified_twitter":false,"components":[{"status":"operational","name":"Weld Sync Management UI","created_at":"2021-07-13T16:14:17.145+02:00","updated_at":"2025-06-12T22:44:19.167+02:00","position":3,"description":"app.weld.so","showcase":true,"start_date":"2021-03-02T01:00:00.000+01:00","id":"s2n4f9hqwltv","page_id":"xbqltl005rl8","group_id":null}]},{"name":"Upgrade internal database","status":"completed","created_at":"2024-04-26T09:52:50.180+02:00","updated_at":"2024-04-28T20:56:41.579+02:00","monitoring_at":null,"resolved_at":"2024-04-28T20:56:41.562+02:00","impact":"maintenance","shortlink":"https://stspg.io/70l0p7r21kb0","scheduled_for":"2024-04-28T20:30:00.000+02:00","scheduled_until":"2024-04-28T22:30:00.000+02:00","scheduled_remind_prior":true,"scheduled_reminded_at":"2024-04-28T19:30:02.722+02:00","impact_override":null,"scheduled_auto_in_progress":true,"scheduled_auto_completed":true,"metadata":{},"started_at":"2024-04-26T09:52:50.174+02:00","reminder_intervals":null,"id":"0ntc0xfjffkg","page_id":"xbqltl005rl8","incident_updates":[{"status":"completed","body":"We are back up with an upgraded database now","created_at":"2024-04-28T20:56:41.562+02:00","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2024-04-28T20:56:41.562+02:00","display_at":"2024-04-28T20:56:41.562+02:00","deliver_notifications":true,"tweet_id":null,"id":"2n7wt6wppyf7","incident_id":"0ntc0xfjffkg","custom_tweet":null,"affected_components":[{"code":"s2n4f9hqwltv","name":"Weld Sync Management UI","old_status":"under_maintenance","new_status":"operational"}]},{"status":"in_progress","body":"We are starting maintenance of our database","created_at":"2024-04-28T20:29:23.401+02:00","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2024-04-28T20:29:23.401+02:00","display_at":"2024-04-28T20:29:23.401+02:00","deliver_notifications":true,"tweet_id":null,"id":"drqtwrtzdnw2","incident_id":"0ntc0xfjffkg","custom_tweet":null,"affected_components":[{"code":"s2n4f9hqwltv","name":"Weld Sync Management UI","old_status":"operational","new_status":"under_maintenance"}]},{"status":"scheduled","body":"Maintenance will begin as scheduled in 60 minutes.","created_at":"2024-04-28T19:30:02.629+02:00","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2024-04-28T19:30:02.629+02:00","display_at":"2024-04-28T19:30:02.629+02:00","deliver_notifications":true,"tweet_id":null,"id":"w543jltbzj3q","incident_id":"0ntc0xfjffkg","custom_tweet":null,"affected_components":null},{"status":"scheduled","body":"We will upgrade our internal database to ensure optimal performance during peak load times.","created_at":"2024-04-26T09:52:50.228+02:00","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2024-04-26T09:52:50.228+02:00","display_at":"2024-04-26T09:52:50.228+02:00","deliver_notifications":true,"tweet_id":null,"id":"gmrh3z606p44","incident_id":"0ntc0xfjffkg","custom_tweet":null,"affected_components":[{"code":"s2n4f9hqwltv","name":"Weld Sync Management UI","old_status":"operational","new_status":"operational"}]}],"postmortem_body":null,"postmortem_body_last_updated_at":null,"postmortem_ignored":false,"postmortem_published_at":null,"postmortem_notified_subscribers":false,"postmortem_notified_twitter":false,"components":[{"status":"operational","name":"Weld Sync Management UI","created_at":"2021-07-13T16:14:17.145+02:00","updated_at":"2025-06-12T22:44:19.167+02:00","position":3,"description":"app.weld.so","showcase":true,"start_date":"2021-03-02T01:00:00.000+01:00","id":"s2n4f9hqwltv","page_id":"xbqltl005rl8","group_id":null}]},{"name":"Orchestration failing","status":"resolved","created_at":"2024-02-16T11:06:36.320+01:00","updated_at":"2024-02-16T13:46:29.807+01:00","monitoring_at":"2024-02-16T11:58:14.366+01:00","resolved_at":"2024-02-16T13:46:29.771+01:00","impact":"major","shortlink":"https://stspg.io/c1mft20jlxt6","scheduled_for":null,"scheduled_until":null,"scheduled_remind_prior":false,"scheduled_reminded_at":null,"impact_override":null,"scheduled_auto_in_progress":false,"scheduled_auto_completed":false,"metadata":{},"started_at":"2024-02-16T11:06:36.315+01:00","reminder_intervals":null,"id":"l1519fvsxg6g","page_id":"xbqltl005rl8","incident_updates":[{"status":"resolved","body":"We have setup better monitoring for the queues. We are working on setting up redundancy in the readers and splitting the code into two microservices that handles the orchestration flow.","created_at":"2024-02-16T13:46:29.771+01:00","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2024-02-16T13:46:29.771+01:00","display_at":"2024-02-16T13:46:29.771+01:00","deliver_notifications":true,"tweet_id":null,"id":"dd69gz6lr532","incident_id":"l1519fvsxg6g","custom_tweet":null,"affected_components":[{"code":"0xpz8s94p1k1","name":"Sync Engine","old_status":"partial_outage","new_status":"operational"}]},{"status":"monitoring","body":"We are continuing to monitor for any further issues.","created_at":"2024-02-16T13:45:48.098+01:00","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2024-02-16T13:45:48.098+01:00","display_at":"2024-02-16T13:45:48.098+01:00","deliver_notifications":true,"tweet_id":null,"id":"ljn16wrv7ww4","incident_id":"l1519fvsxg6g","custom_tweet":null,"affected_components":[{"code":"0xpz8s94p1k1","name":"Sync Engine","old_status":"partial_outage","new_status":"partial_outage"}]},{"status":"monitoring","body":"We had a queue on AWS that had locked itself up. We are deploying monitoring and alerting on this new queue to ensure this doesn't go undetected for this long again.","created_at":"2024-02-16T11:58:14.366+01:00","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2024-02-16T11:58:14.366+01:00","display_at":"2024-02-16T11:58:14.366+01:00","deliver_notifications":true,"tweet_id":null,"id":"fz8btm73dyc6","incident_id":"l1519fvsxg6g","custom_tweet":null,"affected_components":[{"code":"0xpz8s94p1k1","name":"Sync Engine","old_status":"partial_outage","new_status":"partial_outage"}]},{"status":"investigating","body":"We are investigating an issue with the orchestration tool that has resulted in most jobs failing.","created_at":"2024-02-16T11:06:36.380+01:00","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2024-02-16T11:06:36.380+01:00","display_at":"2024-02-16T11:06:36.380+01:00","deliver_notifications":true,"tweet_id":null,"id":"5pcc8twj0717","incident_id":"l1519fvsxg6g","custom_tweet":null,"affected_components":[{"code":"0xpz8s94p1k1","name":"Sync Engine","old_status":"operational","new_status":"partial_outage"}]}],"postmortem_body":null,"postmortem_body_last_updated_at":null,"postmortem_ignored":false,"postmortem_published_at":null,"postmortem_notified_subscribers":false,"postmortem_notified_twitter":false,"components":[{"status":"operational","name":"Sync Engine","created_at":"2021-07-13T16:39:53.083+02:00","updated_at":"2026-07-18T13:01:16.745+02:00","position":4,"description":"Backend that handles all the data processing","showcase":true,"start_date":"2021-03-02T01:00:00.000+01:00","id":"0xpz8s94p1k1","page_id":"xbqltl005rl8","group_id":null}]}]
}