From c47dae13a93d1b54680cb948230173f0de26c68a Mon Sep 17 00:00:00 2001 From: youqiang Date: Tue, 21 May 2024 14:53:57 +0800 Subject: [PATCH 1/4] update: wait until body attached in playwright-service --- apps/playwright-service/main.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/apps/playwright-service/main.py b/apps/playwright-service/main.py index c28bc63..a044597 100644 --- a/apps/playwright-service/main.py +++ b/apps/playwright-service/main.py @@ -5,6 +5,7 @@ from pydantic import BaseModel app = FastAPI() + class UrlModel(BaseModel): url: str wait: int = None @@ -29,9 +30,12 @@ async def shutdown_event(): async def root(body: UrlModel): context = await browser.new_context() page = await context.new_page() - await page.goto(body.url, timeout=15000) # Set max timeout to 15s - if body.wait: # Check if wait parameter is provided in the request body - await page.wait_for_timeout(body.wait) # Convert seconds to milliseconds for playwright + await page.goto( + body.url, + wait_until="load", + timeout=body.wait if body.wait else 15, + ) + await page.wait_for_selector("body", state="attached") page_content = await page.content() await context.close() json_compatible_item_data = {"content": page_content} From 3e63985e53bc795c4633b218b71d5851691fc585 Mon Sep 17 00:00:00 2001 From: Nicolas Date: Wed, 22 May 2024 10:40:47 -0700 Subject: [PATCH 2/4] Update main.py --- apps/playwright-service/main.py | 1 - 1 file changed, 1 deletion(-) diff --git a/apps/playwright-service/main.py b/apps/playwright-service/main.py index a044597..544f113 100644 --- a/apps/playwright-service/main.py +++ b/apps/playwright-service/main.py @@ -35,7 +35,6 @@ async def root(body: UrlModel): wait_until="load", timeout=body.wait if body.wait else 15, ) - await page.wait_for_selector("body", state="attached") page_content = await page.content() await context.close() json_compatible_item_data = {"content": page_content} From 3aa5f266272039cda4fb6407d57a218623af6e93 Mon Sep 17 00:00:00 2001 From: Nicolas Date: Wed, 22 May 2024 10:45:43 -0700 Subject: [PATCH 3/4] Update main.py --- apps/playwright-service/main.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/apps/playwright-service/main.py b/apps/playwright-service/main.py index 544f113..a2f5e52 100644 --- a/apps/playwright-service/main.py +++ b/apps/playwright-service/main.py @@ -33,8 +33,12 @@ async def root(body: UrlModel): await page.goto( body.url, wait_until="load", - timeout=body.wait if body.wait else 15, + timeout=body.timeout if body.timeout else 15000, ) + # Wait != timeout. Wait is the time to wait after the page is loaded - useful in some cases were "load" / "networkidle" is not enough + if body.wait: + await page.wait_for_timeout(body.wait) + # await page.wait_for_selector("body", state="attached") page_content = await page.content() await context.close() json_compatible_item_data = {"content": page_content} From 4e39701644e724dd01bceebf4488aae1ed7b7900 Mon Sep 17 00:00:00 2001 From: Nicolas Date: Wed, 22 May 2024 12:59:56 -0700 Subject: [PATCH 4/4] Update main.py --- apps/playwright-service/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/playwright-service/main.py b/apps/playwright-service/main.py index a2f5e52..8344adc 100644 --- a/apps/playwright-service/main.py +++ b/apps/playwright-service/main.py @@ -38,7 +38,7 @@ async def root(body: UrlModel): # Wait != timeout. Wait is the time to wait after the page is loaded - useful in some cases were "load" / "networkidle" is not enough if body.wait: await page.wait_for_timeout(body.wait) - # await page.wait_for_selector("body", state="attached") + page_content = await page.content() await context.close() json_compatible_item_data = {"content": page_content}