From 9f15bf78a1f293c8d546d02a64e270d46371564e Mon Sep 17 00:00:00 2001 From: ndegz Date: Sun, 9 Jul 2023 12:49:54 +0300 Subject: [PATCH] modified asyncio.TaskGroup() in get_products function --- Backend/scraper/auth_example.json | 5 ----- Backend/scraper/main.py | 27 +++++++++++++++------------ 2 files changed, 15 insertions(+), 17 deletions(-) delete mode 100644 Backend/scraper/auth_example.json diff --git a/Backend/scraper/auth_example.json b/Backend/scraper/auth_example.json deleted file mode 100644 index f1c4473..0000000 --- a/Backend/scraper/auth_example.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "host": "", - "username": "", - "password": "" -} diff --git a/Backend/scraper/main.py b/Backend/scraper/main.py index 004a9dd..754fc4e 100644 --- a/Backend/scraper/main.py +++ b/Backend/scraper/main.py @@ -19,7 +19,7 @@ def load_auth(): - FILE = os.path.join("Scraper", "auth.json") + FILE = os.path.join("scraper", "auth.json") with open(FILE, "r") as f: return json.load(f) @@ -54,27 +54,30 @@ async def get_products(page, search_text, selector, get_product): valid_products = [] words = search_text.split(" ") - async with asyncio.TaskGroup() as tg: - for div in product_divs: - async def task(p_div): - product = await get_product(p_div) + tasks = [] - if not product["price"] or not product["url"]: - return + for div in product_divs: + async def task(p_div): + product = await get_product(p_div) - for word in words: - if not product["name"] or word.lower() not in product["name"].lower(): - break + if not product["price"] or not product["url"]: + return + + for word in words: + if not product["name"] or word.lower() not in product["name"].lower(): + break else: valid_products.append(product) - tg.create_task(task(div)) + tasks.append(task(div)) + + await asyncio.gather(*tasks) return valid_products def save_results(results): data = {"results": results} - FILE = os.path.join("Scraper", "results.json") + FILE = os.path.join("scraper", "results.json") with open(FILE, "w") as f: json.dump(data, f)