From 77c75dc63707554cacd593f5617365e8ec98ec2e Mon Sep 17 00:00:00 2001 From: Tamas Szirtesi Date: Mon, 9 Oct 2023 14:23:30 +0200 Subject: [PATCH] Added models --- hc_spider/model.py | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 hc_spider/model.py diff --git a/hc_spider/model.py b/hc_spider/model.py new file mode 100644 index 0000000..b943fd2 --- /dev/null +++ b/hc_spider/model.py @@ -0,0 +1,31 @@ +import dataclasses +import multiprocessing.connection +import threading + + +@dataclasses.dataclass +class SharedObjects: + visited_nodes: dict + not_visited_nodes: dict + not_valid_urls: list[dict] + urls_with_error: list[dict] + shutdown_event: threading.Event + watchdog_event: threading.Event + worker_finished: threading.Barrier + job_queue: multiprocessing.Queue + log_queue: multiprocessing.Queue + config: dict + lock: multiprocessing.Lock + + def dict(self) -> dict: + return {k: v for k, v in dataclasses.asdict(self).items()} + + +@dataclasses.dataclass +class Response: + links: set[str] + + +@dataclasses.dataclass +class ResponseError: + exc: Exception