diff --git a/hc_spider/model.py b/hc_spider/model.py new file mode 100644 index 0000000..b943fd2 --- /dev/null +++ b/hc_spider/model.py @@ -0,0 +1,31 @@ +import dataclasses +import multiprocessing.connection +import threading + + +@dataclasses.dataclass +class SharedObjects: + visited_nodes: dict + not_visited_nodes: dict + not_valid_urls: list[dict] + urls_with_error: list[dict] + shutdown_event: threading.Event + watchdog_event: threading.Event + worker_finished: threading.Barrier + job_queue: multiprocessing.Queue + log_queue: multiprocessing.Queue + config: dict + lock: multiprocessing.Lock + + def dict(self) -> dict: + return {k: v for k, v in dataclasses.asdict(self).items()} + + +@dataclasses.dataclass +class Response: + links: set[str] + + +@dataclasses.dataclass +class ResponseError: + exc: Exception