diff --git a/core/dbpf.py b/core/dbpf.py index 412a826..9dd926d 100644 --- a/core/dbpf.py +++ b/core/dbpf.py @@ -66,7 +66,18 @@ def __init__(self, filename, offset=0, error_callback=None, require_identifier=T #print(f"DBPF v{self.majorVersion}.{self.minorVersion}") + + def __enter__(self): + + return self + + def __exit__(self, exc_type, exc_value, traceback): + + self.close() + return False + + def close(self): self.file.close() @@ -426,6 +437,9 @@ def error(e): cfg = SC4Config(filename, 0, error) + with open("a9dd6e06.cfg", "wb") as file: + file.write(cfg.decompress_subfile("a9dd6e06").read()) + print(cfg.get_simcity_4_cfg()) #print(f"{dbpf.majorVersion}.{dbpf.minorVersion}") diff --git a/core/events.py b/core/events.py new file mode 100644 index 0000000..58316fa --- /dev/null +++ b/core/events.py @@ -0,0 +1,79 @@ +from __future__ import annotations + +import time +from threading import Lock, Condition + + +class EventChannel: + + + MAX_EVENTS = 1000 + + + def __init__(self): + + self._lock = Lock() + self._condition = Condition(self._lock) + self._events: dict[str, list[dict]] = {} + + + def subscribe(self, user_id: str): + + with self._lock: + if user_id in self._events: + return + self._events.setdefault(user_id, []) + + self.push('user_join', {'user_id': user_id}, exclude=[user_id]) + + + def unsubscribe(self, user_id: str): + + if not user_id: + return + + with self._lock: + if not user_id in self._events: + return + self._events.pop(user_id) + + self.push('user_leave', {'user_id': user_id}, exclude=[user_id]) + + + def push(self, event_type: str, event_context: dict, exclude=None): + + event = { + 'event': event_type, + **event_context + } + + print(event) + + with self._condition: + user_ids = list(self._events.keys()) + for user_id in user_ids: + if exclude and user_id in exclude: + continue + self._events[user_id].append(event) + while len(self._events[user_id]) > self.MAX_EVENTS: + self._events[user_id].pop(0) + self._condition.notify_all() + + + def listen(self, user_id, timeout=5) -> list[dict]: + + with self._condition: + + if user_id not in self._events: + raise ValueError( + f"User {user_id!r} is not subscribed to the event channel." + ) + + while not self._events[user_id]: + if not self._condition.wait(timeout): + return [] + + events = list(self._events[user_id]) + self._events[user_id].clear() + + return events diff --git a/core/networking.py b/core/networking.py index 5f336a0..ad1a249 100644 --- a/core/networking.py +++ b/core/networking.py @@ -31,67 +31,89 @@ COMMAND_TOKEN = 'Token' COMMAND_TIME = 'Time' COMMAND_LOADING_BACKGROUND = 'LdgBkg' +COMMAND_SUBSCRIBE = 'Subscr' def send_json(s: socket.socket, data, length_encoding="I"): - if data is None: - data = {} + try: + + if data is None: + data = {} - data = json.dumps(data).encode() + data = json.dumps(data).encode() - s.sendall(struct.pack(length_encoding, len(data)) + data) + s.sendall(struct.pack(length_encoding, len(data)) + data) + + except NetworkException: + raise + except Exception as e: + raise pretty_exception(e) from e def recv_json(s: socket.socket, length_encoding="I"): - length_header_size = struct.calcsize(length_encoding) - length_header = b"" + try: - while len(length_header) < length_header_size: - if d := s.recv(length_header_size - len(length_header)): - length_header += d - else: - raise ConnectionClosedException() + length_header_size = struct.calcsize(length_encoding) + length_header = b"" - data_size = struct.unpack(length_encoding, length_header)[0] - data_size_read = 0 + while len(length_header) < length_header_size: + if d := s.recv(length_header_size - len(length_header)): + length_header += d + else: + raise ConnectionClosedException() - data = b"" - - while data_size_read < data_size: + data_size = struct.unpack(length_encoding, length_header)[0] + data_size_read = 0 - buffer_size = min(BUFFER_SIZE, data_size - data_size_read) + data = b"" + + while data_size_read < data_size: - if d := s.recv(buffer_size): - data += d - data_size_read += len(d) - else: - raise ConnectionClosedException() + buffer_size = min(BUFFER_SIZE, data_size - data_size_read) + + if d := s.recv(buffer_size): + data += d + data_size_read += len(d) + else: + raise ConnectionClosedException() - if len(data) < 1: - raise NetworkException('No data received.') + if len(data) < 1: + raise NetworkException('No data received.') + + return json.loads(data.decode()) - return json.loads(data.decode()) + except NetworkException: + raise + except Exception as e: + raise pretty_exception(e) from e def recv_exact(s: socket.socket, length: int) -> bytes: - data = bytearray() - remaining = length + try: - while remaining > 0: + data = bytearray() + remaining = length - chunk = s.recv(remaining) + while remaining > 0: - if not chunk: - raise ConnectionClosedException() - - data += chunk - remaining -= len(chunk) + chunk = s.recv(remaining) + + if not chunk: + raise ConnectionClosedException() + + data += chunk + remaining -= len(chunk) + + return bytes(data) + + except NetworkException: + raise + except Exception as e: + raise pretty_exception(e) from e - return bytes(data) - def send_message(s: socket.socket, is_request=True, command="Ping", headers=None): @@ -110,19 +132,19 @@ def send_message(s: socket.socket, is_request=True, command="Ping", headers=None message = m.encode('ascii') while len(message) < 14: - message += b"\x00" + message += b'\x00' - h = json.dumps(headers).encode() + h = json.dumps(headers).encode() if headers else b'' l = struct.pack("H", len(h)) message += l + h s.sendall(message) - except NetworkException as e: - raise e + except NetworkException: + raise except Exception as e: - raise NetworkException(e) from e + raise pretty_exception(e) from e def recv_message(s: socket.socket): @@ -157,36 +179,44 @@ def recv_message(s: socket.socket): # 2 bytes header length l = struct.unpack("H", recv_exact(s, 2))[0] - headers = json.loads(recv_exact(s, l).decode()) + headers = json.loads(recv_exact(s, l).decode()) if l else {} + + return is_request, command, headers except NetworkException: raise except Exception as e: - raise NetworkException(e) from e + raise pretty_exception(e) from e - return is_request, command, headers def request(s, command, **headers) -> dict: + + try: - send_message(s, True, command, headers) + send_message(s, True, command, headers) - is_request, c, h = recv_message(s) + is_request, c, h = recv_message(s) - if is_request: - raise NetworkException( - "Expected response message but received request message." - ) + if is_request: + raise NetworkException( + "Expected response message but received request message." + ) - if c != command: - raise NetworkException( - f"Expected command {command!r} but received {c!r}." - ) - - if error := h.get('error'): - raise NetworkException(error) + if c != command: + raise NetworkException( + f"Expected command {command!r} but received {c!r}." + ) + + if error := h.get('error'): + raise NetworkException(error) - return h + return h + + except NetworkException: + raise + except Exception as e: + raise pretty_exception(e) from e def respond(s, command, **headers): @@ -224,36 +254,43 @@ def pluck_header(headers: dict, key: str, type: Type) -> Any: def recv_files(s: socket.socket, file_table): - for checksum, filesize, relpath in file_table: + try: - def _recv_file(): + for checksum, filesize, relpath in file_table: - filesize_read: int = 0 - checksummer = hashlib.md5() + def _recv_file(): - while filesize_read < filesize: + filesize_read: int = 0 + checksummer = hashlib.md5() - filesize_remaining = filesize - filesize_read - buffersize = min(filesize_remaining, BUFFER_SIZE) + while filesize_read < filesize: - chunk = s.recv(buffersize) + filesize_remaining = filesize - filesize_read + buffersize = min(filesize_remaining, BUFFER_SIZE) - if not chunk: - raise ConnectionClosedException() + chunk = s.recv(buffersize) - filesize_read += len(chunk) - checksummer.update(chunk) + if not chunk: + raise ConnectionClosedException() - yield chunk + filesize_read += len(chunk) + checksummer.update(chunk) - checksum_actual = checksummer.hexdigest() - if checksum != checksum_actual: - raise NetworkException( - f"Checksum mismatch for {relpath!r}: " - f"expected {checksum!r}, got {checksum_actual!r}." - ) + yield chunk - yield checksum, filesize, relpath, _recv_file() + checksum_actual = checksummer.hexdigest() + if checksum != checksum_actual: + raise NetworkException( + f"Checksum mismatch for {relpath!r}: " + f"expected {checksum!r}, got {checksum_actual!r}." + ) + + yield checksum, filesize, relpath, _recv_file() + + except NetworkException: + raise + except Exception as e: + raise pretty_exception(e) from e def interpret_socket_error(e: BaseException) -> str: @@ -335,7 +372,19 @@ def interpret_socket_error(e: BaseException) -> str: return f"Unrecognized socket error: [Errno {err}] {e.strerror or e}" # Fallback for unexpected types - return f"Unknown error: {e.__class__.__name__!r}: {e}" + return f"{e.__class__.__name__!r}: {e}" + + +def pretty_exception(e): + + if isinstance(e, NetworkException): + return e + elif isinstance(e, socket.timeout): + return TimeoutException() + else: + return NetworkException( + interpret_socket_error(e) + ) class Socket(socket.socket): @@ -407,13 +456,14 @@ def __init__(self, address=None, timeout=10, **options): super().__init__(**options) - self.settimeout(timeout) + if timeout is not None: + self.settimeout(timeout) try: if address: self.connect(address) except Exception as e: - raise NetworkException(e) from e + raise pretty_exception(e) from e def add_server(self, host, port, **headers) -> bool: @@ -581,6 +631,21 @@ def save_result(self): return pluck_header(headers, 'result', str) + + def subscribe(self, **headers): + + self.request( + command=COMMAND_SUBSCRIBE, **headers + ) + + + def events(self) -> list: + + _, _, response = self.recv_message() + + return pluck_header(response, 'events', list) + + class ServerSocket(Socket): @@ -635,12 +700,14 @@ def __init__(self, c: Socket, private=False): COMMAND_USER_ID: self.res_user_id, COMMAND_TOKEN: self.res_token, COMMAND_TIME: self.res_time, - COMMAND_LOADING_BACKGROUND: self.res_loading_background + COMMAND_LOADING_BACKGROUND: self.res_loading_background, + COMMAND_SUBSCRIBE: self.res_subscribe } self.require_auth = [ COMMAND_SAVE, - COMMAND_TOKEN + COMMAND_TOKEN, + COMMAND_SUBSCRIBE ] if private: @@ -656,6 +723,7 @@ def authenticate(self): ... def res_add_server(self): self.respond() def res_check_password(self): self.respond() + def res_events(self): self.respond() def res_info(self): self.respond() def res_password_enabled(self): self.respond() def res_ping(self): self.respond() @@ -670,6 +738,7 @@ def res_user_id(self): self.respond() def res_token(self): self.respond() def res_time(self): self.respond() def res_loading_background(self): self.respond() + def res_subscribe(self): self.respond() def get_header(self, key: str, type: Type): @@ -687,6 +756,7 @@ def recv_request(self): if not is_request: raise NetworkException("Expected request but got response.") + self.address = self.c.getpeername()[0] self.command = command self.headers = headers @@ -712,12 +782,7 @@ def respond(self, **headers): class NetworkException(Exception): - def __init__(self, e, *args): - - if isinstance(e, str): - message = e - else: - message = interpret_socket_error(e) + def __init__(self, message, *args): super().__init__(message, *args) @@ -734,3 +799,10 @@ class ConnectionClosedException(NetworkException): def __init__(self): super().__init__("Connection closed.") + + +class TimeoutException(NetworkException): + + def __init__(self): + + super().__init__("Connection timed out.") diff --git a/core/util.py b/core/util.py index 3087e22..0fa691b 100644 --- a/core/util.py +++ b/core/util.py @@ -16,11 +16,13 @@ import struct import subprocess import sys +import time import threading as th import urllib.request import urllib.error from datetime import datetime, timedelta from pathlib import Path +from typing import Callable try: import requests @@ -655,3 +657,15 @@ def sanitize_relpath(basepath: Path, relpath: str) -> Path: return fullpath else: raise ValueError(f"Invalid relative path: \"{relpath}\".") + + +def calculate_latency(function: Callable) -> tuple: + """ + Calls a function and returns the time in miliseconds it took to execute, + as well as the return value of the function. + """ + + start = time.time() + r = function() + end = time.time() + return round(1000 * (end - start)), r diff --git a/sc4mpserver.py b/sc4mpserver.py index 5a18f7a..80eb705 100644 --- a/sc4mpserver.py +++ b/sc4mpserver.py @@ -36,11 +36,11 @@ from core.config import Config from core.database import Database from core.dbpf import SC4Savegame +from core.events import EventChannel from core.networking import ClientSocket, ServerSocket, BaseRequestHandler, \ NetworkException, ConnectionClosedException from core.util import * - # Globals SC4MP_VERSION = "0.9.0" @@ -822,6 +822,7 @@ def __init__(self): self.prep_regions() self.prep_backups() self.prep_server_list() + self.prep_events() # self.prep_upnp() @@ -882,7 +883,7 @@ def run(self): client_requests.setdefault(host, 0) client_requests[host] += 1 - report("Connection accepted with " + str(host) + ":" + str(port) + ".") + # report(f"Connection accepted with {host}.") self.log_client(c) @@ -1442,7 +1443,6 @@ def prep_filetables(self): def prep_server_list(self): - if sc4mp_nostart: return @@ -1457,6 +1457,17 @@ def prep_server_list(self): sc4mp_server_list.start() + def prep_events(self): + + if sc4mp_nostart: + return + + report("Preparing events channel...") + + global sc4mp_event_channel + sc4mp_event_channel = EventChannel() + + def prep_upnp(self): if sc4mp_config['NETWORK']['upnp']: @@ -1881,6 +1892,17 @@ def run(self): # Report success self.outputs[save_id] = "ok" + # Push channel event + sc4mp_event_channel.push( + 'save', { + 'user_id': user_id, + 'save_id': save_id, + 'region': region, + 'coords': (savegameX, savegameY) + }, + exclude=[user_id] + ) + except Exception as e: # Report an error to the request handler @@ -2043,6 +2065,8 @@ def __init__(self, c): super().__init__(c, private=sc4mp_config["SECURITY"]["private"]) + self.user_id = None + def run(self): @@ -2054,17 +2078,11 @@ def run(self): try: - while sc4mp_server_running: - - try: - command, headers = self.recv_request() - except ConnectionClosedException: - break + self.recv_request() - print(f"Request: {command!r}") - # print(f"Request: {command!r} {headers!r}") + print(f"{self.address} - {self.command}") - self.handle_request() + self.handle_request() except ServerException as e: @@ -2441,6 +2459,34 @@ def res_loading_background(self): self.error("Server has no loading background.") + def res_subscribe(self): + + if not self.user_id: + self.error("Invalid user ID.") + + try: + + sc4mp_event_channel.subscribe(self.user_id) + self.respond(status='success') + + while sc4mp_server_running: + try: + self.respond( + events=sc4mp_event_channel.listen(self.user_id) + ) + except NetworkException: + break + + except Exception: + + self.error( + message='An unexpected error occurred in the event channel.' + ) + + finally: + + sc4mp_event_channel.unsubscribe(self.user_id) + class ServerList(th.Thread):