From a324058a657945f7a1c13e8f8053d4ddbcb25b9f Mon Sep 17 00:00:00 2001 From: Bjarne Magnussen Date: Sun, 5 Apr 2020 14:19:19 +0200 Subject: [PATCH 1/2] teos+cli: adds functionality to delete an appointment --- cli/teos_cli.py | 36 +++++++++ teos/api.py | 79 ++++++++++++++++++ teos/watcher.py | 209 ++++++++++++++++++++++++++++++++---------------- 3 files changed, 254 insertions(+), 70 deletions(-) diff --git a/cli/teos_cli.py b/cli/teos_cli.py index b53f8c32..c1c2f82f 100644 --- a/cli/teos_cli.py +++ b/cli/teos_cli.py @@ -175,6 +175,42 @@ def get_appointment(locator, cli_sk, teos_pk, teos_url): return response_json +def delete_appointment(locator, cli_sk, teos_pk, teos_url): + """ + Deletes information about an appointment from the tower. + + Args: + locator (:obj:`str`): the appointment locator used to identify it. + cli_sk (:obj:`PrivateKey`): the client's private key. + teos_pk (:obj:`PublicKey`): the tower's public key. + teos_url (:obj:`str`): the teos base url. + + Returns: + :obj:`dict` or :obj:`None`: a dictionary containing the appointment data if the locator is valid and the tower + responds. ``None`` otherwise. + """ + + # FIXME: All responses from the tower should be signed. Not using teos_pk atm. + + valid_locator = is_locator(locator) + + if not valid_locator: + logger.error("The provided locator is not valid", locator=locator) + return None + + message = "delete appointment {}".format(locator) + signature = Cryptographer.sign(message.encode(), cli_sk) + data = {"locator": locator, "signature": signature} + + # Send request to the server. + delete_appointment_endpoint = "{}/delete_appointment".format(teos_url) + logger.info("Sending appointment deletion request to the Eye of Satoshi") + server_response = post_request(data, delete_appointment_endpoint) + response_json = process_post_response(server_response) + + return response_json + + def load_keys(teos_pk_path, cli_sk_path, cli_pk_path): """ Loads all the keys required so sign, send, and verify the appointment. diff --git a/teos/api.py b/teos/api.py index 0859c86e..718db843 100644 --- a/teos/api.py +++ b/teos/api.py @@ -91,6 +91,7 @@ def __init__(self, inspector, watcher, gatekeeper): "/add_appointment": (self.add_appointment, ["POST"]), "/get_appointment": (self.get_appointment, ["POST"]), "/get_all_appointments": (self.get_all_appointments, ["GET"]), + "/delete_appointment": (self.delete_appointment, ["POST"]), } for url, params in routes.items(): @@ -332,6 +333,84 @@ def get_all_appointments(self): return response + def delete_appointment(self): + """ + Delete information about a given appointment state in the Watchtower. + + The information is requested by ``locator``. + + Returns: + :obj:`str`: A json formatted dictionary containing information about the appointment deletion request. + + Returns not found if the user does not have the requested appointment or the locator is invalid. + + Returns bad request if the appointment does not exist in the Watchtower. + + A ``status`` flag is added to the data that signals the status of the deletion request. + + - A successfully deleted appointment is flagged as ``deletion_accepted``. + - An appointment that did not exist (or was already deleted), or where the locator is invalid or the user + does not have the requested appointment, are flagged as ``deletion_rejected``. + + :obj:`tuple`: A tuple containing the response (:obj:`str`) and response code (:obj:`int`). For accepted + appointments, the ``rcode`` is always 200 and the response contains the receipt signature (json). For + rejected appointments, the ``rcode`` is a 404 or 400 and the value contains an application error, and an error + message. Error messages can be found at :mod:`Errors `. + """ + + # Getting the real IP if the server is behind a reverse proxy + remote_addr = get_remote_addr() + + # Check that data type and content are correct. Abort otherwise. + try: + request_data = get_request_data_json(request) + + except TypeError as e: + logger.info("Received invalid delete_appointment request", from_addr="{}".format(remote_addr)) + return abort(HTTP_BAD_REQUEST, e) + + locator = request_data.get("locator") + + try: + self.inspector.check_locator(locator) + logger.info("Received delete_appointment request", from_addr="{}".format(remote_addr), locator=locator) + + message = "delete appointment {}".format(locator) + signature = request_data.get("signature") + user_pk = self.gatekeeper.identify_user(message.encode(), signature) + + summary, signature = self.watcher.pop_appointment(locator, user_pk) + + if summary and signature: + # Appointment sucessfully deleted. + + # Free up the space in slot. + slot_size = ceil(summary.get("size") / ENCRYPTED_BLOB_MAX_SIZE_HEX) + if slot_size > 0: + self.gatekeeper.free_slots(user_pk, slot_size) + + rcode = HTTP_OK + response = { + "locator": locator, + "signature": signature, + "available_slots": self.gatekeeper.registered_users[user_pk].get("available_slots"), + "status": "deletion_accepted", + } + else: + # Appointment could not be deleted. + rcode = HTTP_BAD_REQUEST + response = { + "locator": locator, + "error": "appointment cannot be found", + "status": "deletion_rejected", + } + + except (InspectionFailed, IdentificationFailure): + rcode = HTTP_NOT_FOUND + response = {"locator": locator, "status": "deletion_rejected"} + + return jsonify(response), rcode + def start(self): """ This function starts the Flask server used to run the API. diff --git a/teos/watcher.py b/teos/watcher.py index bb02ff71..9f350b42 100644 --- a/teos/watcher.py +++ b/teos/watcher.py @@ -1,5 +1,5 @@ from queue import Queue -from threading import Thread +from threading import Thread, Lock import common.cryptographer from common.logger import Logger @@ -70,6 +70,7 @@ def __init__(self, db_manager, block_processor, responder, sk_der, max_appointme self.max_appointments = max_appointments self.expiry_delta = expiry_delta self.signing_key = Cryptographer.load_private_key_der(sk_der) + self.mutex = Lock() def awake(self): """Starts a new thread to monitor the blockchain for channel breaches""" @@ -122,41 +123,101 @@ def add_appointment(self, appointment, user_pk): - ``(False, None)`` otherwise. """ - if len(self.appointments) < self.max_appointments: + # Lock to prevent race conditions. + self.mutex.acquire() + try: + if len(self.appointments) < self.max_appointments: + + # The uuids are generated as the RIPMED160(locator||user_pubkey), that way the tower does not need to know + # anything about the user from this point on (no need to store user_pk in the database). + # If an appointment is requested by the user the uuid can be recomputed and queried straightaway (no maps). + uuid = hash_160("{}{}".format(appointment.locator, user_pk)) + self.appointments[uuid] = { + "locator": appointment.locator, + "end_time": appointment.end_time, + "size": len(appointment.encrypted_blob.data), + } + + if appointment.locator in self.locator_uuid_map: + # If the uuid is already in the map it means this is an update. + if uuid not in self.locator_uuid_map[appointment.locator]: + self.locator_uuid_map[appointment.locator].append(uuid) + + else: + self.locator_uuid_map[appointment.locator] = [uuid] + + self.db_manager.store_watcher_appointment(uuid, appointment.to_dict()) + self.db_manager.create_append_locator_map(appointment.locator, uuid) + + appointment_added = True + signature = Cryptographer.sign(appointment.serialize(), self.signing_key) + + logger.info("New appointment accepted", locator=appointment.locator) + + else: + appointment_added = False + signature = None + + logger.info("Maximum appointments reached, appointment rejected", locator=appointment.locator) + + finally: + # Unlock. + self.mutex.release() + + return appointment_added, signature + + def pop_appointment(self, locator, user_pk): + """ + Pops an appointment from the memory (``appointments`` and + ``locator_uuid_map`` dictionaries) and deletes it from the appointments + database. + + The ``Watcher`` will stop monitoring the blockchain (``do_watch``) for + the appointment. + + Args: + locator (:obj:`str`): a 16-byte hex string identifying the appointment. + user_pk(:obj:`str`): the public key that identifies the user who + request the deletion (33-bytes hex str). + + Returns: + :obj:`tuple`: A tuple with the appointment summary and signaling if + it has been deleted or not. + The structure looks as follows: + + - ``(summary, signature)`` if the appointment was deleted. + - ``(None, None)`` otherwise (e.g. appointment did not exist). + """ + + # Lock to prevent race conditions. + self.mutex.acquire() + + try: # The uuids are generated as the RIPMED160(locator||user_pubkey), that way the tower does not need to know # anything about the user from this point on (no need to store user_pk in the database). # If an appointment is requested by the user the uuid can be recomputed and queried straightaway (no maps). - uuid = hash_160("{}{}".format(appointment.locator, user_pk)) - self.appointments[uuid] = { - "locator": appointment.locator, - "end_time": appointment.end_time, - "size": len(appointment.encrypted_blob.data), - } - - if appointment.locator in self.locator_uuid_map: - # If the uuid is already in the map it means this is an update. - if uuid not in self.locator_uuid_map[appointment.locator]: - self.locator_uuid_map[appointment.locator].append(uuid) - - else: - self.locator_uuid_map[appointment.locator] = [uuid] + uuid = hash_160("{}{}".format(locator, user_pk)) - self.db_manager.store_watcher_appointment(uuid, appointment.to_dict()) - self.db_manager.create_append_locator_map(appointment.locator, uuid) + appointment_summary = self.get_appointment_summary(uuid) - appointment_added = True - signature = Cryptographer.sign(appointment.serialize(), self.signing_key) + if appointment_summary: + # Delete appointment as "completed". + Cleaner.delete_completed_appointments([uuid], self.appointments, self.locator_uuid_map, self.db_manager) - logger.info("New appointment accepted", locator=appointment.locator) + message = "delete appointment {}".format(locator) + signature = Cryptographer.sign(message.encode(), self.signing_key) + logger.info("Appointment deleted", locator=locator) - else: - appointment_added = False - signature = None + else: + signature = None + logger.info("Deletion rejected", locator=locator) - logger.info("Maximum appointments reached, appointment rejected", locator=appointment.locator) + finally: + # Unlock. + self.mutex.release() - return appointment_added, signature + return appointment_summary, signature def do_watch(self): """ @@ -174,57 +235,65 @@ def do_watch(self): if len(self.appointments) > 0 and block is not None: txids = block.get("tx") - expired_appointments = [ - uuid - for uuid, appointment_data in self.appointments.items() - if block["height"] > appointment_data.get("end_time") + self.expiry_delta - ] - - Cleaner.delete_expired_appointments( - expired_appointments, self.appointments, self.locator_uuid_map, self.db_manager - ) - - valid_breaches, invalid_breaches = self.filter_valid_breaches(self.get_breaches(txids)) + # Lock to prevent race conditions. + self.mutex.acquire() - triggered_flags = [] - appointments_to_delete = [] + try: + expired_appointments = [ + uuid + for uuid, appointment_data in self.appointments.items() + if block["height"] > appointment_data.get("end_time") + self.expiry_delta + ] - for uuid, breach in valid_breaches.items(): - logger.info( - "Notifying responder and deleting appointment", - penalty_txid=breach["penalty_txid"], - locator=breach["locator"], - uuid=uuid, + Cleaner.delete_expired_appointments( + expired_appointments, self.appointments, self.locator_uuid_map, self.db_manager ) - receipt = self.responder.handle_breach( - uuid, - breach["locator"], - breach["dispute_txid"], - breach["penalty_txid"], - breach["penalty_rawtx"], - self.appointments[uuid].get("end_time"), - block_hash, + valid_breaches, invalid_breaches = self.filter_valid_breaches(self.get_breaches(txids)) + + triggered_flags = [] + appointments_to_delete = [] + + for uuid, breach in valid_breaches.items(): + logger.info( + "Notifying responder and deleting appointment", + penalty_txid=breach["penalty_txid"], + locator=breach["locator"], + uuid=uuid, + ) + + receipt = self.responder.handle_breach( + uuid, + breach["locator"], + breach["dispute_txid"], + breach["penalty_txid"], + breach["penalty_rawtx"], + self.appointments[uuid].get("end_time"), + block_hash, + ) + + # FIXME: Only necessary because of the triggered appointment approach. Fix if it changes. + + if receipt.delivered: + Cleaner.delete_appointment_from_memory(uuid, self.appointments, self.locator_uuid_map) + triggered_flags.append(uuid) + else: + appointments_to_delete.append(uuid) + + # Appointments are only flagged as triggered if they are delivered, otherwise they are just deleted. + appointments_to_delete.extend(invalid_breaches) + self.db_manager.batch_create_triggered_appointment_flag(triggered_flags) + + Cleaner.delete_completed_appointments( + appointments_to_delete, self.appointments, self.locator_uuid_map, self.db_manager ) - # FIXME: Only necessary because of the triggered appointment approach. Fix if it changes. - - if receipt.delivered: - Cleaner.delete_appointment_from_memory(uuid, self.appointments, self.locator_uuid_map) - triggered_flags.append(uuid) - else: - appointments_to_delete.append(uuid) - - # Appointments are only flagged as triggered if they are delivered, otherwise they are just deleted. - appointments_to_delete.extend(invalid_breaches) - self.db_manager.batch_create_triggered_appointment_flag(triggered_flags) - - Cleaner.delete_completed_appointments( - appointments_to_delete, self.appointments, self.locator_uuid_map, self.db_manager - ) + if len(self.appointments) != 0: + logger.info("No more pending appointments") - if len(self.appointments) != 0: - logger.info("No more pending appointments") + finally: + # Unlock. + self.mutex.release() # Register the last processed block for the watcher self.db_manager.store_last_block_hash_watcher(block_hash) From 6700ef0e664d735d220057f9ac63aaad401ba3e5 Mon Sep 17 00:00:00 2001 From: Sergi Delgado Segura Date: Mon, 15 Jun 2020 16:36:37 +0200 Subject: [PATCH 2/2] teos - reworks delete appointment. Still needs mutex --- teos/api.py | 47 ++++++++++++++++++++------------------- teos/watcher.py | 58 ++++++++++++++++++++++++++++--------------------- 2 files changed, 56 insertions(+), 49 deletions(-) diff --git a/teos/api.py b/teos/api.py index 2c4e16c6..970e0187 100644 --- a/teos/api.py +++ b/teos/api.py @@ -5,8 +5,8 @@ from teos import LOG_PREFIX import common.errors as errors from teos.inspector import InspectionFailed -from teos.watcher import AppointmentLimitReached, AppointmentAlreadyTriggered from teos.gatekeeper import NotEnoughSlots, AuthenticationFailure +from teos.watcher import AppointmentLimitReached, AppointmentAlreadyTriggered, AppointmentNotFound from common.logger import Logger from common.cryptographer import hash_160 @@ -321,8 +321,11 @@ def delete_appointment(self): :obj:`tuple`: A tuple containing the response (:obj:`str`) and response code (:obj:`int`). For accepted appointments, the ``rcode`` is always 200 and the response contains the receipt signature (json). For - rejected appointments, the ``rcode`` is a 404 or 400 and the value contains an application error, and an - error message. Error messages can be found at :mod:`Errors `. + rejected appointments, the ``rcode`` is either 400 or 404: + + If the appointment is not found: 404 + If the request is invalid: 400 + If the appointment is already in the responder: 400 + message """ # Getting the real IP if the server is behind a reverse proxy @@ -343,33 +346,29 @@ def delete_appointment(self): logger.info("Received delete_appointment request", from_addr=remote_addr, locator=locator) message = "delete appointment {}".format(locator).encode() - signature = request_data.get("signature") - user_id = self.watcher.gatekeeper.authenticate_user(message, signature) + user_signature = request_data.get("signature") + user_id = self.watcher.gatekeeper.authenticate_user(message, user_signature) - summary, signature = self.watcher.pop_appointment(locator, user_id) + tower_signature = self.watcher.pop_appointment(locator, user_id, user_signature) - if summary and signature: - # Appointment successfully deleted. + rcode = HTTP_OK + response = { + "locator": locator, + "signature": tower_signature, + "available_slots": self.watcher.gatekeeper.registered_users[user_id].get("available_slots"), + "status": "deletion_accepted", + } - # Free up the space in slot. - slot_size = ceil(summary.get("size") / ENCRYPTED_BLOB_MAX_SIZE_HEX) - if slot_size > 0: - self.gatekeeper.free_slots(user_pk, slot_size) + except AppointmentNotFound: + rcode = HTTP_NOT_FOUND + response = {"locator": locator, "status": "deletion_rejected"} - rcode = HTTP_OK - response = { - "locator": locator, - "signature": signature, - "available_slots": self.gatekeeper.registered_users[user_pk].get("available_slots"), - "status": "deletion_accepted", - } - else: - # Appointment could not be deleted. - rcode = HTTP_BAD_REQUEST - response = {"locator": locator, "error": "appointment cannot be found", "status": "deletion_rejected"} + except AppointmentAlreadyTriggered as e: + rcode = HTTP_BAD_REQUEST + response = {"locator": locator, "status": "deletion_rejected", "error": e.msg} except (InspectionFailed, AuthenticationFailure): - rcode = HTTP_NOT_FOUND + rcode = HTTP_BAD_REQUEST response = {"locator": locator, "status": "deletion_rejected"} return jsonify(response), rcode diff --git a/teos/watcher.py b/teos/watcher.py index 3b094c47..9355face 100644 --- a/teos/watcher.py +++ b/teos/watcher.py @@ -22,6 +22,10 @@ class AppointmentLimitReached(BasicException): """Raised when the tower maximum appointment count has been reached""" +class AppointmentNotFound(BasicException): + """Raised when an appointment is not found in the tower""" + + class AppointmentAlreadyTriggered(BasicException): """Raised when an appointment is sent to the Watcher but that same data has already been sent to the Responder""" @@ -335,46 +339,50 @@ def add_appointment(self, appointment, signature): "subscription_expiry": self.gatekeeper.registered_users[user_id].subscription_expiry, } - def pop_appointment(self, locator, user_pk): + def delete_appointment(self, locator, user_id, user_signature): """ - Pops an appointment from the memory (``appointments`` and - ``locator_uuid_map`` dictionaries) and deletes it from the appointments - database. - - The ``Watcher`` will stop monitoring the blockchain (``do_watch``) for - the appointment. + Deletes an appointment from the ``Watcher``. The tower will stop monitoring the deleted appointment. Args: locator (:obj:`str`): a 16-byte hex string identifying the appointment. - user_pk(:obj:`str`): the public key that identifies the user who - request the deletion (33-bytes hex str). + user_id (:obj:`str`): the public key that identifies the user who request the deletion (33-bytes hex str). + user_signature (:obj:`str`): the signature of the request provided by the user. The tower will sign the + signature deletion is accepted. Returns: - :obj:`tuple`: A tuple with the appointment summary and signaling if - it has been deleted or not. - The structure looks as follows: - - ``(summary, signature)`` if the appointment was deleted. - - ``(None, None)`` otherwise (e.g. appointment did not exist). + :obj:`str`: the signature of the user's signature if the appointment if the deletion is accepted. + + Rises: + :obj:`AppointmentAlreadyTriggered`: if the appointment is already in the Responder. The deletion is + therefore rejected. + :obj:`AppointmentNotFound`: if the appointment cannot be found in the tower. The deletion is therefore + rejected. """ - # The uuids are generated as the RIPEMD160(locator||user_pubkey), that way the tower does not need to know - # anything about the user from this point on (no need to store user_pk in the database). - # If an appointment is requested by the user the uuid can be recomputed and queried straightaway (no maps). - uuid = hash_160("{}{}".format(locator, user_pk)) + uuid = hash_160("{}{}".format(locator, user_id)) + + # FIXME: We need to keep track of deletions if uuid in self.appointments: - # Delete appointment as "completed". + # Delete the appointment from both the Watcher and the Gatekeeper Cleaner.delete_completed_appointments([uuid], self.appointments, self.locator_uuid_map, self.db_manager) + Cleaner.delete_gatekeeper_appointments(self.gatekeeper, {uuid: user_id}) - message = "delete appointment {}".format(locator) - signature = Cryptographer.sign(message.encode(), self.signing_key) + # Sign over the user signature as acceptance of the deletion request. + signature = Cryptographer.sign(user_signature.encode(), self.signing_key) logger.info("Appointment deleted", locator=locator) - else: - signature = None - logger.info("Deletion rejected", locator=locator) + return signature - return signature + elif uuid in self.responder.trackers: + message = "Cannot delete an already triggered appointment" + logger.info(message, locator=locator) + raise AppointmentAlreadyTriggered(message) + + else: + message = "Appointment not found. Deletion rejected" + logger.info(message, locator=locator) + raise AppointmentNotFound(message) def do_watch(self): """