From 674d1a43627ce00c3a4db84ad2e5afd7d8012fc1 Mon Sep 17 00:00:00 2001 From: Julian Benda Date: Thu, 9 Feb 2023 11:56:16 +0100 Subject: [PATCH 01/40] data backend access tracing --- include/core/jconfiguration.h | 1 + lib/core/jbackend.c | 70 +++++++------- lib/core/jconfiguration.c | 23 ++++- lib/core/jconnection-pool.c | 8 ++ lib/core/jtrace.c | 171 +++++++++++++++++++++++++++++++++- lib/object/jobject.c | 2 +- server/loop.c | 8 ++ server/server.c | 1 + 8 files changed, 242 insertions(+), 42 deletions(-) diff --git a/include/core/jconfiguration.h b/include/core/jconfiguration.h index f65738e0d..2e4bb69b4 100644 --- a/include/core/jconfiguration.h +++ b/include/core/jconfiguration.h @@ -115,6 +115,7 @@ guint32 j_configuration_get_max_connections(JConfiguration*); guint64 j_configuration_get_stripe_size(JConfiguration*); gchar const* j_configuration_get_checksum(JConfiguration*); +guint32 j_configuration_get_uid(JConfiguration*); G_END_DECLS diff --git a/lib/core/jbackend.c b/lib/core/jbackend.c index 19ba65179..747ea56ff 100644 --- a/lib/core/jbackend.c +++ b/lib/core/jbackend.c @@ -289,7 +289,7 @@ j_backend_object_init(JBackend* backend, gchar const* path) g_return_val_if_fail(path != NULL, FALSE); { - J_TRACE("backend_init", "%s", path); + J_TRACE("backend_object_init", "%s", path); ret = backend->object.backend_init(path, &(backend->data)); } @@ -305,7 +305,7 @@ j_backend_object_fini(JBackend* backend) g_return_if_fail(backend->type == J_BACKEND_TYPE_OBJECT); { - J_TRACE("backend_fini", NULL); + J_TRACE("backend_object_fini", NULL); backend->object.backend_fini(backend->data); } } @@ -324,7 +324,7 @@ j_backend_object_create(JBackend* backend, gchar const* namespace, gchar const* g_return_val_if_fail(data != NULL, FALSE); { - J_TRACE("backend_create", "%s, %s, %p", namespace, path, (gpointer)data); + J_TRACE("backend_object_create", "%s, %s, %p", namespace, path, (gpointer)data); ret = backend->object.backend_create(backend->data, namespace, path, data); } @@ -345,7 +345,7 @@ j_backend_object_open(JBackend* backend, gchar const* namespace, gchar const* pa g_return_val_if_fail(data != NULL, FALSE); { - J_TRACE("backend_open", "%s, %s, %p", namespace, path, (gpointer)data); + J_TRACE("backend_object_open", "%s, %s, %p", namespace, path, (gpointer)data); ret = backend->object.backend_open(backend->data, namespace, path, data); } @@ -364,7 +364,7 @@ j_backend_object_delete(JBackend* backend, gpointer data) g_return_val_if_fail(data != NULL, FALSE); { - J_TRACE("backend_delete", "%p", data); + J_TRACE("backend_object_delete", "%p", data); ret = backend->object.backend_delete(backend->data, data); } @@ -384,7 +384,7 @@ j_backend_object_get_all(JBackend* backend, gchar const* namespace, gpointer* it g_return_val_if_fail(iterator != NULL, FALSE); { - J_TRACE("backend_get_all", "%s, %p", namespace, (gpointer)iterator); + J_TRACE("backend_object_get_all", "%s, %p", namespace, (gpointer)iterator); ret = backend->object.backend_get_all(backend->data, namespace, iterator); } @@ -405,7 +405,7 @@ j_backend_object_get_by_prefix(JBackend* backend, gchar const* namespace, gchar g_return_val_if_fail(iterator != NULL, FALSE); { - J_TRACE("backend_get_by_prefix", "%s, %s, %p", namespace, prefix, (gpointer)iterator); + J_TRACE("backend_object_get_by_prefix", "%s, %s, %p", namespace, prefix, (gpointer)iterator); ret = backend->object.backend_get_by_prefix(backend->data, namespace, prefix, iterator); } @@ -424,7 +424,7 @@ j_backend_object_iterate(JBackend* backend, gpointer iterator, gchar const** nam g_return_val_if_fail(name != NULL, FALSE); { - J_TRACE("backend_iterate", "%p, %p", iterator, (gpointer)name); + J_TRACE("backend_object_iterate", "%p, %p", iterator, (gpointer)name); ret = backend->object.backend_iterate(backend->data, iterator, name); } @@ -443,7 +443,7 @@ j_backend_object_close(JBackend* backend, gpointer data) g_return_val_if_fail(data != NULL, FALSE); { - J_TRACE("backend_close", "%p", data); + J_TRACE("backend_object_close", "%p", data); ret = backend->object.backend_close(backend->data, data); } @@ -464,7 +464,7 @@ j_backend_object_status(JBackend* backend, gpointer data, gint64* modification_t g_return_val_if_fail(size != NULL, FALSE); { - J_TRACE("backend_status", "%p, %p, %p", data, (gpointer)modification_time, (gpointer)size); + J_TRACE("backend_object_status", "%p, %p, %p", data, (gpointer)modification_time, (gpointer)size); ret = backend->object.backend_status(backend->data, data, modification_time, size); } @@ -483,7 +483,7 @@ j_backend_object_sync(JBackend* backend, gpointer data) g_return_val_if_fail(data != NULL, FALSE); { - J_TRACE("backend_sync", "%p", data); + J_TRACE("backend_object_sync", "%p", data); ret = backend->object.backend_sync(backend->data, data); } @@ -504,7 +504,7 @@ j_backend_object_read(JBackend* backend, gpointer data, gpointer buffer, guint64 g_return_val_if_fail(bytes_read != NULL, FALSE); { - J_TRACE("backend_read", "%p, %p, %" G_GUINT64_FORMAT ", %" G_GUINT64_FORMAT ", %p", data, buffer, length, offset, (gpointer)bytes_read); + J_TRACE("backend_object_read", "%p, %p, %" G_GUINT64_FORMAT ", %" G_GUINT64_FORMAT ", %p", data, buffer, length, offset, (gpointer)bytes_read); ret = backend->object.backend_read(backend->data, data, buffer, length, offset, bytes_read); } @@ -525,7 +525,7 @@ j_backend_object_write(JBackend* backend, gpointer data, gconstpointer buffer, g g_return_val_if_fail(bytes_written != NULL, FALSE); { - J_TRACE("backend_write", "%p, %p, %" G_GUINT64_FORMAT ", %" G_GUINT64_FORMAT ", %p", data, buffer, length, offset, (gpointer)bytes_written); + J_TRACE("backend_object_write", "%p, %p, %" G_GUINT64_FORMAT ", %" G_GUINT64_FORMAT ", %p", data, buffer, length, offset, (gpointer)bytes_written); ret = backend->object.backend_write(backend->data, data, buffer, length, offset, bytes_written); } @@ -544,7 +544,7 @@ j_backend_kv_init(JBackend* backend, gchar const* path) g_return_val_if_fail(path != NULL, FALSE); { - J_TRACE("backend_init", "%s", path); + J_TRACE("backend_kv_init", "%s", path); ret = backend->kv.backend_init(path, &(backend->data)); } @@ -560,7 +560,7 @@ j_backend_kv_fini(JBackend* backend) g_return_if_fail(backend->type == J_BACKEND_TYPE_KV); { - J_TRACE("backend_fini", NULL); + J_TRACE("backend_kv_fini", NULL); backend->kv.backend_fini(backend->data); } } @@ -579,7 +579,7 @@ j_backend_kv_batch_start(JBackend* backend, gchar const* namespace, JSemantics* g_return_val_if_fail(batch != NULL, FALSE); { - J_TRACE("backend_batch_start", "%s, %p, %p", namespace, (gpointer)semantics, (gpointer)batch); + J_TRACE("backend_kv_batch_start", "%s, %p, %p", namespace, (gpointer)semantics, (gpointer)batch); ret = backend->kv.backend_batch_start(backend->data, namespace, semantics, batch); } @@ -598,7 +598,7 @@ j_backend_kv_batch_execute(JBackend* backend, gpointer batch) g_return_val_if_fail(batch != NULL, FALSE); { - J_TRACE("backend_batch_execute", "%p", batch); + J_TRACE("backend_kv_batch_execute", "%p", batch); ret = backend->kv.backend_batch_execute(backend->data, batch); } @@ -619,7 +619,7 @@ j_backend_kv_put(JBackend* backend, gpointer batch, gchar const* key, gconstpoin g_return_val_if_fail(value != NULL, FALSE); { - J_TRACE("backend_put", "%p, %s, %p, %u", batch, key, (gconstpointer)value, value_len); + J_TRACE("backend_kv_put", "%p, %s, %p, %u", batch, key, (gconstpointer)value, value_len); ret = backend->kv.backend_put(backend->data, batch, key, value, value_len); } @@ -639,7 +639,7 @@ j_backend_kv_delete(JBackend* backend, gpointer batch, gchar const* key) g_return_val_if_fail(key != NULL, FALSE); { - J_TRACE("backend_delete", "%p, %s", batch, key); + J_TRACE("backend_kv_delete", "%p, %s", batch, key); ret = backend->kv.backend_delete(backend->data, batch, key); } @@ -661,7 +661,7 @@ j_backend_kv_get(JBackend* backend, gpointer batch, gchar const* key, gpointer* g_return_val_if_fail(value_len != NULL, FALSE); { - J_TRACE("backend_get", "%p, %s, %p, %p", batch, key, (gpointer)value, (gpointer)value_len); + J_TRACE("backend_kv_get", "%p, %s, %p, %p", batch, key, (gpointer)value, (gpointer)value_len); ret = backend->kv.backend_get(backend->data, batch, key, value, value_len); } @@ -681,7 +681,7 @@ j_backend_kv_get_all(JBackend* backend, gchar const* namespace, gpointer* iterat g_return_val_if_fail(iterator != NULL, FALSE); { - J_TRACE("backend_get_all", "%s, %p", namespace, (gpointer)iterator); + J_TRACE("backend_kv_get_all", "%s, %p", namespace, (gpointer)iterator); ret = backend->kv.backend_get_all(backend->data, namespace, iterator); } @@ -702,7 +702,7 @@ j_backend_kv_get_by_prefix(JBackend* backend, gchar const* namespace, gchar cons g_return_val_if_fail(iterator != NULL, FALSE); { - J_TRACE("backend_get_by_prefix", "%s, %s, %p", namespace, prefix, (gpointer)iterator); + J_TRACE("backend_kv_get_by_prefix", "%s, %s, %p", namespace, prefix, (gpointer)iterator); ret = backend->kv.backend_get_by_prefix(backend->data, namespace, prefix, iterator); } @@ -723,7 +723,7 @@ j_backend_kv_iterate(JBackend* backend, gpointer iterator, gchar const** key, gc g_return_val_if_fail(value_len != NULL, FALSE); { - J_TRACE("backend_iterate", "%p, %p, %p, %p", iterator, (gpointer)key, (gpointer)value, (gpointer)value_len); + J_TRACE("backend_kv_iterate", "%p, %p, %p, %p", iterator, (gpointer)key, (gpointer)value, (gpointer)value_len); ret = backend->kv.backend_iterate(backend->data, iterator, key, value, value_len); } @@ -742,7 +742,7 @@ j_backend_db_init(JBackend* backend, gchar const* path) g_return_val_if_fail(path != NULL, FALSE); { - J_TRACE("backend_init", "%s", path); + J_TRACE("backend_db_init", "%s", path); ret = backend->db.backend_init(path, &(backend->data)); } @@ -758,7 +758,7 @@ j_backend_db_fini(JBackend* backend) g_return_if_fail(backend->type == J_BACKEND_TYPE_DB); { - J_TRACE("backend_fini", NULL); + J_TRACE("backend_db_fini", NULL); backend->db.backend_fini(backend->data); } } @@ -778,7 +778,7 @@ j_backend_db_batch_start(JBackend* backend, gchar const* namespace, JSemantics* g_return_val_if_fail(error == NULL || *error == NULL, FALSE); { - J_TRACE("backend_batch_start", "%s, %p, %p, %p", namespace, (gpointer)semantics, (gpointer)batch, (gpointer)error); + J_TRACE("backend_db_batch_start", "%s, %p, %p, %p", namespace, (gpointer)semantics, (gpointer)batch, (gpointer)error); ret = backend->db.backend_batch_start(backend->data, namespace, semantics, batch, error); } @@ -798,7 +798,7 @@ j_backend_db_batch_execute(JBackend* backend, gpointer batch, GError** error) g_return_val_if_fail(error == NULL || *error == NULL, FALSE); { - J_TRACE("backend_batch_execute", "%p, %p", batch, (gpointer)error); + J_TRACE("backend_db_batch_execute", "%p, %p", batch, (gpointer)error); ret = backend->db.backend_batch_execute(backend->data, batch, error); } @@ -820,7 +820,7 @@ j_backend_db_schema_create(JBackend* backend, gpointer batch, gchar const* name, g_return_val_if_fail(error == NULL || *error == NULL, FALSE); { - J_TRACE("backend_schema_create", "%p, %s, %p, %p", batch, name, (gconstpointer)schema, (gpointer)error); + J_TRACE("backend_schema_create", "%p, %s, %u, %p, %p", batch, name, schema->len, (gconstpointer)schema, (gpointer)error); ret = backend->db.backend_schema_create(backend->data, batch, name, schema, error); } @@ -841,7 +841,7 @@ j_backend_db_schema_get(JBackend* backend, gpointer batch, gchar const* name, bs g_return_val_if_fail(error == NULL || *error == NULL, FALSE); { - J_TRACE("backend_schema_get", "%p, %s, %p, %p", batch, name, (gpointer)schema, (gpointer)error); + J_TRACE("backend_kv_schema_get", "%p, %s, %p, %p", batch, name, (gpointer)schema, (gpointer)error); ret = backend->db.backend_schema_get(backend->data, batch, name, schema, error); } @@ -862,7 +862,7 @@ j_backend_db_schema_delete(JBackend* backend, gpointer batch, gchar const* name, g_return_val_if_fail(error == NULL || *error == NULL, FALSE); { - J_TRACE("backend_schema_delete", "%p, %s, %p", batch, name, (gpointer)error); + J_TRACE("backend_db_schema_delete", "%p, %s, %p", batch, name, (gpointer)error); ret = backend->db.backend_schema_delete(backend->data, batch, name, error); } @@ -885,7 +885,7 @@ j_backend_db_insert(JBackend* backend, gpointer batch, gchar const* name, bson_t g_return_val_if_fail(error == NULL || *error == NULL, FALSE); { - J_TRACE("backend_insert", "%p, %s, %p, %p, %p", batch, name, (gconstpointer)metadata, (gpointer)id, (gpointer)error); + J_TRACE("backend_db_insert", "%p, %s, %u, %p, %p, %p", batch, name, metadata->len, (gconstpointer)metadata, (gpointer)id, (gpointer)error); ret = backend->db.backend_insert(backend->data, batch, name, metadata, id, error); } @@ -908,7 +908,7 @@ j_backend_db_update(JBackend* backend, gpointer batch, gchar const* name, bson_t g_return_val_if_fail(error == NULL || *error == NULL, FALSE); { - J_TRACE("backend_update", "%p, %s, %p, %p, %p", batch, name, (gconstpointer)selector, (gconstpointer)metadata, (gpointer)error); + J_TRACE("backend_db_update", "%p, %s, %u, %p, %p, %p", batch, name, metadata->len, (gconstpointer)selector, (gconstpointer)metadata, (gpointer)error); ret = backend->db.backend_update(backend->data, batch, name, selector, metadata, error); } @@ -929,7 +929,7 @@ j_backend_db_delete(JBackend* backend, gpointer batch, gchar const* name, bson_t g_return_val_if_fail(error == NULL || *error == NULL, FALSE); { - J_TRACE("backend_delete", "%p, %s, %p, %p", batch, name, (gconstpointer)selector, (gpointer)error); + J_TRACE("backend_db_delete", "%p, %s, %u, %p, %p", batch, name, selector->len, (gconstpointer)selector, (gpointer)error); ret = backend->db.backend_delete(backend->data, batch, name, selector, error); } @@ -951,7 +951,7 @@ j_backend_db_query(JBackend* backend, gpointer batch, gchar const* name, bson_t g_return_val_if_fail(error == NULL || *error == NULL, FALSE); { - J_TRACE("backend_query", "%p, %s, %p, %p, %p", batch, name, (gconstpointer)selector, (gpointer)iterator, (gpointer)error); + J_TRACE("backend_db_query", "%p, %s, %u, %p, %p, %p", batch, name, selector->len, (gconstpointer)selector, (gpointer)iterator, (gpointer)error); ret = backend->db.backend_query(backend->data, batch, name, selector, iterator, error); } @@ -972,7 +972,7 @@ j_backend_db_iterate(JBackend* backend, gpointer iterator, bson_t* metadata, GEr g_return_val_if_fail(error == NULL || *error == NULL, FALSE); { - J_TRACE("backend_iterate", "%p, %p, %p", iterator, (gpointer)metadata, (gpointer)error); + J_TRACE("backend_db_iterate", "%p, %p, %p", iterator, (gpointer)metadata, (gpointer)error); ret = backend->db.backend_iterate(backend->data, iterator, metadata, error); } diff --git a/lib/core/jconfiguration.c b/lib/core/jconfiguration.c index 3f3faf56a..62e2ca405 100644 --- a/lib/core/jconfiguration.c +++ b/lib/core/jconfiguration.c @@ -148,6 +148,7 @@ struct JConfiguration guint64 stripe_size; gchar* checksum; + guint32 uid; /** * The reference count. @@ -290,12 +291,14 @@ j_configuration_new_for_data(GKeyFile* key_file) guint32 port; guint32 max_connections; guint64 stripe_size; + guint32 uid = 0; g_return_val_if_fail(key_file != NULL, FALSE); max_operation_size = g_key_file_get_uint64(key_file, "core", "max-operation-size", NULL); max_inject_size = g_key_file_get_uint64(key_file, "core", "max-inject-size", NULL); port = g_key_file_get_integer(key_file, "core", "port", NULL); + uid = g_key_file_get_uint64(key_file, "core", "uid", NULL); max_connections = g_key_file_get_integer(key_file, "clients", "max-connections", NULL); stripe_size = g_key_file_get_uint64(key_file, "clients", "stripe-size", NULL); servers_object = g_key_file_get_string_list(key_file, "servers", "object", NULL, NULL); @@ -364,6 +367,7 @@ j_configuration_new_for_data(GKeyFile* key_file) configuration->max_inject_size = max_inject_size; configuration->max_connections = max_connections; configuration->stripe_size = stripe_size; + configuration->uid = uid; configuration->checksum = NULL; configuration->ref_count = 1; @@ -394,8 +398,15 @@ j_configuration_new_for_data(GKeyFile* key_file) configuration->stripe_size = 4 * 1024 * 1024; } + if (configuration->uid == 0) + { + GRand* rand = g_rand_new(); + configuration->uid = g_rand_int(rand); + g_rand_free(rand); + } + key_file_str = g_key_file_to_data(key_file, NULL, NULL); - configuration->checksum = g_compute_checksum_for_string(G_CHECKSUM_SHA512, key_file_str, -1); + configuration->checksum = g_compute_checksum_for_string(G_CHECKSUM_SHA512, key_file_str, -1); return configuration; } @@ -616,6 +627,16 @@ j_configuration_get_checksum(JConfiguration* configuration) return configuration->checksum; } +guint32 +j_configuration_get_uid(JConfiguration* configuration) +{ + J_TRACE_FUNCTION(NULL); + + g_return_val_if_fail(configuration != NULL, 0); + + return configuration->uid; +} + /** * @} **/ diff --git a/lib/core/jconnection-pool.c b/lib/core/jconnection-pool.c index 67b7e6143..2f41e7c57 100644 --- a/lib/core/jconnection-pool.c +++ b/lib/core/jconnection-pool.c @@ -67,6 +67,8 @@ typedef struct JConnectionPool JConnectionPool; static JConnectionPool* j_connection_pool = NULL; +extern GPrivate j_trace_thread_default; + void j_connection_pool_init(JConfiguration* configuration) { @@ -196,6 +198,8 @@ j_connection_pool_pop_internal(GAsyncQueue* queue, guint* count, gchar const* se gchar const* client_checksum; gchar const* server_checksum; + gchar const* client_program_name; + guint32 client_process_uid; guint op_count; client = g_socket_client_new(); @@ -215,9 +219,13 @@ j_connection_pool_pop_internal(GAsyncQueue* queue, guint* count, gchar const* se j_helper_set_nodelay(connection, TRUE); client_checksum = j_configuration_get_checksum(j_configuration()); + client_program_name = g_get_prgname(); + client_process_uid = j_configuration_get_uid(j_configuration()); message = j_message_new(J_MESSAGE_PING, strlen(client_checksum) + 1); j_message_append_string(message, client_checksum); + j_message_append_string(message, client_program_name); + j_message_append_n(message, &client_process_uid, sizeof(guint32)); j_message_send(message, connection); reply = j_message_new_reply(message); diff --git a/lib/core/jtrace.c b/lib/core/jtrace.c index e9f186dc0..3e2991822 100644 --- a/lib/core/jtrace.c +++ b/lib/core/jtrace.c @@ -45,7 +45,8 @@ enum JTraceFlags J_TRACE_OFF = 0, J_TRACE_ECHO = 1 << 0, J_TRACE_OTF = 1 << 1, - J_TRACE_SUMMARY = 1 << 2 + J_TRACE_SUMMARY = 1 << 2, + J_TRACE_ACCESS = 1 << 3 }; typedef enum JTraceFlags JTraceFlags; @@ -82,6 +83,7 @@ struct JTraceThread **/ guint function_depth; + GArray* stack; #ifdef HAVE_OTF @@ -96,6 +98,21 @@ struct JTraceThread guint32 process_id; } otf; #endif + + struct { + char* program_name; + guint32 uid; + } client; + + struct { + struct { + char* namespace; + } kv; + struct { + char* namespace; + char* path; + } object; + } access; }; typedef struct JTraceThread JTraceThread; @@ -162,6 +179,8 @@ j_trace_thread_new(GThread* thread) trace_thread = g_slice_new(JTraceThread); trace_thread->function_depth = 0; trace_thread->stack = g_array_new(FALSE, FALSE, sizeof(JTraceStack)); + trace_thread->client.program_name = NULL; + trace_thread->client.uid = 0; if (thread == NULL) { @@ -385,6 +404,10 @@ j_trace_init(gchar const* name) { j_trace_flags |= J_TRACE_SUMMARY; } + else if (g_strcmp0(trace_parts[i], "access") == 0) + { + j_trace_flags |= J_TRACE_ACCESS; + } } if (j_trace_flags == J_TRACE_OFF) @@ -433,6 +456,7 @@ j_trace_init(gchar const* name) j_trace_summary_table = g_hash_table_new_full(g_str_hash, g_str_equal, g_free, g_free); } + g_free(j_trace_name); j_trace_name = g_strdup(name); } @@ -499,13 +523,57 @@ j_trace_fini(void) g_free(j_trace_name); } +struct Access { + guint64 timestamp; + guint32 uid; + const char* program_name; + const char* backend; + const char* namespace; + const char* name; + const char* operation; + guint64 size; + guint32 complexity; +}; +typedef struct Access Access; + +static void +j_trace_access_print(const Access* row) { + g_printerr("%"G_GUINT64_FORMAT", %u, %s, %s, %s, %s, %s, %"G_GUINT64_FORMAT", %u\n", + row->timestamp, + row->uid, + row->program_name, + row->backend, + row->namespace, + row->name, + row->operation, + row->size, + row->complexity + ); +} + +static void +store_object_path(JTraceThread* trace_thread, const char* namespace, const char* path) +{ + if (trace_thread->access.object.namespace != namespace) { + g_free(trace_thread->access.object.namespace); + trace_thread->access.object.namespace = NULL; + if(namespace) { trace_thread->access.object.namespace = strdup(namespace); } + } + + if(trace_thread->access.object.path != path) { + g_free(trace_thread->access.object.path); + trace_thread->access.object.path = NULL; + if (path) { trace_thread->access.object.path = strdup(path); } + } +} + + JTrace* j_trace_enter(gchar const* name, gchar const* format, ...) { JTraceThread* trace_thread; JTrace* trace; guint64 timestamp; - va_list args; if (j_trace_flags == J_TRACE_OFF) { @@ -528,8 +596,7 @@ j_trace_enter(gchar const* name, gchar const* format, ...) trace->name = g_strdup(name); trace->enter_time = timestamp; - va_start(args, format); - + if (j_trace_flags & J_TRACE_ECHO) { G_LOCK(j_trace_echo); @@ -537,10 +604,13 @@ j_trace_enter(gchar const* name, gchar const* format, ...) if (format != NULL) { + va_list args; g_autofree gchar* arguments = NULL; + va_start(args, format); arguments = g_strdup_vprintf(format, args); g_printerr("ENTER %s (%s)\n", name, arguments); + va_end(args); } else { @@ -550,6 +620,98 @@ j_trace_enter(gchar const* name, gchar const* format, ...) G_UNLOCK(j_trace_echo); } + if (j_trace_flags & J_TRACE_ACCESS) + { + if (strcmp(name, "ping") == 0) + { + va_list args; + va_start(args, format); + g_free(trace_thread->client.program_name); + trace_thread->client.program_name = strdup(va_arg(args, const char*)); + trace_thread->client.uid = va_arg(args, guint32); + va_end(args); + } + else if (strncmp(name, "backend_", 8) == 0) + { + Access row; + va_list args; + memset(&row, 0, sizeof(Access)); + name += 8; + row.uid = trace_thread->client.uid; + row.program_name = trace_thread->client.program_name; + row.timestamp = timestamp; + if (strncmp(name, "kv_", 3) == 0) + { + name += 3; + row.backend = "kv"; + row.operation = name; + } + else if (strncmp(name, "db_", 3) == 0) + { + name += 3; + row.backend = "db"; + row.operation = name; + } + else if (strncmp(name, "object_", 7) == 0) + { + name += 7; + row.backend = "object"; + row.operation = name; + row.namespace = trace_thread->access.object.namespace; + row.name = trace_thread->access.object.path; + if (strcmp(name, "create") == 0) { + va_start(args, format); + row.namespace = va_arg(args, const char*); + row.name = va_arg(args, const char*); + va_end(args); + } else if (strcmp(name, "open") == 0) { + va_start(args, format); + row.namespace = va_arg(args, const char*); + row.name = va_arg(args, const char*); + va_end(args); + } else if (strcmp(name, "get_all") == 0) { + va_start(args, format); + row.namespace = va_arg(args, const char*); + row.name = NULL; + va_end(args); + } else if (strcmp(name, "get_by_prefix") == 0) { + va_start(args, format); + row.namespace = va_arg(args, const char*); + row.name = va_arg(args, const char*); + va_end(args); + } else if (strcmp(name, "read") == 0) { + va_start(args, format); + va_arg(args, void*); + va_arg(args, void*); + row.size = va_arg(args, guint64); + va_end(args); + } else if (strcmp(name, "write") == 0) { + va_start(args, format); + va_arg(args, void*); + va_arg(args, void*); + row.size = va_arg(args, guint64); + va_end(args); + } + // status, sync, iterate, fini, init <- no further data + + G_LOCK(j_trace_echo); + j_trace_access_print(&row); + G_UNLOCK(j_trace_echo); + + { + if (strcmp(name, "close") == 0 || strcmp(name, "delete") == 0) { + row.namespace = NULL; + row.name = NULL; + } + store_object_path(trace_thread, row.namespace, row.name); + } + } + else + { + } + } + } + #ifdef HAVE_OTF if (j_trace_flags & J_TRACE_OTF) { @@ -596,7 +758,6 @@ j_trace_enter(gchar const* name, gchar const* format, ...) g_array_append_val(trace_thread->stack, current_stack); } - va_end(args); trace_thread->function_depth++; diff --git a/lib/object/jobject.c b/lib/object/jobject.c index 6a0a290ff..21c9f84d7 100644 --- a/lib/object/jobject.c +++ b/lib/object/jobject.c @@ -595,7 +595,7 @@ static gboolean j_object_write_exec(JList* operations, JSemantics* semantics) { J_TRACE_FUNCTION(NULL); - + /// \todo check return value for messages gboolean ret = TRUE; diff --git a/server/loop.c b/server/loop.c index 427503e12..8dee68ab6 100644 --- a/server/loop.c +++ b/server/loop.c @@ -458,6 +458,8 @@ jd_handle_message(JMessage* message, GSocketConnection* connection, JMemoryChunk g_autoptr(JMessage) reply = NULL; gchar const* client_checksum; gchar const* server_checksum; + gchar const* client_program_name; + guint32 client_process_uid; guint num; num = g_atomic_int_add(&jd_thread_num, 1); @@ -465,6 +467,8 @@ jd_handle_message(JMessage* message, GSocketConnection* connection, JMemoryChunk //g_message("HELLO %d", num); client_checksum = j_message_get_string(message); + client_program_name = j_message_get_string(message); + client_process_uid = *(guint32*)j_message_get_n(message, sizeof(guint32)); server_checksum = j_configuration_get_checksum(jd_configuration); if (g_strcmp0(client_checksum, server_checksum) != 0) @@ -472,6 +476,10 @@ jd_handle_message(JMessage* message, GSocketConnection* connection, JMemoryChunk g_warning("Client %d uses different configuration than server.", num); } + { + J_TRACE("ping", "%s Thread: %u", client_program_name, client_process_uid); + } + reply = j_message_new_reply(message); j_message_append_string(reply, server_checksum); diff --git a/server/server.c b/server/server.c index d6428b45b..e2adb8695 100644 --- a/server/server.c +++ b/server/server.c @@ -28,6 +28,7 @@ #include #include #include +#include #include From 4644c0715b84f4bb1bf7e93f5e41379dd7a36a4f Mon Sep 17 00:00:00 2001 From: Julian Benda Date: Thu, 9 Feb 2023 15:24:22 +0100 Subject: [PATCH 02/40] Add traces for db and kv --- lib/core/jbackend.c | 4 +- lib/core/jtrace.c | 289 +++++++++++++++++++++++++++++++++++--------- 2 files changed, 235 insertions(+), 58 deletions(-) diff --git a/lib/core/jbackend.c b/lib/core/jbackend.c index 747ea56ff..63e34f5a8 100644 --- a/lib/core/jbackend.c +++ b/lib/core/jbackend.c @@ -820,7 +820,7 @@ j_backend_db_schema_create(JBackend* backend, gpointer batch, gchar const* name, g_return_val_if_fail(error == NULL || *error == NULL, FALSE); { - J_TRACE("backend_schema_create", "%p, %s, %u, %p, %p", batch, name, schema->len, (gconstpointer)schema, (gpointer)error); + J_TRACE("backend_db_schema_create", "%p, %s, %u, %p, %p", batch, name, schema->len, (gconstpointer)schema, (gpointer)error); ret = backend->db.backend_schema_create(backend->data, batch, name, schema, error); } @@ -841,7 +841,7 @@ j_backend_db_schema_get(JBackend* backend, gpointer batch, gchar const* name, bs g_return_val_if_fail(error == NULL || *error == NULL, FALSE); { - J_TRACE("backend_kv_schema_get", "%p, %s, %p, %p", batch, name, (gpointer)schema, (gpointer)error); + J_TRACE("backend_db_schema_get", "%p, %s, %p, %p", batch, name, (gpointer)schema, (gpointer)error); ret = backend->db.backend_schema_get(backend->data, batch, name, schema, error); } diff --git a/lib/core/jtrace.c b/lib/core/jtrace.c index 3e2991822..0a95f1a90 100644 --- a/lib/core/jtrace.c +++ b/lib/core/jtrace.c @@ -67,6 +67,19 @@ struct JTraceTime typedef struct JTraceTime JTraceTime; +struct Access { + guint64 timestamp; + guint32 uid; + const char* program_name; + const char* backend; + const char* namespace; + const char* name; + const char* operation; + guint64 size; + guint32 complexity; +}; +typedef struct Access Access; + /** * A trace thread. **/ @@ -105,8 +118,15 @@ struct JTraceThread } client; struct { + gboolean inside; + Access row; + const void* utility_ptr; + struct { + char* namespace; + } db; struct { char* namespace; + char* name; } kv; struct { char* namespace; @@ -115,6 +135,7 @@ struct JTraceThread } access; }; + typedef struct JTraceThread JTraceThread; struct JTrace @@ -154,6 +175,10 @@ static GHashTable* j_trace_summary_table = NULL; G_LOCK_DEFINE_STATIC(j_trace_echo); G_LOCK_DEFINE_STATIC(j_trace_summary); +static void +j_trace_access_print_header(void) { + g_printerr("time, process_uid, program_name, backend, namespace, name, operation, size, complexity, duration\n"); +} /** * Creates a new trace thread. * @@ -456,6 +481,13 @@ j_trace_init(gchar const* name) j_trace_summary_table = g_hash_table_new_full(g_str_hash, g_str_equal, g_free, g_free); } + if (j_trace_flags & J_TRACE_ACCESS) + { + G_LOCK(j_trace_echo); + j_trace_access_print_header(); + G_UNLOCK(j_trace_echo); + } + g_free(j_trace_name); j_trace_name = g_strdup(name); @@ -523,23 +555,12 @@ j_trace_fini(void) g_free(j_trace_name); } -struct Access { - guint64 timestamp; - guint32 uid; - const char* program_name; - const char* backend; - const char* namespace; - const char* name; - const char* operation; - guint64 size; - guint32 complexity; -}; -typedef struct Access Access; static void -j_trace_access_print(const Access* row) { - g_printerr("%"G_GUINT64_FORMAT", %u, %s, %s, %s, %s, %s, %"G_GUINT64_FORMAT", %u\n", - row->timestamp, +j_trace_access_print(const Access* row, guint64 duration) { + g_printerr("%"G_GUINT64_FORMAT".06%"G_GUINT64_FORMAT", %u, %s, %s, %s, %s, %s, %"G_GUINT64_FORMAT", %u, %"G_GUINT64_FORMAT".06%"G_GUINT64_FORMAT"\n", + row->timestamp / G_USEC_PER_SEC, + row->timestamp % G_USEC_PER_SEC, row->uid, row->program_name, row->backend, @@ -547,7 +568,9 @@ j_trace_access_print(const Access* row) { row->name, row->operation, row->size, - row->complexity + row->complexity, + duration / G_USEC_PER_SEC, + duration % G_USEC_PER_SEC ); } @@ -567,6 +590,30 @@ store_object_path(JTraceThread* trace_thread, const char* namespace, const char* } } +static void +store_kv_namespace(JTraceThread* trace_thread, const char* namespace, const char* name) +{ + if (trace_thread->access.kv.namespace != namespace) { + g_free(trace_thread->access.kv.namespace); + trace_thread->access.kv.namespace = NULL; + if(namespace) { trace_thread->access.kv.namespace = strdup(namespace); } + } + if (trace_thread->access.kv.name != name) { + g_free(trace_thread->access.kv.name); + trace_thread->access.kv.name = NULL; + if(name) { trace_thread->access.kv.name = strdup(name); } + } +} + +static void +store_db_namespace(JTraceThread* trace_thread, const char* namespace) +{ + if (trace_thread->access.db.namespace != namespace) { + g_free(trace_thread->access.db.namespace); + trace_thread->access.db.namespace = NULL; + if (namespace) { trace_thread->access.db.namespace = strdup(namespace); } + } +} JTrace* j_trace_enter(gchar const* name, gchar const* format, ...) @@ -633,82 +680,177 @@ j_trace_enter(gchar const* name, gchar const* format, ...) } else if (strncmp(name, "backend_", 8) == 0) { - Access row; - va_list args; - memset(&row, 0, sizeof(Access)); + int type = 0; name += 8; - row.uid = trace_thread->client.uid; - row.program_name = trace_thread->client.program_name; - row.timestamp = timestamp; - if (strncmp(name, "kv_", 3) == 0) + if (strncmp(name, "kv_", 3) == 0) { type = 1; } + else if (strncmp(name, "db_", 3) == 0) { type = 2; } + else if (strncmp(name, "object_", 7) == 0) { type = 3; } + if (type != 0) { + Access* row = &trace_thread->access.row; + va_list args; + memset(row, 0, sizeof(Access)); + row->uid = trace_thread->client.uid; + row->program_name = trace_thread->client.program_name; + row->timestamp = timestamp; + trace_thread->access.inside = TRUE; + + if (type == 1) { name += 3; - row.backend = "kv"; - row.operation = name; + row->backend = "kv"; + row->operation = name; + row->namespace = trace_thread->access.kv.namespace; + row->name = trace_thread->access.kv.name; + if (strcmp(name, "batch_start") == 0) + { + va_start(args, format); + row->namespace = va_arg(args, const char*); + va_end(args); + store_kv_namespace(trace_thread, row->namespace, NULL); + } else if (strcmp(name, "put") == 0) { + va_start(args, format); + va_arg(args, void*); + row->name = va_arg(args, const char*); + va_arg(args, void*); + row->size = va_arg(args, guint32); + va_end(args); + } else if (strcmp(name, "delete") == 0) { + va_start(args, format); + va_arg(args, void*); + row->name = va_arg(args, const char*); + va_end(args); + } else if (strcmp(name, "get") == 0){ + va_start(args, format); + va_arg(args, void*); + row->name = va_arg(args, const char*); + va_arg(args, void*); + trace_thread->access.utility_ptr = va_arg(args, void*); + va_end(args); + } else if (strcmp(name, "get_all") == 0) { + va_start(args, format); + row->namespace = va_arg(args, const char*); + va_end(args); + store_kv_namespace(trace_thread, row->namespace, NULL); + } else if (strcmp(name, "get_by_prefix") == 0) { + va_start(args, format); + row->namespace = va_arg(args, const char*); + row->name = va_arg(args, const char*); + va_end(args); + store_kv_namespace(trace_thread, row->namespace, row->name); + } + + // iterate, init, fini <- no further deatils + // batch_execute <- handlen in leave } - else if (strncmp(name, "db_", 3) == 0) + else if (type == 2) { name += 3; - row.backend = "db"; - row.operation = name; + row->backend = "db"; + row->operation = name; + row->namespace = trace_thread->access.db.namespace; + if (strcmp(name, "batch_start") == 0) { + va_start(args, format); + row->namespace = va_arg(args, const char*); + va_end(args); + store_db_namespace(trace_thread, row->namespace); + } else if (strcmp(name, "schema_create") == 0) { + va_start(args, format); + va_arg(args, void*); + row->name = va_arg(args, const char*); + row->size = va_arg(args, guint32); + va_end(args); + } else if (strcmp(name, "schema_get") == 0) { + va_start(args, format); + va_arg(args, void*); + row->name = va_arg(args, const char*); + va_end(args); + } else if (strcmp(name, "schema_delete") == 0) { + va_start(args, format); + va_arg(args, void*); + row->name = va_arg(args, const char*); + va_end(args); + } else if (strcmp(name, "insert") == 0) { + va_start(args, format); + va_arg(args, void*); + row->name = va_arg(args, const char*); + row->size = va_arg(args, guint32); + /// \TODO complexity from selector + va_end(args); + } else if (strcmp(name, "update") == 0) { + va_start(args, format); + va_arg(args, void*); + row->name = va_arg(args, const char*); + row->size = va_arg(args, guint32); + /// \TODO complexity from selector + va_end(args); + } else if (strcmp(name, "delete") == 0) { + va_start(args, format); + va_arg(args, void*); + row->name = va_arg(args, const char*); + /// \TODO complexity from selector + va_end(args); + } else if (strcmp(name, "query") == 0) { + va_start(args, format); + va_arg(args, void*); + row->name = va_arg(args, const char*); + /// \TODO complexvity from selector + va_end(args); + } + // iterate, init, fini <- no further details + // batch_execute <- handled in leave } - else if (strncmp(name, "object_", 7) == 0) + else if (type == 3) { name += 7; - row.backend = "object"; - row.operation = name; - row.namespace = trace_thread->access.object.namespace; - row.name = trace_thread->access.object.path; + row->backend = "object"; + row->operation = name; + row->namespace = trace_thread->access.object.namespace; + row->name = trace_thread->access.object.path; if (strcmp(name, "create") == 0) { va_start(args, format); - row.namespace = va_arg(args, const char*); - row.name = va_arg(args, const char*); + row->namespace = va_arg(args, const char*); + row->name = va_arg(args, const char*); va_end(args); + store_object_path(trace_thread, row->namespace, row->name); } else if (strcmp(name, "open") == 0) { va_start(args, format); - row.namespace = va_arg(args, const char*); - row.name = va_arg(args, const char*); + row->namespace = va_arg(args, const char*); + row->name = va_arg(args, const char*); va_end(args); + store_object_path(trace_thread, row->namespace, row->name); } else if (strcmp(name, "get_all") == 0) { va_start(args, format); - row.namespace = va_arg(args, const char*); - row.name = NULL; + row->namespace = va_arg(args, const char*); + row->name = NULL; va_end(args); + store_object_path(trace_thread, row->namespace, row->name); } else if (strcmp(name, "get_by_prefix") == 0) { va_start(args, format); - row.namespace = va_arg(args, const char*); - row.name = va_arg(args, const char*); + row->namespace = va_arg(args, const char*); + row->name = va_arg(args, const char*); va_end(args); + store_object_path(trace_thread, row->namespace, row->name); } else if (strcmp(name, "read") == 0) { va_start(args, format); va_arg(args, void*); va_arg(args, void*); - row.size = va_arg(args, guint64); + row->size = va_arg(args, guint64); va_end(args); } else if (strcmp(name, "write") == 0) { va_start(args, format); va_arg(args, void*); va_arg(args, void*); - row.size = va_arg(args, guint64); + row->size = va_arg(args, guint64); va_end(args); } // status, sync, iterate, fini, init <- no further data + // close/delte handle at leave - G_LOCK(j_trace_echo); - j_trace_access_print(&row); - G_UNLOCK(j_trace_echo); - - { - if (strcmp(name, "close") == 0 || strcmp(name, "delete") == 0) { - row.namespace = NULL; - row.name = NULL; - } - store_object_path(trace_thread, row.namespace, row.name); - } } - else - { + else { + /// \TODO handle unknown backends } + } } } @@ -809,6 +951,41 @@ j_trace_leave(JTrace* trace) G_UNLOCK(j_trace_echo); } + if (j_trace_flags & J_TRACE_ACCESS) + { + if (trace_thread->access.inside) { + if (strncmp(trace->name, "backend_", 8) == 0) { + if(strncmp(trace->name + 8, "db_", 3) == 0 || strncmp(trace->name+8, "kv_", 3) == 0 || strncmp(trace->name+8, "object_", 7)== 0) { + guint64 duration; + Access* row = &trace_thread->access.row; + + duration = timestamp - trace->enter_time; + + if (strcmp(trace->name, "backend_kv_get") == 0) + { + row->size = *(const guint64*)trace_thread->access.utility_ptr; + } + + G_LOCK(j_trace_echo); + j_trace_access_print(row, duration); + G_UNLOCK(j_trace_echo); + + trace_thread->access.inside = FALSE; + if(strcmp(trace->name, "backend_kv_batch_execute") == 0) { + store_kv_namespace(trace_thread, NULL, NULL); + } + if (strcmp(trace->name, "backend_db_batch_execute") == 0) { + store_db_namespace(trace_thread, NULL); + } + else if (strcmp(trace->name, "backend_object_delete") == 0 || strcmp(trace->name, "backend_object_close") == 0 ) + { + store_object_path(trace_thread, NULL, NULL); + } + } + } + } + } + #ifdef HAVE_OTF if (j_trace_flags & J_TRACE_OTF) { From e811c4c0e779969f5a03724012b0425924f8b947 Mon Sep 17 00:00:00 2001 From: Julian Benda Date: Thu, 9 Feb 2023 16:28:59 +0100 Subject: [PATCH 03/40] Fix for unknown program name --- lib/core/jconnection-pool.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/core/jconnection-pool.c b/lib/core/jconnection-pool.c index 2f41e7c57..947d729d4 100644 --- a/lib/core/jconnection-pool.c +++ b/lib/core/jconnection-pool.c @@ -169,6 +169,7 @@ j_connection_pool_fini(void) g_slice_free(JConnectionPool, pool); } +extern const char *__progname; static GSocketConnection* j_connection_pool_pop_internal(GAsyncQueue* queue, guint* count, gchar const* server) { @@ -220,6 +221,8 @@ j_connection_pool_pop_internal(GAsyncQueue* queue, guint* count, gchar const* se client_checksum = j_configuration_get_checksum(j_configuration()); client_program_name = g_get_prgname(); + if (client_program_name == NULL) { client_program_name = __progname; } + if (client_program_name == NULL) { client_program_name = ""; } client_process_uid = j_configuration_get_uid(j_configuration()); message = j_message_new(J_MESSAGE_PING, strlen(client_checksum) + 1); From b7ba0309afcf9914240c6a197154d65644aab95f Mon Sep 17 00:00:00 2001 From: Julian Benda Date: Fri, 10 Feb 2023 15:33:46 +0100 Subject: [PATCH 04/40] sceletcton of replay tool --- lib/core/jtrace.c | 2 +- meson.build | 6 + tools/access_replay.c | 310 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 317 insertions(+), 1 deletion(-) create mode 100644 tools/access_replay.c diff --git a/lib/core/jtrace.c b/lib/core/jtrace.c index 0a95f1a90..2b2afc274 100644 --- a/lib/core/jtrace.c +++ b/lib/core/jtrace.c @@ -558,7 +558,7 @@ j_trace_fini(void) static void j_trace_access_print(const Access* row, guint64 duration) { - g_printerr("%"G_GUINT64_FORMAT".06%"G_GUINT64_FORMAT", %u, %s, %s, %s, %s, %s, %"G_GUINT64_FORMAT", %u, %"G_GUINT64_FORMAT".06%"G_GUINT64_FORMAT"\n", + g_printerr("%"G_GUINT64_FORMAT".%06"G_GUINT64_FORMAT", %u, %s, %s, %s, %s, %s, %"G_GUINT64_FORMAT", %u, %"G_GUINT64_FORMAT".%06"G_GUINT64_FORMAT"\n", row->timestamp / G_USEC_PER_SEC, row->timestamp % G_USEC_PER_SEC, row->uid, diff --git a/meson.build b/meson.build index 58bbd62b0..d0cc51f51 100644 --- a/meson.build +++ b/meson.build @@ -723,6 +723,12 @@ executable('julea-statistics', 'tools/statistics.c', install: true, ) +executable('julea-access-replay', 'tools/access_replay.c', + dependencies: common_deps + [julea_dep], + include_directories: julea_incs, + install: ture, +) + if hdf_dep.found() executable('julea-h5migrate', 'tools/h5migrate.c', dependencies: common_deps + [julea_dep] + [hdf_dep], diff --git a/tools/access_replay.c b/tools/access_replay.c new file mode 100644 index 000000000..6c98bac8d --- /dev/null +++ b/tools/access_replay.c @@ -0,0 +1,310 @@ +/* + * JULEA - Flexible storage framework + * Copyright (C) 2023-2023 Julian Benda + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + */ + +#include + +#include +#include +#include +#include +#include + +#include + +JBackend* object_backend = NULL; +JBackend* kv_backend = NULL; +JBackend* db_backend = NULL; + +static gboolean +setup_backend(JConfiguration* configuration, JBackendType type, gchar const* port_str, GModule** module, JBackend** j_backend) +{ + gchar const* backend; + gchar const* component; + g_autofree gchar* path; + + backend = j_configuration_get_backend(configuration, type); + component = j_configuration_get_backend_component(configuration, type); + path = j_helper_str_replace(j_configuration_get_backend_path(configuration, type), "{PORT}", port_str); + + if(j_backend_load_server(backend, component, type, module, j_backend)) { + gboolean res = TRUE; + if(j_backend == NULL) { + g_warning("Failed to load server: %s.", backend); + return FALSE; + } + switch (type) { + case J_BACKEND_TYPE_OBJECT: res = j_backend_object_init(*j_backend, path); break; + case J_BACKEND_TYPE_KV: res = j_backend_kv_init(*j_backend, path); break; + case J_BACKEND_TYPE_DB: res = j_backend_db_init(*j_backend, path); break; + } + if (!res) { + g_warning("Failed to initelize backend: %s", backend); + return FALSE; + } + return TRUE; + } + return FALSE; +} + +static gboolean +split(char* line, const char* parts[9]) { + int cnt = 0; + const char* section; + section = strtok(line, ","); + while(section != NULL) { + if (cnt == 9) { + g_warning("to many parts in line"); + return FALSE; + } + parts[cnt++] = section; + section = strtok(NULL, ","); + } + if (cnt != 9) { + g_warning("to few parts in line"); + return FALSE; + } + return TRUE; +} +#define BACKEND 3 +#define NAMESPACE 4 +#define NAME 5 +#define OPERATION 6 +#define SIZE 7 + + +static gboolean +replay_object(JMemoryChunk* memory_chunk, guint64 memory_chunk_size, char* parts[9]){ + static gpointer data = NULL; + const char* op = parts[OPERATION]; + gboolean ret = FALSE; + (void) memory_chunk_size; /// \TODO usage? + if(strcmp(op, "create") == 0) { + if (data != NULL) { g_warning("open new object while still one open"); } + ret = j_backend_object_create(object_backend, parts[NAMESPACE], parts[NAME], &data); + } else if (strcmp(op, "open") == 0) { + if (data != NULL) { g_warning("open new object while still one open"); } + ret = j_backend_object_open(object_backend, parts[NAMESPACE], parts[NAME], &data); + } else if (strcmp(op, "get_all") == 0) { + if (data != NULL) { g_warning("open new object while still one open"); } + ret = j_backend_object_get_all(object_backend, parts[NAMESPACE], data); + } else if (strcmp(op, "ge_by_prefix") == 0) { + if (data != NULL) { g_warning("open new object while still one open"); } + ret = j_backend_kv_get_by_prefix(object_backend, parts[NAMESPACE], parts[NAME], &data); + } else if (strcmp(op, "read") == 0) { + static guint64 bytes_read = 0; + guint64 size = strtoul(parts[SIZE], NULL, 10); + if (size > memory_chunk_size) { + g_warning("unable to replay: chunk size to samll!%lu vs %lu", size, memory_chunk_size); + } else { + ret = j_backend_object_read(object_backend, data, memory_chunk, size, 0, &bytes_read); + } + } else if (strcmp(op, "write") == 0) { + static guint64 bytes_written = 0; + guint64 size = strtoul(parts[SIZE], NULL, 10); + if (size > memory_chunk_size) { + g_warning("unable to replay: chunk size to samll!%lu vs %lu", size, memory_chunk_size); + } else { + ret = j_backend_object_write(object_backend, data, memory_chunk, size, 0, &bytes_written); + } + } else if (strcmp(op, "status") == 0) { + static gint64 modification_time; + static guint64 size; + ret = j_backend_object_status(object_backend, data, &modification_time, &size); + } else if (strcmp(op, "sync") == 0) { + ret = j_backend_object_sync(object_backend, data); + } else if (strcmp(op, "iterate") == 0) { + static const char* name = NULL; + ret = j_backend_object_iterate(object_backend, data, &name); + if (!ret) { data = NULL; ret = TRUE; } + } else if (strcmp(op, "close") == 0) { + ret = j_backend_object_close(object_backend, data); + data = NULL; + } else if (strcmp(op, "delete") == 0) { + ret = j_backend_object_delete(object_backend, data); + data = NULL; + } else if (strcmp(op, "init") == 0) { + ret = TRUE; + } else if (strcmp(op, "fini") == 0) { + ret = TRUE; + } else { + g_warning("unkown object operation: '%s'", op); + } + return ret; +} +static gboolean +replay_kv(JMemoryChunk* memory_chunk, guint64 memory_chunk_size, char* parts[9]){ + gboolean ret = FALSE; + const char* op = parts[OPERATION]; + + if (strcmp(op, "batch_start") == 0) { + } else if (strcmp(op, "put") == 0) { + } else if (strcmp(op, "delete") == 0) { + } else if (strcmp(op, "get") == 0) { + } else if (strcmp(op, "get_all") == 0) { + } else if (strcmp(op, "get_by_prefix") == 0) { + } else if (strcmp(op, "iterate") == 0) { + } else if (strcmp(op, "init") == 0) { + } else if (strcmp(op, "fini") == 0) { + } else if (strcmp(op, "batch_execute") == 0) { + } else { + g_warning("unkown operation: '%s'", op); + } + return ret; +} +static gboolean +replay_db(JMemoryChunk* memory_chunk, guint64 memory_chunk_size, char* parts[9]){ + gboolean ret = FALSE; + static gpointer batch; + /// \TODO how to determine semantics used? + static JSemantics* semantics = NULL; + GError* error; + const char* op = parts[OPERATION]; + if (semantics == NULL) { semantics = j_semantics_new(J_SEMANTICS_TEMPLATE_DEFAULT); } + + if (strcmp(op, "batch_start") == 0) { + if (batch != NULL) { g_warning("starting a new batch, but old one is not finished"); } + ret = j_backend_db_batch_start(db_backend, parts[NAMESPACE], semantics, &batch, &error); + } else if (strcmp(op, "schema_create") == 0) { + bson_t* schema; + ret = j_backend_db_schema_create(db_backend, batch, parts[NAME], schema, &error); + } else if (strcmp(op, "schema_get") == 0) { + } else if (strcmp(op, "schema_delete") == 0) { + } else if (strcmp(op, "insert") == 0) { + } else if (strcmp(op, "delete") == 0) { + } else if (strcmp(op, "update") == 0) { + } else if (strcmp(op, "delete") == 0) { + } else if (strcmp(op, "query") == 0) { + } else if (strcmp(op, "iterate") == 0) { + static bson_t* entry; + ret = j_backend_db_iterate(db_backend, batch, entry, &error); + if(!ret) { batch = NULL; ret = TRUE; } + } else if (strcmp(op, "init") == 0) { + ret = TRUE; + } else if (strcmp(op, "fini") == 0) { + ret = TRUE; + } else if (strcmp(op, "batch_execute") == 0) { + j_backend_db_batch_execute(db_backend, batch, &error); + batch = NULL; + } else { + g_warning("unknown operation: '%s'", op); + } + return ret && error == NULL; +} + +static gboolean +replay(JMemoryChunk* memory_chunk, guint64 memory_chunck_size, char* line) +{ + const char* parts[9]; + if (!split(line, parts)) { return FALSE; } + + if(strcmp(parts[3], " object") == 0) { if(!replay_object(memory_chunk, memory_chunck_size, parts)) return FALSE; } + if(strcmp(parts[3], " kv") == 0) { if(!replay_kv(memory_chunk, memory_chunck_size, parts)) return FALSE; } + if(strcmp(parts[3], " db") == 0) { if(!replay_db(memory_chunk, memory_chunck_size, parts)) return FALSE; } + + return TRUE; +} + +int +main(int argc, char** argv) +{ + JConfiguration* configuration = NULL; + const char* record_file_name = NULL; + FILE* record_file = NULL; + JTrace* trace = NULL; + GModule* db_module = NULL; + GModule* kv_module = NULL; + GModule* object_module = NULL; + g_autofree gchar* port_str = NULL; + gboolean res = FALSE; + JMemoryChunk* memory_chunk = NULL; + guint64 memory_chunck_size = 0; + + setlocal(LC_ALL, "C.UTF-8"); + + configuration = j_configuration(); + if (configuration == NULL) { + g_warning("unable to read config"); + goto end; + if (argc < 2) { + g_warning("useage: %s ", argv[0]); + goto end; + } + } + record_file_name = argv[1]; + if (access(record_file_name, R_OK) != 0) { + g_warning("file does not exists, or no read permission '%s'", record_file_name); + goto end; + } + record_file = fopen(record_file_name, "r"); + if (record_file == NULL) { + g_warning("failed to open file '%s'", record_file_name); + goto end; + } + + j_trace_init("julea-replay"); + trace = j_trace_enter(G_STRFUNC, NULL); + + port_str = g_strdup_printf("%d", j_configuration_get_port(configuration)); + if(!setup_backend(configuration, J_BACKEND_TYPE_OBJECT, port_str, &object_module, &object_backend)) { goto end; } + if(!setup_backend(configuration, J_BACKEND_TYPE_KV, port_str, &kv_module, &kv_backend)) { goto end; } + if(!setup_backend(configuration, J_BACKEND_TYPE_DB, port_str, &db_module, &db_backend)) { goto end; } + + memory_chunck_size = j_configuration_get_max_operation_size(configuration); + memory_chunk = j_memory_chunk_new(memory_chunck_size); + + { + char* line = NULL; + size_t len = 0; + ssize_t read = 0; + read = getline(&line, &len, record_file); + if (read == -1 || strcmp(line, "time, process_uid, program_name, backend, namespace, name, operation, size, complexity, duration") != 0) { + g_warning("Invalid header in dump!"); + goto end; + } + while ((read = getline(&line, &len, record_file)) != -1) { + if (!replay(memory_chunk, memory_chunck_size, line)) { + g_warning("failed to replay dump!"); + goto end; + } + } + } + res = TRUE; +end: + if (record_file) { + fclose(record_file); + } + if (db_backend != NULL) + { + j_backend_db_fini(db_backend); + } + if (kv_backend != NULL) + { + j_backend_kv_fini(kv_backend); + } + if(object_backend != NULL) + { + j_backend_db_fini(db_backend); + } + if(db_module != NULL) { g_module_close(db_module); } + if(kv_module != NULL) { g_module_close(kv_module); } + if(object_module != NULL) { g_module_close(object_module); } + j_configuration_unref(configuration); + j_tarce_leave(trace); + j_trace_fini(); + return res; +} \ No newline at end of file From f7d227463e2155550ac50699204bdedfbc68b4cf Mon Sep 17 00:00:00 2001 From: Julian Benda Date: Fri, 10 Feb 2023 22:20:17 +0100 Subject: [PATCH 05/40] Working replay, but with segfault --- lib/core/jtrace.c | 34 ++++++-- lib/db-util/jbson.c | 8 +- meson.build | 2 +- tools/access_replay.c | 189 ++++++++++++++++++++++++++++++++---------- 4 files changed, 179 insertions(+), 54 deletions(-) diff --git a/lib/core/jtrace.c b/lib/core/jtrace.c index 2b2afc274..027e71539 100644 --- a/lib/core/jtrace.c +++ b/lib/core/jtrace.c @@ -31,6 +31,8 @@ #include +#include + /** * \addtogroup JTrace * @@ -77,6 +79,7 @@ struct Access { const char* operation; guint64 size; guint32 complexity; + const bson_t* bson; }; typedef struct Access Access; @@ -177,7 +180,7 @@ G_LOCK_DEFINE_STATIC(j_trace_summary); static void j_trace_access_print_header(void) { - g_printerr("time, process_uid, program_name, backend, namespace, name, operation, size, complexity, duration\n"); + g_printerr("time,process_uid,program_name,backend,namespace,name,operation,size,complexity,duration,bson\n"); } /** * Creates a new trace thread. @@ -202,6 +205,7 @@ j_trace_thread_new(GThread* thread) } trace_thread = g_slice_new(JTraceThread); + memset(trace_thread, 0, sizeof(JTraceThread)); trace_thread->function_depth = 0; trace_thread->stack = g_array_new(FALSE, FALSE, sizeof(JTraceStack)); trace_thread->client.program_name = NULL; @@ -558,7 +562,7 @@ j_trace_fini(void) static void j_trace_access_print(const Access* row, guint64 duration) { - g_printerr("%"G_GUINT64_FORMAT".%06"G_GUINT64_FORMAT", %u, %s, %s, %s, %s, %s, %"G_GUINT64_FORMAT", %u, %"G_GUINT64_FORMAT".%06"G_GUINT64_FORMAT"\n", + g_printerr("%"G_GUINT64_FORMAT".%06"G_GUINT64_FORMAT",%u,%s,%s,%s,%s,%s,%"G_GUINT64_FORMAT",%u,%"G_GUINT64_FORMAT".%06"G_GUINT64_FORMAT",\"%s\"\n", row->timestamp / G_USEC_PER_SEC, row->timestamp % G_USEC_PER_SEC, row->uid, @@ -570,7 +574,8 @@ j_trace_access_print(const Access* row, guint64 duration) { row->size, row->complexity, duration / G_USEC_PER_SEC, - duration % G_USEC_PER_SEC + duration % G_USEC_PER_SEC, + row->bson == NULL ? "{}" : bson_as_json(row->bson, NULL) ); } @@ -758,6 +763,7 @@ j_trace_enter(gchar const* name, gchar const* format, ...) va_arg(args, void*); row->name = va_arg(args, const char*); row->size = va_arg(args, guint32); + row->bson = va_arg(args, const bson_t*); va_end(args); } else if (strcmp(name, "schema_get") == 0) { va_start(args, format); @@ -774,25 +780,35 @@ j_trace_enter(gchar const* name, gchar const* format, ...) va_arg(args, void*); row->name = va_arg(args, const char*); row->size = va_arg(args, guint32); + row->bson = va_arg(args, const bson_t*); /// \TODO complexity from selector va_end(args); } else if (strcmp(name, "update") == 0) { + static bson_t bson; + bson_init(&bson); va_start(args, format); va_arg(args, void*); row->name = va_arg(args, const char*); row->size = va_arg(args, guint32); + bson_append_document(&bson, "selector", -1, va_arg(args, const bson_t*)); + bson_append_document(&bson, "entry", -1, va_arg(args, const bson_t*)); + row->bson = &bson; /// \TODO complexity from selector va_end(args); } else if (strcmp(name, "delete") == 0) { va_start(args, format); va_arg(args, void*); row->name = va_arg(args, const char*); + va_arg(args, guint32); + row->bson = va_arg(args, const bson_t*); /// \TODO complexity from selector va_end(args); } else if (strcmp(name, "query") == 0) { va_start(args, format); va_arg(args, void*); row->name = va_arg(args, const char*); + va_arg(args, guint32); + row->bson = va_arg(args, const bson_t*); /// \TODO complexvity from selector va_end(args); } @@ -835,12 +851,14 @@ j_trace_enter(gchar const* name, gchar const* format, ...) va_arg(args, void*); va_arg(args, void*); row->size = va_arg(args, guint64); + row->complexity = va_arg(args, guint64); va_end(args); } else if (strcmp(name, "write") == 0) { va_start(args, format); va_arg(args, void*); va_arg(args, void*); row->size = va_arg(args, guint64); + row->complexity = va_arg(args, guint64); va_end(args); } // status, sync, iterate, fini, init <- no further data @@ -973,16 +991,16 @@ j_trace_leave(JTrace* trace) trace_thread->access.inside = FALSE; if(strcmp(trace->name, "backend_kv_batch_execute") == 0) { store_kv_namespace(trace_thread, NULL, NULL); - } - if (strcmp(trace->name, "backend_db_batch_execute") == 0) { + } else if (strcmp(trace->name, "backend_dbupdate") == 0){ + bson_destroy((bson_t*)row->bson); + } else if (strcmp(trace->name, "backend_db_batch_execute") == 0) { store_db_namespace(trace_thread, NULL); - } - else if (strcmp(trace->name, "backend_object_delete") == 0 || strcmp(trace->name, "backend_object_close") == 0 ) + } else if (strcmp(trace->name, "backend_object_delete") == 0 || strcmp(trace->name, "backend_object_close") == 0 ) { store_object_path(trace_thread, NULL, NULL); } - } } + } } } diff --git a/lib/db-util/jbson.c b/lib/db-util/jbson.c index 7b156c7d0..45627e6cc 100644 --- a/lib/db-util/jbson.c +++ b/lib/db-util/jbson.c @@ -341,7 +341,7 @@ j_bson_iter_value(bson_iter_t* iter, JDBType type, JDBTypeValue* value, GError** break; case J_DB_TYPE_SINT64: - if (G_UNLIKELY(!BSON_ITER_HOLDS_INT64(iter))) + if (G_UNLIKELY(!BSON_ITER_HOLDS_INT64(iter) && !BSON_ITER_HOLDS_INT32(iter))) { g_set_error_literal(error, J_BACKEND_BSON_ERROR, J_BACKEND_BSON_ERROR_ITER_INVALID_TYPE, "bson iter invalid type"); goto _error; @@ -349,7 +349,11 @@ j_bson_iter_value(bson_iter_t* iter, JDBType type, JDBTypeValue* value, GError** if (value) { - value->val_sint64 = bson_iter_int64(iter); + if(BSON_ITER_HOLDS_INT64(iter)) { + value->val_sint64 = bson_iter_int64(iter); + } else { + value->val_sint64 = bson_iter_int32(iter); + } } break; diff --git a/meson.build b/meson.build index d0cc51f51..c2158eb6a 100644 --- a/meson.build +++ b/meson.build @@ -726,7 +726,7 @@ executable('julea-statistics', 'tools/statistics.c', executable('julea-access-replay', 'tools/access_replay.c', dependencies: common_deps + [julea_dep], include_directories: julea_incs, - install: ture, + install: true, ) if hdf_dep.found() diff --git a/tools/access_replay.c b/tools/access_replay.c index 6c98bac8d..b943a0ecd 100644 --- a/tools/access_replay.c +++ b/tools/access_replay.c @@ -61,34 +61,47 @@ setup_backend(JConfiguration* configuration, JBackendType type, gchar const* por return FALSE; } +#define BACKEND 3 +#define NAMESPACE 4 +#define NAME 5 +#define OPERATION 6 +#define SIZE 7 +#define COMPLEXITY 8 +#define DURATION 9 +#define JSON 10 + static gboolean -split(char* line, const char* parts[9]) { +split(char* line, const char* parts[11]) { int cnt = 0; const char* section; section = strtok(line, ","); while(section != NULL) { - if (cnt == 9) { - g_warning("to many parts in line"); - return FALSE; - } parts[cnt++] = section; + if (cnt == 10) { break; } section = strtok(NULL, ","); } - if (cnt != 9) { + while(*++section); + parts[cnt] = section+1; + if (cnt != 10) { g_warning("to few parts in line"); return FALSE; } + parts[JSON] += 1; + { + char* itr = parts[JSON]; + while(*++itr); + itr[-1] = 0; + } return TRUE; } -#define BACKEND 3 -#define NAMESPACE 4 -#define NAME 5 -#define OPERATION 6 -#define SIZE 7 +static guint64 +parse_ul(const char* str) { + return strtoul(str, NULL, 10); +} static gboolean -replay_object(JMemoryChunk* memory_chunk, guint64 memory_chunk_size, char* parts[9]){ +replay_object(void* memory_chunk, guint64 memory_chunk_size, const char* parts[11]){ static gpointer data = NULL; const char* op = parts[OPERATION]; gboolean ret = FALSE; @@ -107,20 +120,20 @@ replay_object(JMemoryChunk* memory_chunk, guint64 memory_chunk_size, char* parts ret = j_backend_kv_get_by_prefix(object_backend, parts[NAMESPACE], parts[NAME], &data); } else if (strcmp(op, "read") == 0) { static guint64 bytes_read = 0; - guint64 size = strtoul(parts[SIZE], NULL, 10); + guint64 size = parse_ul(parts[SIZE]); if (size > memory_chunk_size) { g_warning("unable to replay: chunk size to samll!%lu vs %lu", size, memory_chunk_size); } else { - ret = j_backend_object_read(object_backend, data, memory_chunk, size, 0, &bytes_read); + ret = j_backend_object_read(object_backend, data, memory_chunk, size, parse_ul(parts[COMPLEXITY]), &bytes_read); } } else if (strcmp(op, "write") == 0) { static guint64 bytes_written = 0; - guint64 size = strtoul(parts[SIZE], NULL, 10); + guint64 size = parse_ul(parts[SIZE]); if (size > memory_chunk_size) { g_warning("unable to replay: chunk size to samll!%lu vs %lu", size, memory_chunk_size); } else { - ret = j_backend_object_write(object_backend, data, memory_chunk, size, 0, &bytes_written); - } + ret = j_backend_object_write(object_backend, data, memory_chunk, size, parse_ul(parts[COMPLEXITY]), &bytes_written); + } } else if (strcmp(op, "status") == 0) { static gint64 modification_time; static guint64 size; @@ -147,74 +160,159 @@ replay_object(JMemoryChunk* memory_chunk, guint64 memory_chunk_size, char* parts return ret; } static gboolean -replay_kv(JMemoryChunk* memory_chunk, guint64 memory_chunk_size, char* parts[9]){ +replay_kv(void* memory_chunk, guint64 memory_chunk_size, const char* parts[11]){ gboolean ret = FALSE; const char* op = parts[OPERATION]; + gpointer batch = NULL; + static JSemantics* semantics = NULL; + if (semantics == NULL) { semantics = j_semantics_new(J_SEMANTICS_TEMPLATE_DEFAULT); } if (strcmp(op, "batch_start") == 0) { + if (batch != NULL) { g_warning("starting a new batch, but old one is not finished"); } + ret = j_backend_kv_batch_start(kv_backend, parts[NAMESPACE], semantics, &batch); } else if (strcmp(op, "put") == 0) { + ret = j_backend_kv_put(kv_backend, batch, parts[NAME], memory_chunk, parse_ul(parts[SIZE])); } else if (strcmp(op, "delete") == 0) { + ret = j_backend_kv_delete(kv_backend, batch, parts[NAME]); } else if (strcmp(op, "get") == 0) { + guint32 size; + gpointer value; + ret = j_backend_kv_get(kv_backend, batch, parts[NAME], &value, &size); } else if (strcmp(op, "get_all") == 0) { + if (batch != NULL) { g_warning("start iterating with remaining batch or iteration"); } + ret = j_backend_kv_get_all(kv_backend, parts[NAMESPACE], &batch); } else if (strcmp(op, "get_by_prefix") == 0) { + if (batch != NULL) { g_warning("start iterating with remaining batch or iteration"); } + ret = j_backend_kv_get_by_prefix(kv_backend, parts[NAMESPACE], parts[NAME], &batch); } else if (strcmp(op, "iterate") == 0) { + const char* name; + gconstpointer value; + guint32 len; + ret = j_backend_kv_iterate(kv_backend, batch, &name, &value, &len); + if (!ret) { batch = NULL; ret = TRUE;} } else if (strcmp(op, "init") == 0) { + ret = TRUE; } else if (strcmp(op, "fini") == 0) { + ret = TRUE; } else if (strcmp(op, "batch_execute") == 0) { + ret = j_backend_kv_batch_execute(kv_backend, batch); } else { g_warning("unkown operation: '%s'", op); } return ret; } +struct JSqlBatch +{ + const gchar* namespace; + JSemantics* semantics; + gboolean open; + gboolean aborted; +}; static gboolean -replay_db(JMemoryChunk* memory_chunk, guint64 memory_chunk_size, char* parts[9]){ +replay_db(void* memory_chunk, guint64 memory_chunk_size, const char* parts[11]){ gboolean ret = FALSE; - static gpointer batch; + static gpointer batch = NULL; + static gpointer itr = NULL; + static gchar* namespace = NULL; /// \TODO how to determine semantics used? static JSemantics* semantics = NULL; - GError* error; + GError* error = NULL; const char* op = parts[OPERATION]; + static GArray* bsons = NULL; + if (bsons == NULL) { bsons = g_array_sized_new(FALSE, TRUE, sizeof(bson_t*), 0); } if (semantics == NULL) { semantics = j_semantics_new(J_SEMANTICS_TEMPLATE_DEFAULT); } if (strcmp(op, "batch_start") == 0) { if (batch != NULL) { g_warning("starting a new batch, but old one is not finished"); } - ret = j_backend_db_batch_start(db_backend, parts[NAMESPACE], semantics, &batch, &error); + namespace = strdup(parts[NAMESPACE]); + ret = j_backend_db_batch_start(db_backend, namespace, semantics, &batch, &error); } else if (strcmp(op, "schema_create") == 0) { - bson_t* schema; + bson_t* schema = bson_new_from_json(parts[JSON], -1, NULL); ret = j_backend_db_schema_create(db_backend, batch, parts[NAME], schema, &error); + g_array_append_val(bsons, schema); } else if (strcmp(op, "schema_get") == 0) { + bson_t* schema = bson_new(); + ret = j_backend_db_schema_get(db_backend, batch, parts[NAME], schema, &error); + g_array_append_val(bsons, schema); + // schema do not exists + if (error && error->code == 8) { + g_error_free(error); + error = NULL; + ret = TRUE; + } } else if (strcmp(op, "schema_delete") == 0) { + ret = j_backend_db_schema_delete(db_backend, batch, parts[NAME], &error); } else if (strcmp(op, "insert") == 0) { + bson_t* entry = bson_new_from_json(parts[JSON], -1, NULL); + bson_t* res = bson_new(); + ret = j_backend_db_insert(db_backend, batch, parts[NAME], entry, res, &error); + g_array_append_val(bsons, entry); + g_array_append_val(bsons, res); } else if (strcmp(op, "delete") == 0) { + bson_t* entry = bson_new_from_json(parts[JSON], -1, NULL); + ret = j_backend_db_delete(db_backend, batch, parts[NAME], entry, &error); + g_array_append_val(bsons, entry); } else if (strcmp(op, "update") == 0) { - } else if (strcmp(op, "delete") == 0) { + bson_iter_t bson_iter; + bson_t* selector; + bson_t* entry; + const uint8_t* doc; + uint32_t len; + bson_t* bson = bson_new_from_json(parts[JSON], -1, NULL); + bson_iter_init_find(&bson_iter, bson, "entry"); + bson_iter_document(&bson_iter, &len, &doc); + entry = bson_new_from_data(doc, len); + bson_iter_init_find(&bson_iter, bson, "selector"); + bson_iter_document(&bson_iter, &len, &doc); + selector = bson_new_from_data(doc, len); + ret = j_backend_db_update(db_backend, batch, parts[NAME], selector, entry, &error); + g_array_append_val(bsons, entry); + g_array_append_val(bsons, selector); + g_array_append_val(bsons, bson); } else if (strcmp(op, "query") == 0) { + bson_t* selector = bson_new_from_json(parts[JSON], -1, NULL); + if(itr != NULL) { g_warning("start new db iteration without finishing old one!"); } + ret = j_backend_db_query(db_backend, batch, parts[NAME], selector, &itr, &error); + g_array_append_val(bsons, selector); } else if (strcmp(op, "iterate") == 0) { - static bson_t* entry; - ret = j_backend_db_iterate(db_backend, batch, entry, &error); - if(!ret) { batch = NULL; ret = TRUE; } + bson_t* entry = bson_new(); + ret = j_backend_db_iterate(db_backend, itr, entry, &error); + g_array_append_val(bsons, entry); + if(!ret) { itr = NULL; ret = TRUE; g_error_free(error); error = NULL; } } else if (strcmp(op, "init") == 0) { ret = TRUE; } else if (strcmp(op, "fini") == 0) { ret = TRUE; } else if (strcmp(op, "batch_execute") == 0) { - j_backend_db_batch_execute(db_backend, batch, &error); + ret = j_backend_db_batch_execute(db_backend, batch, &error); batch = NULL; + for(guint i = 0; i < bsons->len; ++i) { + bson_destroy(g_array_index(bsons, bson_t*, i)); + } + g_array_remove_range(bsons, 0, bsons->len); + // no operation to do + if (error && error->code == 7) { + g_error_free(error); + error = NULL; + ret = TRUE; + } } else { g_warning("unknown operation: '%s'", op); } + if (error) { + g_warning("db error(%d): %s", error->code, error->message); + } return ret && error == NULL; } static gboolean -replay(JMemoryChunk* memory_chunk, guint64 memory_chunck_size, char* line) +replay(void* memory_chunk, guint64 memory_chunck_size, char* line) { - const char* parts[9]; + const char* parts[11]; if (!split(line, parts)) { return FALSE; } - - if(strcmp(parts[3], " object") == 0) { if(!replay_object(memory_chunk, memory_chunck_size, parts)) return FALSE; } - if(strcmp(parts[3], " kv") == 0) { if(!replay_kv(memory_chunk, memory_chunck_size, parts)) return FALSE; } - if(strcmp(parts[3], " db") == 0) { if(!replay_db(memory_chunk, memory_chunck_size, parts)) return FALSE; } + if(strcmp(parts[BACKEND], "object") == 0) { if(!replay_object(memory_chunk, memory_chunck_size, parts)) return FALSE; } + if(strcmp(parts[BACKEND], "kv") == 0) { if(!replay_kv(memory_chunk, memory_chunck_size, parts)) return FALSE; } + if(strcmp(parts[BACKEND], "db") == 0) { if(!replay_db(memory_chunk, memory_chunck_size, parts)) return FALSE; } return TRUE; } @@ -234,7 +332,7 @@ main(int argc, char** argv) JMemoryChunk* memory_chunk = NULL; guint64 memory_chunck_size = 0; - setlocal(LC_ALL, "C.UTF-8"); + setlocale(LC_ALL, "C.UTF-8"); configuration = j_configuration(); if (configuration == NULL) { @@ -256,8 +354,8 @@ main(int argc, char** argv) goto end; } - j_trace_init("julea-replay"); - trace = j_trace_enter(G_STRFUNC, NULL); + // j_trace_init("julea-replay"); + // trace = j_trace_enter(G_STRFUNC, NULL); port_str = g_strdup_printf("%d", j_configuration_get_port(configuration)); if(!setup_backend(configuration, J_BACKEND_TYPE_OBJECT, port_str, &object_module, &object_backend)) { goto end; } @@ -268,16 +366,21 @@ main(int argc, char** argv) memory_chunk = j_memory_chunk_new(memory_chunck_size); { + const char* header = "time,process_uid,program_name,backend,namespace,name,operation,size,complexity,duration,bson"; char* line = NULL; size_t len = 0; ssize_t read = 0; + void* chunk_data = *((void**)((char*)memory_chunk+8)); + memset(chunk_data, 0, memory_chunck_size); read = getline(&line, &len, record_file); - if (read == -1 || strcmp(line, "time, process_uid, program_name, backend, namespace, name, operation, size, complexity, duration") != 0) { - g_warning("Invalid header in dump!"); + line[read-1] = 0; + if (read == -1 || strcmp(line, header) != 0) { + g_warning("Invalid header in dump!\n'%s'\n'%s'\n", header, line); goto end; } while ((read = getline(&line, &len, record_file)) != -1) { - if (!replay(memory_chunk, memory_chunck_size, line)) { + line[read-1] = 0; + if (!replay(chunk_data, memory_chunck_size, line)) { g_warning("failed to replay dump!"); goto end; } @@ -304,7 +407,7 @@ main(int argc, char** argv) if(kv_module != NULL) { g_module_close(kv_module); } if(object_module != NULL) { g_module_close(object_module); } j_configuration_unref(configuration); - j_tarce_leave(trace); - j_trace_fini(); + // j_trace_leave(trace); + // j_trace_fini(); return res; -} \ No newline at end of file +} From e59a3ba54b1ac34ef00ed751ef54859703290fed Mon Sep 17 00:00:00 2001 From: Julian Benda Date: Sat, 11 Feb 2023 00:08:32 +0100 Subject: [PATCH 06/40] first reply example --- tools/access_replay.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/access_replay.c b/tools/access_replay.c index b943a0ecd..c850ababe 100644 --- a/tools/access_replay.c +++ b/tools/access_replay.c @@ -401,7 +401,7 @@ main(int argc, char** argv) } if(object_backend != NULL) { - j_backend_db_fini(db_backend); + j_backend_object_fini(object_backend); } if(db_module != NULL) { g_module_close(db_module); } if(kv_module != NULL) { g_module_close(kv_module); } From 3a9d3fe3e75aa185c1fecc008042e4c08cfd0f70 Mon Sep 17 00:00:00 2001 From: Julian Benda Date: Sun, 12 Feb 2023 11:35:07 +0100 Subject: [PATCH 07/40] Add complexity guess for db selectors --- lib/core/jbackend.c | 1 + lib/core/jtrace.c | 35 ++++++++++++++++++++++++++++++----- tools/access_replay.c | 34 ++++++++++++++++++++-------------- 3 files changed, 51 insertions(+), 19 deletions(-) diff --git a/lib/core/jbackend.c b/lib/core/jbackend.c index 63e34f5a8..a24597555 100644 --- a/lib/core/jbackend.c +++ b/lib/core/jbackend.c @@ -263,6 +263,7 @@ j_backend_load_server(gchar const* name, gchar const* component, JBackendType ty g_return_val_if_fail(type == J_BACKEND_TYPE_OBJECT || type == J_BACKEND_TYPE_KV || type == J_BACKEND_TYPE_DB, FALSE); g_return_val_if_fail(module != NULL, FALSE); g_return_val_if_fail(backend != NULL, FALSE); + *module = NULL; *backend = NULL; diff --git a/lib/core/jtrace.c b/lib/core/jtrace.c index 027e71539..42a9a4034 100644 --- a/lib/core/jtrace.c +++ b/lib/core/jtrace.c @@ -620,6 +620,29 @@ store_db_namespace(JTraceThread* trace_thread, const char* namespace) } } +static guint32 +_count_keys_recursive(bson_iter_t* itr) +{ + guint32 cnt = 0; + do{ + cnt +=1; + if (bson_iter_type(itr) == BSON_TYPE_ARRAY || bson_iter_type(itr) == BSON_TYPE_DOCUMENT) { + bson_iter_t child; + bson_iter_recurse(itr, &child); + cnt += _count_keys_recursive(&child); + } + } while(bson_iter_next(itr)); + return cnt; +} + +static guint32 +count_keys_recursive(const bson_t* bson) +{ + bson_iter_t itr; + g_return_val_if_fail(bson_iter_init(&itr, bson), 0); + return _count_keys_recursive(&itr); +} + JTrace* j_trace_enter(gchar const* name, gchar const* format, ...) { @@ -781,19 +804,21 @@ j_trace_enter(gchar const* name, gchar const* format, ...) row->name = va_arg(args, const char*); row->size = va_arg(args, guint32); row->bson = va_arg(args, const bson_t*); - /// \TODO complexity from selector + /// \TODO complexity needed? va_end(args); } else if (strcmp(name, "update") == 0) { static bson_t bson; + const bson_t* selector = NULL; bson_init(&bson); va_start(args, format); va_arg(args, void*); row->name = va_arg(args, const char*); row->size = va_arg(args, guint32); - bson_append_document(&bson, "selector", -1, va_arg(args, const bson_t*)); + selector = va_arg(args, const bson_t*); + bson_append_document(&bson, "selector", -1, selector); bson_append_document(&bson, "entry", -1, va_arg(args, const bson_t*)); row->bson = &bson; - /// \TODO complexity from selector + row->complexity = count_keys_recursive(selector) - 2; // because the added two top level keys va_end(args); } else if (strcmp(name, "delete") == 0) { va_start(args, format); @@ -801,7 +826,7 @@ j_trace_enter(gchar const* name, gchar const* format, ...) row->name = va_arg(args, const char*); va_arg(args, guint32); row->bson = va_arg(args, const bson_t*); - /// \TODO complexity from selector + row->complexity = count_keys_recursive(row->bson); va_end(args); } else if (strcmp(name, "query") == 0) { va_start(args, format); @@ -809,7 +834,7 @@ j_trace_enter(gchar const* name, gchar const* format, ...) row->name = va_arg(args, const char*); va_arg(args, guint32); row->bson = va_arg(args, const bson_t*); - /// \TODO complexvity from selector + row->complexity = count_keys_recursive(row->bson); va_end(args); } // iterate, init, fini <- no further details diff --git a/tools/access_replay.c b/tools/access_replay.c index c850ababe..9d7234c98 100644 --- a/tools/access_replay.c +++ b/tools/access_replay.c @@ -51,6 +51,9 @@ setup_backend(JConfiguration* configuration, JBackendType type, gchar const* por case J_BACKEND_TYPE_OBJECT: res = j_backend_object_init(*j_backend, path); break; case J_BACKEND_TYPE_KV: res = j_backend_kv_init(*j_backend, path); break; case J_BACKEND_TYPE_DB: res = j_backend_db_init(*j_backend, path); break; + default: + g_warning("unknown backend type: (%d), unable to setup backend!", type); + res = FALSE; } if (!res) { g_warning("Failed to initelize backend: %s", backend); @@ -171,7 +174,12 @@ replay_kv(void* memory_chunk, guint64 memory_chunk_size, const char* parts[11]){ if (batch != NULL) { g_warning("starting a new batch, but old one is not finished"); } ret = j_backend_kv_batch_start(kv_backend, parts[NAMESPACE], semantics, &batch); } else if (strcmp(op, "put") == 0) { - ret = j_backend_kv_put(kv_backend, batch, parts[NAME], memory_chunk, parse_ul(parts[SIZE])); + guint64 size = parse_ul(parts[SIZE]); + if(size > memory_chunk_size){ + g_warning("unable to replay: chunk size to samll!%lu vs %lu", size, memory_chunk_size); + } else { + ret = j_backend_kv_put(kv_backend, batch, parts[NAME], memory_chunk, size); + } } else if (strcmp(op, "delete") == 0) { ret = j_backend_kv_delete(kv_backend, batch, parts[NAME]); } else if (strcmp(op, "get") == 0) { @@ -209,7 +217,7 @@ struct JSqlBatch gboolean aborted; }; static gboolean -replay_db(void* memory_chunk, guint64 memory_chunk_size, const char* parts[11]){ +replay_db(const char* parts[11]){ gboolean ret = FALSE; static gpointer batch = NULL; static gpointer itr = NULL; @@ -227,7 +235,7 @@ replay_db(void* memory_chunk, guint64 memory_chunk_size, const char* parts[11]){ namespace = strdup(parts[NAMESPACE]); ret = j_backend_db_batch_start(db_backend, namespace, semantics, &batch, &error); } else if (strcmp(op, "schema_create") == 0) { - bson_t* schema = bson_new_from_json(parts[JSON], -1, NULL); + bson_t* schema = bson_new_from_json((const uint8_t*)parts[JSON], -1, NULL); ret = j_backend_db_schema_create(db_backend, batch, parts[NAME], schema, &error); g_array_append_val(bsons, schema); } else if (strcmp(op, "schema_get") == 0) { @@ -243,13 +251,13 @@ replay_db(void* memory_chunk, guint64 memory_chunk_size, const char* parts[11]){ } else if (strcmp(op, "schema_delete") == 0) { ret = j_backend_db_schema_delete(db_backend, batch, parts[NAME], &error); } else if (strcmp(op, "insert") == 0) { - bson_t* entry = bson_new_from_json(parts[JSON], -1, NULL); + bson_t* entry = bson_new_from_json((const uint8_t*)parts[JSON], -1, NULL); bson_t* res = bson_new(); ret = j_backend_db_insert(db_backend, batch, parts[NAME], entry, res, &error); g_array_append_val(bsons, entry); g_array_append_val(bsons, res); } else if (strcmp(op, "delete") == 0) { - bson_t* entry = bson_new_from_json(parts[JSON], -1, NULL); + bson_t* entry = bson_new_from_json((const uint8_t*)parts[JSON], -1, NULL); ret = j_backend_db_delete(db_backend, batch, parts[NAME], entry, &error); g_array_append_val(bsons, entry); } else if (strcmp(op, "update") == 0) { @@ -258,7 +266,7 @@ replay_db(void* memory_chunk, guint64 memory_chunk_size, const char* parts[11]){ bson_t* entry; const uint8_t* doc; uint32_t len; - bson_t* bson = bson_new_from_json(parts[JSON], -1, NULL); + bson_t* bson = bson_new_from_json((const uint8_t*)parts[JSON], -1, NULL); bson_iter_init_find(&bson_iter, bson, "entry"); bson_iter_document(&bson_iter, &len, &doc); entry = bson_new_from_data(doc, len); @@ -270,7 +278,7 @@ replay_db(void* memory_chunk, guint64 memory_chunk_size, const char* parts[11]){ g_array_append_val(bsons, selector); g_array_append_val(bsons, bson); } else if (strcmp(op, "query") == 0) { - bson_t* selector = bson_new_from_json(parts[JSON], -1, NULL); + bson_t* selector = bson_new_from_json((const uint8_t*)parts[JSON], -1, NULL); if(itr != NULL) { g_warning("start new db iteration without finishing old one!"); } ret = j_backend_db_query(db_backend, batch, parts[NAME], selector, &itr, &error); g_array_append_val(bsons, selector); @@ -312,7 +320,7 @@ replay(void* memory_chunk, guint64 memory_chunck_size, char* line) if (!split(line, parts)) { return FALSE; } if(strcmp(parts[BACKEND], "object") == 0) { if(!replay_object(memory_chunk, memory_chunck_size, parts)) return FALSE; } if(strcmp(parts[BACKEND], "kv") == 0) { if(!replay_kv(memory_chunk, memory_chunck_size, parts)) return FALSE; } - if(strcmp(parts[BACKEND], "db") == 0) { if(!replay_db(memory_chunk, memory_chunck_size, parts)) return FALSE; } + if(strcmp(parts[BACKEND], "db") == 0) { if(!replay_db(parts)) return FALSE; } return TRUE; } @@ -323,13 +331,12 @@ main(int argc, char** argv) JConfiguration* configuration = NULL; const char* record_file_name = NULL; FILE* record_file = NULL; - JTrace* trace = NULL; GModule* db_module = NULL; GModule* kv_module = NULL; GModule* object_module = NULL; g_autofree gchar* port_str = NULL; gboolean res = FALSE; - JMemoryChunk* memory_chunk = NULL; + gchar* memory_chunk; guint64 memory_chunck_size = 0; setlocale(LC_ALL, "C.UTF-8"); @@ -363,15 +370,14 @@ main(int argc, char** argv) if(!setup_backend(configuration, J_BACKEND_TYPE_DB, port_str, &db_module, &db_backend)) { goto end; } memory_chunck_size = j_configuration_get_max_operation_size(configuration); - memory_chunk = j_memory_chunk_new(memory_chunck_size); + memory_chunk = malloc(memory_chunck_size); { const char* header = "time,process_uid,program_name,backend,namespace,name,operation,size,complexity,duration,bson"; char* line = NULL; size_t len = 0; ssize_t read = 0; - void* chunk_data = *((void**)((char*)memory_chunk+8)); - memset(chunk_data, 0, memory_chunck_size); + memset(memory_chunk, 0, memory_chunck_size); read = getline(&line, &len, record_file); line[read-1] = 0; if (read == -1 || strcmp(line, header) != 0) { @@ -380,7 +386,7 @@ main(int argc, char** argv) } while ((read = getline(&line, &len, record_file)) != -1) { line[read-1] = 0; - if (!replay(chunk_data, memory_chunck_size, line)) { + if (!replay(memory_chunk, memory_chunck_size, line)) { g_warning("failed to replay dump!"); goto end; } From c1e7b028a4ffcae8283f4a5af8c41ae6f9d7d1fa Mon Sep 17 00:00:00 2001 From: Julian Benda Date: Wed, 15 Feb 2023 16:15:55 +0100 Subject: [PATCH 08/40] Formatting --- lib/core/jbackend.c | 1 - lib/core/jconfiguration.c | 8 +- lib/core/jconnection-pool.c | 14 +- lib/core/jtrace.c | 562 +++++++++++++---------- lib/db-util/jbson.c | 7 +- lib/object/jobject.c | 2 +- tools/access_replay.c | 865 ++++++++++++++++++++++-------------- 7 files changed, 870 insertions(+), 589 deletions(-) diff --git a/lib/core/jbackend.c b/lib/core/jbackend.c index a24597555..63e34f5a8 100644 --- a/lib/core/jbackend.c +++ b/lib/core/jbackend.c @@ -263,7 +263,6 @@ j_backend_load_server(gchar const* name, gchar const* component, JBackendType ty g_return_val_if_fail(type == J_BACKEND_TYPE_OBJECT || type == J_BACKEND_TYPE_KV || type == J_BACKEND_TYPE_DB, FALSE); g_return_val_if_fail(module != NULL, FALSE); g_return_val_if_fail(backend != NULL, FALSE); - *module = NULL; *backend = NULL; diff --git a/lib/core/jconfiguration.c b/lib/core/jconfiguration.c index 62e2ca405..26b271343 100644 --- a/lib/core/jconfiguration.c +++ b/lib/core/jconfiguration.c @@ -291,14 +291,12 @@ j_configuration_new_for_data(GKeyFile* key_file) guint32 port; guint32 max_connections; guint64 stripe_size; - guint32 uid = 0; g_return_val_if_fail(key_file != NULL, FALSE); max_operation_size = g_key_file_get_uint64(key_file, "core", "max-operation-size", NULL); max_inject_size = g_key_file_get_uint64(key_file, "core", "max-inject-size", NULL); port = g_key_file_get_integer(key_file, "core", "port", NULL); - uid = g_key_file_get_uint64(key_file, "core", "uid", NULL); max_connections = g_key_file_get_integer(key_file, "clients", "max-connections", NULL); stripe_size = g_key_file_get_uint64(key_file, "clients", "stripe-size", NULL); servers_object = g_key_file_get_string_list(key_file, "servers", "object", NULL, NULL); @@ -367,7 +365,6 @@ j_configuration_new_for_data(GKeyFile* key_file) configuration->max_inject_size = max_inject_size; configuration->max_connections = max_connections; configuration->stripe_size = stripe_size; - configuration->uid = uid; configuration->checksum = NULL; configuration->ref_count = 1; @@ -398,15 +395,14 @@ j_configuration_new_for_data(GKeyFile* key_file) configuration->stripe_size = 4 * 1024 * 1024; } - if (configuration->uid == 0) - { + { // uid should be unique per process in one application GRand* rand = g_rand_new(); configuration->uid = g_rand_int(rand); g_rand_free(rand); } key_file_str = g_key_file_to_data(key_file, NULL, NULL); - configuration->checksum = g_compute_checksum_for_string(G_CHECKSUM_SHA512, key_file_str, -1); + configuration->checksum = g_compute_checksum_for_string(G_CHECKSUM_SHA512, key_file_str, -1); return configuration; } diff --git a/lib/core/jconnection-pool.c b/lib/core/jconnection-pool.c index 947d729d4..efc98f499 100644 --- a/lib/core/jconnection-pool.c +++ b/lib/core/jconnection-pool.c @@ -169,7 +169,7 @@ j_connection_pool_fini(void) g_slice_free(JConnectionPool, pool); } -extern const char *__progname; +extern const char* __progname; static GSocketConnection* j_connection_pool_pop_internal(GAsyncQueue* queue, guint* count, gchar const* server) { @@ -221,14 +221,20 @@ j_connection_pool_pop_internal(GAsyncQueue* queue, guint* count, gchar const* se client_checksum = j_configuration_get_checksum(j_configuration()); client_program_name = g_get_prgname(); - if (client_program_name == NULL) { client_program_name = __progname; } - if (client_program_name == NULL) { client_program_name = ""; } + if (client_program_name == NULL) + { + client_program_name = __progname; + } + if (client_program_name == NULL) + { + client_program_name = ""; + } client_process_uid = j_configuration_get_uid(j_configuration()); message = j_message_new(J_MESSAGE_PING, strlen(client_checksum) + 1); j_message_append_string(message, client_checksum); j_message_append_string(message, client_program_name); - j_message_append_n(message, &client_process_uid, sizeof(guint32)); + j_message_append_n(message, &client_process_uid, sizeof(guint32)); j_message_send(message, connection); reply = j_message_new_reply(message); diff --git a/lib/core/jtrace.c b/lib/core/jtrace.c index 42a9a4034..e9755aecb 100644 --- a/lib/core/jtrace.c +++ b/lib/core/jtrace.c @@ -69,7 +69,8 @@ struct JTraceTime typedef struct JTraceTime JTraceTime; -struct Access { +struct Access +{ guint64 timestamp; guint32 uid; const char* program_name; @@ -99,7 +100,6 @@ struct JTraceThread **/ guint function_depth; - GArray* stack; #ifdef HAVE_OTF @@ -115,30 +115,34 @@ struct JTraceThread } otf; #endif - struct { + struct + { char* program_name; guint32 uid; } client; - struct { + struct + { gboolean inside; Access row; const void* utility_ptr; - struct { + struct + { char* namespace; } db; - struct { + struct + { char* namespace; char* name; } kv; - struct { + struct + { char* namespace; char* path; } object; } access; }; - typedef struct JTraceThread JTraceThread; struct JTrace @@ -179,7 +183,8 @@ G_LOCK_DEFINE_STATIC(j_trace_echo); G_LOCK_DEFINE_STATIC(j_trace_summary); static void -j_trace_access_print_header(void) { +j_trace_access_print_header(void) +{ g_printerr("time,process_uid,program_name,backend,namespace,name,operation,size,complexity,duration,bson\n"); } /** @@ -492,7 +497,6 @@ j_trace_init(gchar const* name) G_UNLOCK(j_trace_echo); } - g_free(j_trace_name); j_trace_name = g_strdup(name); } @@ -559,64 +563,83 @@ j_trace_fini(void) g_free(j_trace_name); } - static void -j_trace_access_print(const Access* row, guint64 duration) { - g_printerr("%"G_GUINT64_FORMAT".%06"G_GUINT64_FORMAT",%u,%s,%s,%s,%s,%s,%"G_GUINT64_FORMAT",%u,%"G_GUINT64_FORMAT".%06"G_GUINT64_FORMAT",\"%s\"\n", - row->timestamp / G_USEC_PER_SEC, - row->timestamp % G_USEC_PER_SEC, - row->uid, - row->program_name, - row->backend, - row->namespace, - row->name, - row->operation, - row->size, - row->complexity, - duration / G_USEC_PER_SEC, - duration % G_USEC_PER_SEC, - row->bson == NULL ? "{}" : bson_as_json(row->bson, NULL) - ); +j_trace_access_print(const Access* row, guint64 duration) +{ + g_printerr("%" G_GUINT64_FORMAT ".%06" G_GUINT64_FORMAT ",%u,%s,%s,%s,%s,%s,%" G_GUINT64_FORMAT ",%u,%" G_GUINT64_FORMAT ".%06" G_GUINT64_FORMAT ",\"%s\"\n", + row->timestamp / G_USEC_PER_SEC, + row->timestamp % G_USEC_PER_SEC, + row->uid, + row->program_name, + row->backend, + row->namespace, + row->name, + row->operation, + row->size, + row->complexity, + duration / G_USEC_PER_SEC, + duration % G_USEC_PER_SEC, + row->bson == NULL ? "{}" : bson_as_json(row->bson, NULL)); } static void store_object_path(JTraceThread* trace_thread, const char* namespace, const char* path) { - if (trace_thread->access.object.namespace != namespace) { + if (trace_thread->access.object.namespace != namespace) + { g_free(trace_thread->access.object.namespace); trace_thread->access.object.namespace = NULL; - if(namespace) { trace_thread->access.object.namespace = strdup(namespace); } + if (namespace) + { + trace_thread->access.object.namespace = strdup(namespace); + } } - if(trace_thread->access.object.path != path) { + if (trace_thread->access.object.path != path) + { g_free(trace_thread->access.object.path); trace_thread->access.object.path = NULL; - if (path) { trace_thread->access.object.path = strdup(path); } + if (path) + { + trace_thread->access.object.path = strdup(path); + } } } static void store_kv_namespace(JTraceThread* trace_thread, const char* namespace, const char* name) { - if (trace_thread->access.kv.namespace != namespace) { + if (trace_thread->access.kv.namespace != namespace) + { g_free(trace_thread->access.kv.namespace); trace_thread->access.kv.namespace = NULL; - if(namespace) { trace_thread->access.kv.namespace = strdup(namespace); } + if (namespace) + { + trace_thread->access.kv.namespace = strdup(namespace); + } } - if (trace_thread->access.kv.name != name) { + if (trace_thread->access.kv.name != name) + { g_free(trace_thread->access.kv.name); trace_thread->access.kv.name = NULL; - if(name) { trace_thread->access.kv.name = strdup(name); } + if (name) + { + trace_thread->access.kv.name = strdup(name); + } } } static void store_db_namespace(JTraceThread* trace_thread, const char* namespace) { - if (trace_thread->access.db.namespace != namespace) { + if (trace_thread->access.db.namespace != namespace) + { g_free(trace_thread->access.db.namespace); trace_thread->access.db.namespace = NULL; - if (namespace) { trace_thread->access.db.namespace = strdup(namespace); } + if (namespace) + { + trace_thread->access.db.namespace = strdup(namespace); + } } } @@ -624,14 +647,16 @@ static guint32 _count_keys_recursive(bson_iter_t* itr) { guint32 cnt = 0; - do{ - cnt +=1; - if (bson_iter_type(itr) == BSON_TYPE_ARRAY || bson_iter_type(itr) == BSON_TYPE_DOCUMENT) { + do + { + cnt += 1; + if (bson_iter_type(itr) == BSON_TYPE_ARRAY || bson_iter_type(itr) == BSON_TYPE_DOCUMENT) + { bson_iter_t child; bson_iter_recurse(itr, &child); cnt += _count_keys_recursive(&child); } - } while(bson_iter_next(itr)); + } while (bson_iter_next(itr)); return cnt; } @@ -671,7 +696,6 @@ j_trace_enter(gchar const* name, gchar const* format, ...) trace->name = g_strdup(name); trace->enter_time = timestamp; - if (j_trace_flags & J_TRACE_ECHO) { G_LOCK(j_trace_echo); @@ -710,190 +734,236 @@ j_trace_enter(gchar const* name, gchar const* format, ...) { int type = 0; name += 8; - if (strncmp(name, "kv_", 3) == 0) { type = 1; } - else if (strncmp(name, "db_", 3) == 0) { type = 2; } - else if (strncmp(name, "object_", 7) == 0) { type = 3; } - if (type != 0) { - Access* row = &trace_thread->access.row; - va_list args; - memset(row, 0, sizeof(Access)); - row->uid = trace_thread->client.uid; - row->program_name = trace_thread->client.program_name; - row->timestamp = timestamp; - trace_thread->access.inside = TRUE; - - if (type == 1) + if (strncmp(name, "kv_", 3) == 0) { - name += 3; - row->backend = "kv"; - row->operation = name; - row->namespace = trace_thread->access.kv.namespace; - row->name = trace_thread->access.kv.name; - if (strcmp(name, "batch_start") == 0) - { - va_start(args, format); - row->namespace = va_arg(args, const char*); - va_end(args); - store_kv_namespace(trace_thread, row->namespace, NULL); - } else if (strcmp(name, "put") == 0) { - va_start(args, format); - va_arg(args, void*); - row->name = va_arg(args, const char*); - va_arg(args, void*); - row->size = va_arg(args, guint32); - va_end(args); - } else if (strcmp(name, "delete") == 0) { - va_start(args, format); - va_arg(args, void*); - row->name = va_arg(args, const char*); - va_end(args); - } else if (strcmp(name, "get") == 0){ - va_start(args, format); - va_arg(args, void*); - row->name = va_arg(args, const char*); - va_arg(args, void*); - trace_thread->access.utility_ptr = va_arg(args, void*); - va_end(args); - } else if (strcmp(name, "get_all") == 0) { - va_start(args, format); - row->namespace = va_arg(args, const char*); - va_end(args); - store_kv_namespace(trace_thread, row->namespace, NULL); - } else if (strcmp(name, "get_by_prefix") == 0) { - va_start(args, format); - row->namespace = va_arg(args, const char*); - row->name = va_arg(args, const char*); - va_end(args); - store_kv_namespace(trace_thread, row->namespace, row->name); - } - - // iterate, init, fini <- no further deatils - // batch_execute <- handlen in leave + type = 1; + } + else if (strncmp(name, "db_", 3) == 0) + { + type = 2; + } + else if (strncmp(name, "object_", 7) == 0) + { + type = 3; } - else if (type == 2) + if (type != 0) { - name += 3; - row->backend = "db"; - row->operation = name; - row->namespace = trace_thread->access.db.namespace; - if (strcmp(name, "batch_start") == 0) { - va_start(args, format); - row->namespace = va_arg(args, const char*); - va_end(args); - store_db_namespace(trace_thread, row->namespace); - } else if (strcmp(name, "schema_create") == 0) { + Access* row = &trace_thread->access.row; + va_list args; + memset(row, 0, sizeof(Access)); + row->uid = trace_thread->client.uid; + row->program_name = trace_thread->client.program_name; + row->timestamp = timestamp; + trace_thread->access.inside = TRUE; + + if (type == 1) + { + name += 3; + row->backend = "kv"; + row->operation = name; + row->namespace = trace_thread->access.kv.namespace; + row->name = trace_thread->access.kv.name; + if (strcmp(name, "batch_start") == 0) + { + va_start(args, format); + row->namespace = va_arg(args, const char*); + va_end(args); + store_kv_namespace(trace_thread, row->namespace, NULL); + } + else if (strcmp(name, "put") == 0) + { + va_start(args, format); + va_arg(args, void*); + row->name = va_arg(args, const char*); + va_arg(args, void*); + row->size = va_arg(args, guint32); + va_end(args); + } + else if (strcmp(name, "delete") == 0) + { + va_start(args, format); + va_arg(args, void*); + row->name = va_arg(args, const char*); + va_end(args); + } + else if (strcmp(name, "get") == 0) + { + va_start(args, format); + va_arg(args, void*); + row->name = va_arg(args, const char*); + va_arg(args, void*); + trace_thread->access.utility_ptr = va_arg(args, void*); + va_end(args); + } + else if (strcmp(name, "get_all") == 0) + { + va_start(args, format); + row->namespace = va_arg(args, const char*); + va_end(args); + store_kv_namespace(trace_thread, row->namespace, NULL); + } + else if (strcmp(name, "get_by_prefix") == 0) + { + va_start(args, format); + row->namespace = va_arg(args, const char*); + row->name = va_arg(args, const char*); + va_end(args); + store_kv_namespace(trace_thread, row->namespace, row->name); + } + + // iterate, init, fini <- no further deatils + // batch_execute <- handlen in leave + } + else if (type == 2) + { + name += 3; + row->backend = "db"; + row->operation = name; + row->namespace = trace_thread->access.db.namespace; + if (strcmp(name, "batch_start") == 0) + { + va_start(args, format); + row->namespace = va_arg(args, const char*); + va_end(args); + store_db_namespace(trace_thread, row->namespace); + } + else if (strcmp(name, "schema_create") == 0) + { va_start(args, format); va_arg(args, void*); row->name = va_arg(args, const char*); row->size = va_arg(args, guint32); row->bson = va_arg(args, const bson_t*); va_end(args); - } else if (strcmp(name, "schema_get") == 0) { - va_start(args, format); - va_arg(args, void*); - row->name = va_arg(args, const char*); - va_end(args); - } else if (strcmp(name, "schema_delete") == 0) { - va_start(args, format); - va_arg(args, void*); - row->name = va_arg(args, const char*); - va_end(args); - } else if (strcmp(name, "insert") == 0) { - va_start(args, format); - va_arg(args, void*); - row->name = va_arg(args, const char*); - row->size = va_arg(args, guint32); - row->bson = va_arg(args, const bson_t*); - /// \TODO complexity needed? - va_end(args); - } else if (strcmp(name, "update") == 0) { - static bson_t bson; - const bson_t* selector = NULL; - bson_init(&bson); - va_start(args, format); - va_arg(args, void*); - row->name = va_arg(args, const char*); - row->size = va_arg(args, guint32); - selector = va_arg(args, const bson_t*); - bson_append_document(&bson, "selector", -1, selector); - bson_append_document(&bson, "entry", -1, va_arg(args, const bson_t*)); - row->bson = &bson; - row->complexity = count_keys_recursive(selector) - 2; // because the added two top level keys - va_end(args); - } else if (strcmp(name, "delete") == 0) { - va_start(args, format); - va_arg(args, void*); - row->name = va_arg(args, const char*); - va_arg(args, guint32); - row->bson = va_arg(args, const bson_t*); - row->complexity = count_keys_recursive(row->bson); - va_end(args); - } else if (strcmp(name, "query") == 0) { - va_start(args, format); - va_arg(args, void*); - row->name = va_arg(args, const char*); - va_arg(args, guint32); - row->bson = va_arg(args, const bson_t*); - row->complexity = count_keys_recursive(row->bson); - va_end(args); + } + else if (strcmp(name, "schema_get") == 0) + { + va_start(args, format); + va_arg(args, void*); + row->name = va_arg(args, const char*); + va_end(args); + } + else if (strcmp(name, "schema_delete") == 0) + { + va_start(args, format); + va_arg(args, void*); + row->name = va_arg(args, const char*); + va_end(args); + } + else if (strcmp(name, "insert") == 0) + { + va_start(args, format); + va_arg(args, void*); + row->name = va_arg(args, const char*); + row->size = va_arg(args, guint32); + row->bson = va_arg(args, const bson_t*); + /// \TODO complexity needed? + va_end(args); + } + else if (strcmp(name, "update") == 0) + { + static bson_t bson; + const bson_t* selector = NULL; + bson_init(&bson); + va_start(args, format); + va_arg(args, void*); + row->name = va_arg(args, const char*); + row->size = va_arg(args, guint32); + selector = va_arg(args, const bson_t*); + bson_append_document(&bson, "selector", -1, selector); + bson_append_document(&bson, "entry", -1, va_arg(args, const bson_t*)); + row->bson = &bson; + row->complexity = count_keys_recursive(selector) - 2; // because the added two top level keys + va_end(args); + } + else if (strcmp(name, "delete") == 0) + { + va_start(args, format); + va_arg(args, void*); + row->name = va_arg(args, const char*); + va_arg(args, guint32); + row->bson = va_arg(args, const bson_t*); + row->complexity = count_keys_recursive(row->bson); + va_end(args); + } + else if (strcmp(name, "query") == 0) + { + va_start(args, format); + va_arg(args, void*); + row->name = va_arg(args, const char*); + va_arg(args, guint32); + row->bson = va_arg(args, const bson_t*); + row->complexity = count_keys_recursive(row->bson); + va_end(args); + } + // iterate, init, fini <- no further details + // batch_execute <- handled in leave } - // iterate, init, fini <- no further details - // batch_execute <- handled in leave - } - else if (type == 3) - { - name += 7; - row->backend = "object"; - row->operation = name; - row->namespace = trace_thread->access.object.namespace; - row->name = trace_thread->access.object.path; - if (strcmp(name, "create") == 0) { - va_start(args, format); - row->namespace = va_arg(args, const char*); - row->name = va_arg(args, const char*); - va_end(args); - store_object_path(trace_thread, row->namespace, row->name); - } else if (strcmp(name, "open") == 0) { - va_start(args, format); - row->namespace = va_arg(args, const char*); - row->name = va_arg(args, const char*); - va_end(args); - store_object_path(trace_thread, row->namespace, row->name); - } else if (strcmp(name, "get_all") == 0) { - va_start(args, format); - row->namespace = va_arg(args, const char*); - row->name = NULL; - va_end(args); - store_object_path(trace_thread, row->namespace, row->name); - } else if (strcmp(name, "get_by_prefix") == 0) { - va_start(args, format); - row->namespace = va_arg(args, const char*); - row->name = va_arg(args, const char*); - va_end(args); - store_object_path(trace_thread, row->namespace, row->name); - } else if (strcmp(name, "read") == 0) { - va_start(args, format); - va_arg(args, void*); - va_arg(args, void*); - row->size = va_arg(args, guint64); - row->complexity = va_arg(args, guint64); - va_end(args); - } else if (strcmp(name, "write") == 0) { - va_start(args, format); - va_arg(args, void*); - va_arg(args, void*); - row->size = va_arg(args, guint64); - row->complexity = va_arg(args, guint64); - va_end(args); + else if (type == 3) + { + name += 7; + row->backend = "object"; + row->operation = name; + row->namespace = trace_thread->access.object.namespace; + row->name = trace_thread->access.object.path; + if (strcmp(name, "create") == 0) + { + va_start(args, format); + row->namespace = va_arg(args, const char*); + row->name = va_arg(args, const char*); + va_end(args); + store_object_path(trace_thread, row->namespace, row->name); + } + else if (strcmp(name, "open") == 0) + { + va_start(args, format); + row->namespace = va_arg(args, const char*); + row->name = va_arg(args, const char*); + va_end(args); + store_object_path(trace_thread, row->namespace, row->name); + } + else if (strcmp(name, "get_all") == 0) + { + va_start(args, format); + row->namespace = va_arg(args, const char*); + row->name = NULL; + va_end(args); + store_object_path(trace_thread, row->namespace, row->name); + } + else if (strcmp(name, "get_by_prefix") == 0) + { + va_start(args, format); + row->namespace = va_arg(args, const char*); + row->name = va_arg(args, const char*); + va_end(args); + store_object_path(trace_thread, row->namespace, row->name); + } + else if (strcmp(name, "read") == 0) + { + va_start(args, format); + va_arg(args, void*); + va_arg(args, void*); + row->size = va_arg(args, guint64); + row->complexity = va_arg(args, guint64); + va_end(args); + } + else if (strcmp(name, "write") == 0) + { + va_start(args, format); + va_arg(args, void*); + va_arg(args, void*); + row->size = va_arg(args, guint64); + row->complexity = va_arg(args, guint64); + va_end(args); + } + // status, sync, iterate, fini, init <- no further data + // close/delte handle at leave + } + else + { + /// \TODO handle unknown backends } - // status, sync, iterate, fini, init <- no further data - // close/delte handle at leave - - } - else { - /// \TODO handle unknown backends } - } } } @@ -943,7 +1013,6 @@ j_trace_enter(gchar const* name, gchar const* format, ...) g_array_append_val(trace_thread->stack, current_stack); } - trace_thread->function_depth++; return trace; @@ -996,34 +1065,43 @@ j_trace_leave(JTrace* trace) if (j_trace_flags & J_TRACE_ACCESS) { - if (trace_thread->access.inside) { - if (strncmp(trace->name, "backend_", 8) == 0) { - if(strncmp(trace->name + 8, "db_", 3) == 0 || strncmp(trace->name+8, "kv_", 3) == 0 || strncmp(trace->name+8, "object_", 7)== 0) { - guint64 duration; - Access* row = &trace_thread->access.row; - - duration = timestamp - trace->enter_time; - - if (strcmp(trace->name, "backend_kv_get") == 0) - { - row->size = *(const guint64*)trace_thread->access.utility_ptr; - } - - G_LOCK(j_trace_echo); - j_trace_access_print(row, duration); - G_UNLOCK(j_trace_echo); - - trace_thread->access.inside = FALSE; - if(strcmp(trace->name, "backend_kv_batch_execute") == 0) { - store_kv_namespace(trace_thread, NULL, NULL); - } else if (strcmp(trace->name, "backend_dbupdate") == 0){ - bson_destroy((bson_t*)row->bson); - } else if (strcmp(trace->name, "backend_db_batch_execute") == 0) { - store_db_namespace(trace_thread, NULL); - } else if (strcmp(trace->name, "backend_object_delete") == 0 || strcmp(trace->name, "backend_object_close") == 0 ) + if (trace_thread->access.inside) + { + if (strncmp(trace->name, "backend_", 8) == 0) + { + if (strncmp(trace->name + 8, "db_", 3) == 0 || strncmp(trace->name + 8, "kv_", 3) == 0 || strncmp(trace->name + 8, "object_", 7) == 0) { - store_object_path(trace_thread, NULL, NULL); - } + guint64 duration; + Access* row = &trace_thread->access.row; + + duration = timestamp - trace->enter_time; + + if (strcmp(trace->name, "backend_kv_get") == 0) + { + row->size = *(const guint64*)trace_thread->access.utility_ptr; + } + + G_LOCK(j_trace_echo); + j_trace_access_print(row, duration); + G_UNLOCK(j_trace_echo); + + trace_thread->access.inside = FALSE; + if (strcmp(trace->name, "backend_kv_batch_execute") == 0) + { + store_kv_namespace(trace_thread, NULL, NULL); + } + else if (strcmp(trace->name, "backend_dbupdate") == 0) + { + bson_destroy((bson_t*)row->bson); + } + else if (strcmp(trace->name, "backend_db_batch_execute") == 0) + { + store_db_namespace(trace_thread, NULL); + } + else if (strcmp(trace->name, "backend_object_delete") == 0 || strcmp(trace->name, "backend_object_close") == 0) + { + store_object_path(trace_thread, NULL, NULL); + } } } } diff --git a/lib/db-util/jbson.c b/lib/db-util/jbson.c index 45627e6cc..6ab3dac98 100644 --- a/lib/db-util/jbson.c +++ b/lib/db-util/jbson.c @@ -349,9 +349,12 @@ j_bson_iter_value(bson_iter_t* iter, JDBType type, JDBTypeValue* value, GError** if (value) { - if(BSON_ITER_HOLDS_INT64(iter)) { + if (BSON_ITER_HOLDS_INT64(iter)) + { value->val_sint64 = bson_iter_int64(iter); - } else { + } + else + { value->val_sint64 = bson_iter_int32(iter); } } diff --git a/lib/object/jobject.c b/lib/object/jobject.c index 21c9f84d7..6a0a290ff 100644 --- a/lib/object/jobject.c +++ b/lib/object/jobject.c @@ -595,7 +595,7 @@ static gboolean j_object_write_exec(JList* operations, JSemantics* semantics) { J_TRACE_FUNCTION(NULL); - + /// \todo check return value for messages gboolean ret = TRUE; diff --git a/tools/access_replay.c b/tools/access_replay.c index 9d7234c98..726e3dee2 100644 --- a/tools/access_replay.c +++ b/tools/access_replay.c @@ -33,35 +33,45 @@ JBackend* db_backend = NULL; static gboolean setup_backend(JConfiguration* configuration, JBackendType type, gchar const* port_str, GModule** module, JBackend** j_backend) { - gchar const* backend; - gchar const* component; - g_autofree gchar* path; + gchar const* backend; + gchar const* component; + g_autofree gchar* path; - backend = j_configuration_get_backend(configuration, type); - component = j_configuration_get_backend_component(configuration, type); - path = j_helper_str_replace(j_configuration_get_backend_path(configuration, type), "{PORT}", port_str); + backend = j_configuration_get_backend(configuration, type); + component = j_configuration_get_backend_component(configuration, type); + path = j_helper_str_replace(j_configuration_get_backend_path(configuration, type), "{PORT}", port_str); - if(j_backend_load_server(backend, component, type, module, j_backend)) { - gboolean res = TRUE; - if(j_backend == NULL) { - g_warning("Failed to load server: %s.", backend); - return FALSE; - } - switch (type) { - case J_BACKEND_TYPE_OBJECT: res = j_backend_object_init(*j_backend, path); break; - case J_BACKEND_TYPE_KV: res = j_backend_kv_init(*j_backend, path); break; - case J_BACKEND_TYPE_DB: res = j_backend_db_init(*j_backend, path); break; - default: - g_warning("unknown backend type: (%d), unable to setup backend!", type); - res = FALSE; - } - if (!res) { - g_warning("Failed to initelize backend: %s", backend); - return FALSE; - } - return TRUE; - } - return FALSE; + if (j_backend_load_server(backend, component, type, module, j_backend)) + { + gboolean res = TRUE; + if (j_backend == NULL) + { + g_warning("Failed to load server: %s.", backend); + return FALSE; + } + switch (type) + { + case J_BACKEND_TYPE_OBJECT: + res = j_backend_object_init(*j_backend, path); + break; + case J_BACKEND_TYPE_KV: + res = j_backend_kv_init(*j_backend, path); + break; + case J_BACKEND_TYPE_DB: + res = j_backend_db_init(*j_backend, path); + break; + default: + g_warning("unknown backend type: (%d), unable to setup backend!", type); + res = FALSE; + } + if (!res) + { + g_warning("Failed to initelize backend: %s", backend); + return FALSE; + } + return TRUE; + } + return FALSE; } #define BACKEND 3 @@ -74,140 +84,240 @@ setup_backend(JConfiguration* configuration, JBackendType type, gchar const* por #define JSON 10 static gboolean -split(char* line, const char* parts[11]) { - int cnt = 0; - const char* section; - section = strtok(line, ","); - while(section != NULL) { - parts[cnt++] = section; - if (cnt == 10) { break; } - section = strtok(NULL, ","); - } - while(*++section); - parts[cnt] = section+1; - if (cnt != 10) { - g_warning("to few parts in line"); - return FALSE; - } - parts[JSON] += 1; - { - char* itr = parts[JSON]; - while(*++itr); - itr[-1] = 0; - } - return TRUE; +split(char* line, const char* parts[11]) +{ + int cnt = 0; + const char* section; + section = strtok(line, ","); + while (section != NULL) + { + parts[cnt++] = section; + if (cnt == 10) + { + break; + } + section = strtok(NULL, ","); + } + while (*++section) + ; + parts[cnt] = section + 1; + if (cnt != 10) + { + g_warning("to few parts in line"); + return FALSE; + } + parts[JSON] += 1; + { + char* itr = parts[JSON]; + while (*++itr) + ; + itr[-1] = 0; + } + return TRUE; } static guint64 -parse_ul(const char* str) { - return strtoul(str, NULL, 10); +parse_ul(const char* str) +{ + return strtoul(str, NULL, 10); } static gboolean -replay_object(void* memory_chunk, guint64 memory_chunk_size, const char* parts[11]){ - static gpointer data = NULL; - const char* op = parts[OPERATION]; - gboolean ret = FALSE; - (void) memory_chunk_size; /// \TODO usage? - if(strcmp(op, "create") == 0) { - if (data != NULL) { g_warning("open new object while still one open"); } - ret = j_backend_object_create(object_backend, parts[NAMESPACE], parts[NAME], &data); - } else if (strcmp(op, "open") == 0) { - if (data != NULL) { g_warning("open new object while still one open"); } - ret = j_backend_object_open(object_backend, parts[NAMESPACE], parts[NAME], &data); - } else if (strcmp(op, "get_all") == 0) { - if (data != NULL) { g_warning("open new object while still one open"); } - ret = j_backend_object_get_all(object_backend, parts[NAMESPACE], data); - } else if (strcmp(op, "ge_by_prefix") == 0) { - if (data != NULL) { g_warning("open new object while still one open"); } - ret = j_backend_kv_get_by_prefix(object_backend, parts[NAMESPACE], parts[NAME], &data); - } else if (strcmp(op, "read") == 0) { - static guint64 bytes_read = 0; - guint64 size = parse_ul(parts[SIZE]); - if (size > memory_chunk_size) { - g_warning("unable to replay: chunk size to samll!%lu vs %lu", size, memory_chunk_size); - } else { - ret = j_backend_object_read(object_backend, data, memory_chunk, size, parse_ul(parts[COMPLEXITY]), &bytes_read); - } - } else if (strcmp(op, "write") == 0) { - static guint64 bytes_written = 0; - guint64 size = parse_ul(parts[SIZE]); - if (size > memory_chunk_size) { - g_warning("unable to replay: chunk size to samll!%lu vs %lu", size, memory_chunk_size); - } else { - ret = j_backend_object_write(object_backend, data, memory_chunk, size, parse_ul(parts[COMPLEXITY]), &bytes_written); - } - } else if (strcmp(op, "status") == 0) { - static gint64 modification_time; - static guint64 size; - ret = j_backend_object_status(object_backend, data, &modification_time, &size); - } else if (strcmp(op, "sync") == 0) { - ret = j_backend_object_sync(object_backend, data); - } else if (strcmp(op, "iterate") == 0) { - static const char* name = NULL; - ret = j_backend_object_iterate(object_backend, data, &name); - if (!ret) { data = NULL; ret = TRUE; } - } else if (strcmp(op, "close") == 0) { - ret = j_backend_object_close(object_backend, data); - data = NULL; - } else if (strcmp(op, "delete") == 0) { - ret = j_backend_object_delete(object_backend, data); - data = NULL; - } else if (strcmp(op, "init") == 0) { - ret = TRUE; - } else if (strcmp(op, "fini") == 0) { - ret = TRUE; - } else { - g_warning("unkown object operation: '%s'", op); - } - return ret; +replay_object(void* memory_chunk, guint64 memory_chunk_size, const char* parts[11]) +{ + static gpointer data = NULL; + const char* op = parts[OPERATION]; + gboolean ret = FALSE; + (void)memory_chunk_size; /// \TODO usage? + if (strcmp(op, "create") == 0) + { + if (data != NULL) + { + g_warning("open new object while still one open"); + } + ret = j_backend_object_create(object_backend, parts[NAMESPACE], parts[NAME], &data); + } + else if (strcmp(op, "open") == 0) + { + if (data != NULL) + { + g_warning("open new object while still one open"); + } + ret = j_backend_object_open(object_backend, parts[NAMESPACE], parts[NAME], &data); + } + else if (strcmp(op, "get_all") == 0) + { + if (data != NULL) + { + g_warning("open new object while still one open"); + } + ret = j_backend_object_get_all(object_backend, parts[NAMESPACE], data); + } + else if (strcmp(op, "ge_by_prefix") == 0) + { + if (data != NULL) + { + g_warning("open new object while still one open"); + } + ret = j_backend_kv_get_by_prefix(object_backend, parts[NAMESPACE], parts[NAME], &data); + } + else if (strcmp(op, "read") == 0) + { + static guint64 bytes_read = 0; + guint64 size = parse_ul(parts[SIZE]); + if (size > memory_chunk_size) + { + g_warning("unable to replay: chunk size to samll!%lu vs %lu", size, memory_chunk_size); + } + else + { + ret = j_backend_object_read(object_backend, data, memory_chunk, size, parse_ul(parts[COMPLEXITY]), &bytes_read); + } + } + else if (strcmp(op, "write") == 0) + { + static guint64 bytes_written = 0; + guint64 size = parse_ul(parts[SIZE]); + if (size > memory_chunk_size) + { + g_warning("unable to replay: chunk size to samll!%lu vs %lu", size, memory_chunk_size); + } + else + { + ret = j_backend_object_write(object_backend, data, memory_chunk, size, parse_ul(parts[COMPLEXITY]), &bytes_written); + } + } + else if (strcmp(op, "status") == 0) + { + static gint64 modification_time; + static guint64 size; + ret = j_backend_object_status(object_backend, data, &modification_time, &size); + } + else if (strcmp(op, "sync") == 0) + { + ret = j_backend_object_sync(object_backend, data); + } + else if (strcmp(op, "iterate") == 0) + { + static const char* name = NULL; + ret = j_backend_object_iterate(object_backend, data, &name); + if (!ret) + { + data = NULL; + ret = TRUE; + } + } + else if (strcmp(op, "close") == 0) + { + ret = j_backend_object_close(object_backend, data); + data = NULL; + } + else if (strcmp(op, "delete") == 0) + { + ret = j_backend_object_delete(object_backend, data); + data = NULL; + } + else if (strcmp(op, "init") == 0) + { + ret = TRUE; + } + else if (strcmp(op, "fini") == 0) + { + ret = TRUE; + } + else + { + g_warning("unkown object operation: '%s'", op); + } + return ret; } static gboolean -replay_kv(void* memory_chunk, guint64 memory_chunk_size, const char* parts[11]){ - gboolean ret = FALSE; - const char* op = parts[OPERATION]; - gpointer batch = NULL; - static JSemantics* semantics = NULL; - if (semantics == NULL) { semantics = j_semantics_new(J_SEMANTICS_TEMPLATE_DEFAULT); } +replay_kv(void* memory_chunk, guint64 memory_chunk_size, const char* parts[11]) +{ + gboolean ret = FALSE; + const char* op = parts[OPERATION]; + gpointer batch = NULL; + static JSemantics* semantics = NULL; + if (semantics == NULL) + { + semantics = j_semantics_new(J_SEMANTICS_TEMPLATE_DEFAULT); + } - if (strcmp(op, "batch_start") == 0) { - if (batch != NULL) { g_warning("starting a new batch, but old one is not finished"); } - ret = j_backend_kv_batch_start(kv_backend, parts[NAMESPACE], semantics, &batch); - } else if (strcmp(op, "put") == 0) { - guint64 size = parse_ul(parts[SIZE]); - if(size > memory_chunk_size){ - g_warning("unable to replay: chunk size to samll!%lu vs %lu", size, memory_chunk_size); - } else { - ret = j_backend_kv_put(kv_backend, batch, parts[NAME], memory_chunk, size); - } - } else if (strcmp(op, "delete") == 0) { - ret = j_backend_kv_delete(kv_backend, batch, parts[NAME]); - } else if (strcmp(op, "get") == 0) { - guint32 size; - gpointer value; - ret = j_backend_kv_get(kv_backend, batch, parts[NAME], &value, &size); - } else if (strcmp(op, "get_all") == 0) { - if (batch != NULL) { g_warning("start iterating with remaining batch or iteration"); } - ret = j_backend_kv_get_all(kv_backend, parts[NAMESPACE], &batch); - } else if (strcmp(op, "get_by_prefix") == 0) { - if (batch != NULL) { g_warning("start iterating with remaining batch or iteration"); } - ret = j_backend_kv_get_by_prefix(kv_backend, parts[NAMESPACE], parts[NAME], &batch); - } else if (strcmp(op, "iterate") == 0) { - const char* name; - gconstpointer value; - guint32 len; - ret = j_backend_kv_iterate(kv_backend, batch, &name, &value, &len); - if (!ret) { batch = NULL; ret = TRUE;} - } else if (strcmp(op, "init") == 0) { - ret = TRUE; - } else if (strcmp(op, "fini") == 0) { - ret = TRUE; - } else if (strcmp(op, "batch_execute") == 0) { - ret = j_backend_kv_batch_execute(kv_backend, batch); - } else { - g_warning("unkown operation: '%s'", op); - } - return ret; + if (strcmp(op, "batch_start") == 0) + { + if (batch != NULL) + { + g_warning("starting a new batch, but old one is not finished"); + } + ret = j_backend_kv_batch_start(kv_backend, parts[NAMESPACE], semantics, &batch); + } + else if (strcmp(op, "put") == 0) + { + guint64 size = parse_ul(parts[SIZE]); + if (size > memory_chunk_size) + { + g_warning("unable to replay: chunk size to samll!%lu vs %lu", size, memory_chunk_size); + } + else + { + ret = j_backend_kv_put(kv_backend, batch, parts[NAME], memory_chunk, size); + } + } + else if (strcmp(op, "delete") == 0) + { + ret = j_backend_kv_delete(kv_backend, batch, parts[NAME]); + } + else if (strcmp(op, "get") == 0) + { + guint32 size; + gpointer value; + ret = j_backend_kv_get(kv_backend, batch, parts[NAME], &value, &size); + } + else if (strcmp(op, "get_all") == 0) + { + if (batch != NULL) + { + g_warning("start iterating with remaining batch or iteration"); + } + ret = j_backend_kv_get_all(kv_backend, parts[NAMESPACE], &batch); + } + else if (strcmp(op, "get_by_prefix") == 0) + { + if (batch != NULL) + { + g_warning("start iterating with remaining batch or iteration"); + } + ret = j_backend_kv_get_by_prefix(kv_backend, parts[NAMESPACE], parts[NAME], &batch); + } + else if (strcmp(op, "iterate") == 0) + { + const char* name; + gconstpointer value; + guint32 len; + ret = j_backend_kv_iterate(kv_backend, batch, &name, &value, &len); + if (!ret) + { + batch = NULL; + ret = TRUE; + } + } + else if (strcmp(op, "init") == 0) + { + ret = TRUE; + } + else if (strcmp(op, "fini") == 0) + { + ret = TRUE; + } + else if (strcmp(op, "batch_execute") == 0) + { + ret = j_backend_kv_batch_execute(kv_backend, batch); + } + else + { + g_warning("unkown operation: '%s'", op); + } + return ret; } struct JSqlBatch { @@ -217,203 +327,292 @@ struct JSqlBatch gboolean aborted; }; static gboolean -replay_db(const char* parts[11]){ - gboolean ret = FALSE; - static gpointer batch = NULL; - static gpointer itr = NULL; - static gchar* namespace = NULL; - /// \TODO how to determine semantics used? - static JSemantics* semantics = NULL; - GError* error = NULL; - const char* op = parts[OPERATION]; - static GArray* bsons = NULL; - if (bsons == NULL) { bsons = g_array_sized_new(FALSE, TRUE, sizeof(bson_t*), 0); } - if (semantics == NULL) { semantics = j_semantics_new(J_SEMANTICS_TEMPLATE_DEFAULT); } +replay_db(const char* parts[11]) +{ + gboolean ret = FALSE; + static gpointer batch = NULL; + static gpointer itr = NULL; + static gchar* namespace = NULL; + /// \TODO how to determine semantics used? + static JSemantics* semantics = NULL; + GError* error = NULL; + const char* op = parts[OPERATION]; + static GArray* bsons = NULL; + if (bsons == NULL) + { + bsons = g_array_sized_new(FALSE, TRUE, sizeof(bson_t*), 0); + } + if (semantics == NULL) + { + semantics = j_semantics_new(J_SEMANTICS_TEMPLATE_DEFAULT); + } - if (strcmp(op, "batch_start") == 0) { - if (batch != NULL) { g_warning("starting a new batch, but old one is not finished"); } - namespace = strdup(parts[NAMESPACE]); - ret = j_backend_db_batch_start(db_backend, namespace, semantics, &batch, &error); - } else if (strcmp(op, "schema_create") == 0) { - bson_t* schema = bson_new_from_json((const uint8_t*)parts[JSON], -1, NULL); - ret = j_backend_db_schema_create(db_backend, batch, parts[NAME], schema, &error); - g_array_append_val(bsons, schema); - } else if (strcmp(op, "schema_get") == 0) { - bson_t* schema = bson_new(); - ret = j_backend_db_schema_get(db_backend, batch, parts[NAME], schema, &error); - g_array_append_val(bsons, schema); - // schema do not exists - if (error && error->code == 8) { - g_error_free(error); - error = NULL; - ret = TRUE; - } - } else if (strcmp(op, "schema_delete") == 0) { - ret = j_backend_db_schema_delete(db_backend, batch, parts[NAME], &error); - } else if (strcmp(op, "insert") == 0) { - bson_t* entry = bson_new_from_json((const uint8_t*)parts[JSON], -1, NULL); - bson_t* res = bson_new(); - ret = j_backend_db_insert(db_backend, batch, parts[NAME], entry, res, &error); - g_array_append_val(bsons, entry); - g_array_append_val(bsons, res); - } else if (strcmp(op, "delete") == 0) { - bson_t* entry = bson_new_from_json((const uint8_t*)parts[JSON], -1, NULL); - ret = j_backend_db_delete(db_backend, batch, parts[NAME], entry, &error); - g_array_append_val(bsons, entry); - } else if (strcmp(op, "update") == 0) { - bson_iter_t bson_iter; - bson_t* selector; - bson_t* entry; - const uint8_t* doc; - uint32_t len; - bson_t* bson = bson_new_from_json((const uint8_t*)parts[JSON], -1, NULL); - bson_iter_init_find(&bson_iter, bson, "entry"); - bson_iter_document(&bson_iter, &len, &doc); - entry = bson_new_from_data(doc, len); - bson_iter_init_find(&bson_iter, bson, "selector"); - bson_iter_document(&bson_iter, &len, &doc); - selector = bson_new_from_data(doc, len); - ret = j_backend_db_update(db_backend, batch, parts[NAME], selector, entry, &error); - g_array_append_val(bsons, entry); - g_array_append_val(bsons, selector); - g_array_append_val(bsons, bson); - } else if (strcmp(op, "query") == 0) { - bson_t* selector = bson_new_from_json((const uint8_t*)parts[JSON], -1, NULL); - if(itr != NULL) { g_warning("start new db iteration without finishing old one!"); } - ret = j_backend_db_query(db_backend, batch, parts[NAME], selector, &itr, &error); - g_array_append_val(bsons, selector); - } else if (strcmp(op, "iterate") == 0) { - bson_t* entry = bson_new(); - ret = j_backend_db_iterate(db_backend, itr, entry, &error); - g_array_append_val(bsons, entry); - if(!ret) { itr = NULL; ret = TRUE; g_error_free(error); error = NULL; } - } else if (strcmp(op, "init") == 0) { - ret = TRUE; - } else if (strcmp(op, "fini") == 0) { - ret = TRUE; - } else if (strcmp(op, "batch_execute") == 0) { - ret = j_backend_db_batch_execute(db_backend, batch, &error); - batch = NULL; - for(guint i = 0; i < bsons->len; ++i) { - bson_destroy(g_array_index(bsons, bson_t*, i)); - } - g_array_remove_range(bsons, 0, bsons->len); - // no operation to do - if (error && error->code == 7) { - g_error_free(error); - error = NULL; - ret = TRUE; - } - } else { - g_warning("unknown operation: '%s'", op); - } - if (error) { - g_warning("db error(%d): %s", error->code, error->message); - } - return ret && error == NULL; + if (strcmp(op, "batch_start") == 0) + { + if (batch != NULL) + { + g_warning("starting a new batch, but old one is not finished"); + } + namespace = strdup(parts[NAMESPACE]); + ret = j_backend_db_batch_start(db_backend, namespace, semantics, &batch, &error); + } + else if (strcmp(op, "schema_create") == 0) + { + bson_t* schema = bson_new_from_json((const uint8_t*)parts[JSON], -1, NULL); + ret = j_backend_db_schema_create(db_backend, batch, parts[NAME], schema, &error); + g_array_append_val(bsons, schema); + } + else if (strcmp(op, "schema_get") == 0) + { + bson_t* schema = bson_new(); + ret = j_backend_db_schema_get(db_backend, batch, parts[NAME], schema, &error); + g_array_append_val(bsons, schema); + // schema do not exists + if (error && error->code == 8) + { + g_error_free(error); + error = NULL; + ret = TRUE; + } + } + else if (strcmp(op, "schema_delete") == 0) + { + ret = j_backend_db_schema_delete(db_backend, batch, parts[NAME], &error); + } + else if (strcmp(op, "insert") == 0) + { + bson_t* entry = bson_new_from_json((const uint8_t*)parts[JSON], -1, NULL); + bson_t* res = bson_new(); + ret = j_backend_db_insert(db_backend, batch, parts[NAME], entry, res, &error); + g_array_append_val(bsons, entry); + g_array_append_val(bsons, res); + } + else if (strcmp(op, "delete") == 0) + { + bson_t* entry = bson_new_from_json((const uint8_t*)parts[JSON], -1, NULL); + ret = j_backend_db_delete(db_backend, batch, parts[NAME], entry, &error); + g_array_append_val(bsons, entry); + } + else if (strcmp(op, "update") == 0) + { + bson_iter_t bson_iter; + bson_t* selector; + bson_t* entry; + const uint8_t* doc; + uint32_t len; + bson_t* bson = bson_new_from_json((const uint8_t*)parts[JSON], -1, NULL); + bson_iter_init_find(&bson_iter, bson, "entry"); + bson_iter_document(&bson_iter, &len, &doc); + entry = bson_new_from_data(doc, len); + bson_iter_init_find(&bson_iter, bson, "selector"); + bson_iter_document(&bson_iter, &len, &doc); + selector = bson_new_from_data(doc, len); + ret = j_backend_db_update(db_backend, batch, parts[NAME], selector, entry, &error); + g_array_append_val(bsons, entry); + g_array_append_val(bsons, selector); + g_array_append_val(bsons, bson); + } + else if (strcmp(op, "query") == 0) + { + bson_t* selector = bson_new_from_json((const uint8_t*)parts[JSON], -1, NULL); + if (itr != NULL) + { + g_warning("start new db iteration without finishing old one!"); + } + ret = j_backend_db_query(db_backend, batch, parts[NAME], selector, &itr, &error); + g_array_append_val(bsons, selector); + } + else if (strcmp(op, "iterate") == 0) + { + bson_t* entry = bson_new(); + ret = j_backend_db_iterate(db_backend, itr, entry, &error); + g_array_append_val(bsons, entry); + if (!ret) + { + itr = NULL; + ret = TRUE; + g_error_free(error); + error = NULL; + } + } + else if (strcmp(op, "init") == 0) + { + ret = TRUE; + } + else if (strcmp(op, "fini") == 0) + { + ret = TRUE; + } + else if (strcmp(op, "batch_execute") == 0) + { + ret = j_backend_db_batch_execute(db_backend, batch, &error); + batch = NULL; + for (guint i = 0; i < bsons->len; ++i) + { + bson_destroy(g_array_index(bsons, bson_t*, i)); + } + g_array_remove_range(bsons, 0, bsons->len); + // no operation to do + if (error && error->code == 7) + { + g_error_free(error); + error = NULL; + ret = TRUE; + } + } + else + { + g_warning("unknown operation: '%s'", op); + } + if (error) + { + g_warning("db error(%d): %s", error->code, error->message); + } + return ret && error == NULL; } static gboolean replay(void* memory_chunk, guint64 memory_chunck_size, char* line) { - const char* parts[11]; - if (!split(line, parts)) { return FALSE; } - if(strcmp(parts[BACKEND], "object") == 0) { if(!replay_object(memory_chunk, memory_chunck_size, parts)) return FALSE; } - if(strcmp(parts[BACKEND], "kv") == 0) { if(!replay_kv(memory_chunk, memory_chunck_size, parts)) return FALSE; } - if(strcmp(parts[BACKEND], "db") == 0) { if(!replay_db(parts)) return FALSE; } - - return TRUE; + const char* parts[11]; + if (!split(line, parts)) + { + return FALSE; + } + if (strcmp(parts[BACKEND], "object") == 0) + { + if (!replay_object(memory_chunk, memory_chunck_size, parts)) + return FALSE; + } + if (strcmp(parts[BACKEND], "kv") == 0) + { + if (!replay_kv(memory_chunk, memory_chunck_size, parts)) + return FALSE; + } + if (strcmp(parts[BACKEND], "db") == 0) + { + if (!replay_db(parts)) + return FALSE; + } + + return TRUE; } int main(int argc, char** argv) { - JConfiguration* configuration = NULL; - const char* record_file_name = NULL; - FILE* record_file = NULL; - GModule* db_module = NULL; - GModule* kv_module = NULL; - GModule* object_module = NULL; - g_autofree gchar* port_str = NULL; - gboolean res = FALSE; - gchar* memory_chunk; - guint64 memory_chunck_size = 0; + JConfiguration* configuration = NULL; + const char* record_file_name = NULL; + FILE* record_file = NULL; + GModule* db_module = NULL; + GModule* kv_module = NULL; + GModule* object_module = NULL; + g_autofree gchar* port_str = NULL; + gboolean res = FALSE; + gchar* memory_chunk; + guint64 memory_chunck_size = 0; - setlocale(LC_ALL, "C.UTF-8"); + setlocale(LC_ALL, "C.UTF-8"); - configuration = j_configuration(); - if (configuration == NULL) { - g_warning("unable to read config"); - goto end; - if (argc < 2) { - g_warning("useage: %s ", argv[0]); - goto end; - } - } - record_file_name = argv[1]; - if (access(record_file_name, R_OK) != 0) { - g_warning("file does not exists, or no read permission '%s'", record_file_name); - goto end; - } - record_file = fopen(record_file_name, "r"); - if (record_file == NULL) { - g_warning("failed to open file '%s'", record_file_name); - goto end; - } + configuration = j_configuration(); + if (configuration == NULL) + { + g_warning("unable to read config"); + goto end; + if (argc < 2) + { + g_warning("useage: %s ", argv[0]); + goto end; + } + } + record_file_name = argv[1]; + if (access(record_file_name, R_OK) != 0) + { + g_warning("file does not exists, or no read permission '%s'", record_file_name); + goto end; + } + record_file = fopen(record_file_name, "r"); + if (record_file == NULL) + { + g_warning("failed to open file '%s'", record_file_name); + goto end; + } - // j_trace_init("julea-replay"); - // trace = j_trace_enter(G_STRFUNC, NULL); + // j_trace_init("julea-replay"); + // trace = j_trace_enter(G_STRFUNC, NULL); - port_str = g_strdup_printf("%d", j_configuration_get_port(configuration)); - if(!setup_backend(configuration, J_BACKEND_TYPE_OBJECT, port_str, &object_module, &object_backend)) { goto end; } - if(!setup_backend(configuration, J_BACKEND_TYPE_KV, port_str, &kv_module, &kv_backend)) { goto end; } - if(!setup_backend(configuration, J_BACKEND_TYPE_DB, port_str, &db_module, &db_backend)) { goto end; } + port_str = g_strdup_printf("%d", j_configuration_get_port(configuration)); + if (!setup_backend(configuration, J_BACKEND_TYPE_OBJECT, port_str, &object_module, &object_backend)) + { + goto end; + } + if (!setup_backend(configuration, J_BACKEND_TYPE_KV, port_str, &kv_module, &kv_backend)) + { + goto end; + } + if (!setup_backend(configuration, J_BACKEND_TYPE_DB, port_str, &db_module, &db_backend)) + { + goto end; + } - memory_chunck_size = j_configuration_get_max_operation_size(configuration); - memory_chunk = malloc(memory_chunck_size); + memory_chunck_size = j_configuration_get_max_operation_size(configuration); + memory_chunk = malloc(memory_chunck_size); - { - const char* header = "time,process_uid,program_name,backend,namespace,name,operation,size,complexity,duration,bson"; - char* line = NULL; - size_t len = 0; - ssize_t read = 0; - memset(memory_chunk, 0, memory_chunck_size); - read = getline(&line, &len, record_file); - line[read-1] = 0; - if (read == -1 || strcmp(line, header) != 0) { - g_warning("Invalid header in dump!\n'%s'\n'%s'\n", header, line); - goto end; - } - while ((read = getline(&line, &len, record_file)) != -1) { - line[read-1] = 0; - if (!replay(memory_chunk, memory_chunck_size, line)) { - g_warning("failed to replay dump!"); - goto end; - } - } - } - res = TRUE; + { + const char* header = "time,process_uid,program_name,backend,namespace,name,operation,size,complexity,duration,bson"; + char* line = NULL; + size_t len = 0; + ssize_t read = 0; + memset(memory_chunk, 0, memory_chunck_size); + read = getline(&line, &len, record_file); + line[read - 1] = 0; + if (read == -1 || strcmp(line, header) != 0) + { + g_warning("Invalid header in dump!\n'%s'\n'%s'\n", header, line); + goto end; + } + while ((read = getline(&line, &len, record_file)) != -1) + { + line[read - 1] = 0; + if (!replay(memory_chunk, memory_chunck_size, line)) + { + g_warning("failed to replay dump!"); + goto end; + } + } + } + res = TRUE; end: - if (record_file) { - fclose(record_file); - } - if (db_backend != NULL) - { - j_backend_db_fini(db_backend); - } - if (kv_backend != NULL) - { - j_backend_kv_fini(kv_backend); - } - if(object_backend != NULL) - { - j_backend_object_fini(object_backend); - } - if(db_module != NULL) { g_module_close(db_module); } - if(kv_module != NULL) { g_module_close(kv_module); } - if(object_module != NULL) { g_module_close(object_module); } - j_configuration_unref(configuration); - // j_trace_leave(trace); - // j_trace_fini(); - return res; + if (record_file) + { + fclose(record_file); + } + if (db_backend != NULL) + { + j_backend_db_fini(db_backend); + } + if (kv_backend != NULL) + { + j_backend_kv_fini(kv_backend); + } + if (object_backend != NULL) + { + j_backend_object_fini(object_backend); + } + if (db_module != NULL) + { + g_module_close(db_module); + } + if (kv_module != NULL) + { + g_module_close(kv_module); + } + if (object_module != NULL) + { + g_module_close(object_module); + } + j_configuration_unref(configuration); + // j_trace_leave(trace); + // j_trace_fini(); + return res; } From b79a64b3802e50e20af2365394107cb44e0f0c74 Mon Sep 17 00:00:00 2001 From: Julian Benda Date: Wed, 15 Feb 2023 16:43:19 +0100 Subject: [PATCH 09/40] Adding documentation for decission support --- doc/decission-support.md | 63 +++++++++++++++++++++++ lib/core/jbackend.c | 105 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 168 insertions(+) create mode 100644 doc/decission-support.md diff --git a/doc/decission-support.md b/doc/decission-support.md new file mode 100644 index 000000000..ede643272 --- /dev/null +++ b/doc/decission-support.md @@ -0,0 +1,63 @@ +# Decission Support + +JULEA is able to trace backend accesses granulare. This allows further +allplication analysies, and also analiesing object access across applictions. + +Further can this be used to provide a first rough estimate which backend combination +is prefarable for the application setup. + +## access record + +To create a `access-record` execute `julea-server` with `JULEA_TRACE=access` and +store the output in a file. +Then run your applications and stop the server iff finished. + +```sh +JULEA_TRACE=access julea-server 2> access-record.csv +``` + +The created file then can be used to executes replays. +A replay will execute the same backend operations in the same order as listed +in a `access-record`. This allows compare backend performance, without +executing the whole application. + +Important is that the backends are in the same state before the replay, then they +were before the processes where executed, the eseiast would be an empty state. + +## replay + +A `access-record` can be replayed with `julea-access-replay`. This program +need as argument a `access-record` file, to log the result you need to set +`JULEA_TRACE=access` to create them. + +It should be notice that the performnce of a replay can be vary to the original +data, depending on the load of the device. + +```sh +JULEA_TRACE=access julea-access-replay access-record.csv 2> replay-record.csv +``` + +## setup testing + +The script `./scripts/decission-support.sh` can help to select a optimal backend +combination for server. +It will test different backend configurations and creates a directory `evaluation`. + +The script will replay each access to a backend type, and test every servre +backend available for this type. And will produce multiple +`access-_.csv` files. + +This directory can then be evaluated for example with `./scripts/decision-support.R`. +A example flow is shown below. + +For scripts assumes that the backends are empty at the begging !! + +```sh +JULEA_TRACE=access julea-server 2> access-record.csv +# run application +# stop julea-server (Ctrl+C) +./scripts/decission-support.sh acces-record.csv evaluation +Rscript ./scripts/decission-support.R evaluation +``` + + diff --git a/lib/core/jbackend.c b/lib/core/jbackend.c index 63e34f5a8..c8987eebe 100644 --- a/lib/core/jbackend.c +++ b/lib/core/jbackend.c @@ -289,6 +289,9 @@ j_backend_object_init(JBackend* backend, gchar const* path) g_return_val_if_fail(path != NULL, FALSE); { + /** This macro call is used for access recording + * The name and argument order is therfore important! + * \sa j_trace_enter() */ J_TRACE("backend_object_init", "%s", path); ret = backend->object.backend_init(path, &(backend->data)); } @@ -305,6 +308,9 @@ j_backend_object_fini(JBackend* backend) g_return_if_fail(backend->type == J_BACKEND_TYPE_OBJECT); { + /** This macro call is used for access recording + * The name and argument order is therfore important! + * \sa j_trace_enter() */ J_TRACE("backend_object_fini", NULL); backend->object.backend_fini(backend->data); } @@ -324,6 +330,9 @@ j_backend_object_create(JBackend* backend, gchar const* namespace, gchar const* g_return_val_if_fail(data != NULL, FALSE); { + /** This macro call is used for access recording + * The name and argument order is therfore important! + * \sa j_trace_enter() */ J_TRACE("backend_object_create", "%s, %s, %p", namespace, path, (gpointer)data); ret = backend->object.backend_create(backend->data, namespace, path, data); } @@ -345,6 +354,9 @@ j_backend_object_open(JBackend* backend, gchar const* namespace, gchar const* pa g_return_val_if_fail(data != NULL, FALSE); { + /** This macro call is used for access recording + * The name and argument order is therfore important! + * \sa j_trace_enter() */ J_TRACE("backend_object_open", "%s, %s, %p", namespace, path, (gpointer)data); ret = backend->object.backend_open(backend->data, namespace, path, data); } @@ -364,6 +376,9 @@ j_backend_object_delete(JBackend* backend, gpointer data) g_return_val_if_fail(data != NULL, FALSE); { + /** This macro call is used for access recording + * The name and argument order is therfore important! + * \sa j_trace_enter() */ J_TRACE("backend_object_delete", "%p", data); ret = backend->object.backend_delete(backend->data, data); } @@ -384,6 +399,9 @@ j_backend_object_get_all(JBackend* backend, gchar const* namespace, gpointer* it g_return_val_if_fail(iterator != NULL, FALSE); { + /** This macro call is used for access recording + * The name and argument order is therfore important! + * \sa j_trace_enter() */ J_TRACE("backend_object_get_all", "%s, %p", namespace, (gpointer)iterator); ret = backend->object.backend_get_all(backend->data, namespace, iterator); } @@ -405,6 +423,9 @@ j_backend_object_get_by_prefix(JBackend* backend, gchar const* namespace, gchar g_return_val_if_fail(iterator != NULL, FALSE); { + /** This macro call is used for access recording + * The name and argument order is therfore important! + * \sa j_trace_enter() */ J_TRACE("backend_object_get_by_prefix", "%s, %s, %p", namespace, prefix, (gpointer)iterator); ret = backend->object.backend_get_by_prefix(backend->data, namespace, prefix, iterator); } @@ -424,6 +445,9 @@ j_backend_object_iterate(JBackend* backend, gpointer iterator, gchar const** nam g_return_val_if_fail(name != NULL, FALSE); { + /** This macro call is used for access recording + * The name and argument order is therfore important! + * \sa j_trace_enter() */ J_TRACE("backend_object_iterate", "%p, %p", iterator, (gpointer)name); ret = backend->object.backend_iterate(backend->data, iterator, name); } @@ -443,6 +467,9 @@ j_backend_object_close(JBackend* backend, gpointer data) g_return_val_if_fail(data != NULL, FALSE); { + /** This macro call is used for access recording + * The name and argument order is therfore important! + * \sa j_trace_enter() */ J_TRACE("backend_object_close", "%p", data); ret = backend->object.backend_close(backend->data, data); } @@ -464,6 +491,9 @@ j_backend_object_status(JBackend* backend, gpointer data, gint64* modification_t g_return_val_if_fail(size != NULL, FALSE); { + /** This macro call is used for access recording + * The name and argument order is therfore important! + * \sa j_trace_enter() */ J_TRACE("backend_object_status", "%p, %p, %p", data, (gpointer)modification_time, (gpointer)size); ret = backend->object.backend_status(backend->data, data, modification_time, size); } @@ -483,6 +513,9 @@ j_backend_object_sync(JBackend* backend, gpointer data) g_return_val_if_fail(data != NULL, FALSE); { + /** This macro call is used for access recording + * The name and argument order is therfore important! + * \sa j_trace_enter() */ J_TRACE("backend_object_sync", "%p", data); ret = backend->object.backend_sync(backend->data, data); } @@ -504,6 +537,9 @@ j_backend_object_read(JBackend* backend, gpointer data, gpointer buffer, guint64 g_return_val_if_fail(bytes_read != NULL, FALSE); { + /** This macro call is used for access recording + * The name and argument order is therfore important! + * \sa j_trace_enter() */ J_TRACE("backend_object_read", "%p, %p, %" G_GUINT64_FORMAT ", %" G_GUINT64_FORMAT ", %p", data, buffer, length, offset, (gpointer)bytes_read); ret = backend->object.backend_read(backend->data, data, buffer, length, offset, bytes_read); } @@ -525,6 +561,9 @@ j_backend_object_write(JBackend* backend, gpointer data, gconstpointer buffer, g g_return_val_if_fail(bytes_written != NULL, FALSE); { + /** This macro call is used for access recording + * The name and argument order is therfore important! + * \sa j_trace_enter() */ J_TRACE("backend_object_write", "%p, %p, %" G_GUINT64_FORMAT ", %" G_GUINT64_FORMAT ", %p", data, buffer, length, offset, (gpointer)bytes_written); ret = backend->object.backend_write(backend->data, data, buffer, length, offset, bytes_written); } @@ -544,6 +583,9 @@ j_backend_kv_init(JBackend* backend, gchar const* path) g_return_val_if_fail(path != NULL, FALSE); { + /** This macro call is used for access recording + * The name and argument order is therfore important! + * \sa j_trace_enter() */ J_TRACE("backend_kv_init", "%s", path); ret = backend->kv.backend_init(path, &(backend->data)); } @@ -560,6 +602,9 @@ j_backend_kv_fini(JBackend* backend) g_return_if_fail(backend->type == J_BACKEND_TYPE_KV); { + /** This macro call is used for access recording + * The name and argument order is therfore important! + * \sa j_trace_enter() */ J_TRACE("backend_kv_fini", NULL); backend->kv.backend_fini(backend->data); } @@ -579,6 +624,9 @@ j_backend_kv_batch_start(JBackend* backend, gchar const* namespace, JSemantics* g_return_val_if_fail(batch != NULL, FALSE); { + /** This macro call is used for access recording + * The name and argument order is therfore important! + * \sa j_trace_enter() */ J_TRACE("backend_kv_batch_start", "%s, %p, %p", namespace, (gpointer)semantics, (gpointer)batch); ret = backend->kv.backend_batch_start(backend->data, namespace, semantics, batch); } @@ -598,6 +646,9 @@ j_backend_kv_batch_execute(JBackend* backend, gpointer batch) g_return_val_if_fail(batch != NULL, FALSE); { + /** This macro call is used for access recording + * The name and argument order is therfore important! + * \sa j_trace_enter() */ J_TRACE("backend_kv_batch_execute", "%p", batch); ret = backend->kv.backend_batch_execute(backend->data, batch); } @@ -619,6 +670,9 @@ j_backend_kv_put(JBackend* backend, gpointer batch, gchar const* key, gconstpoin g_return_val_if_fail(value != NULL, FALSE); { + /** This macro call is used for access recording + * The name and argument order is therfore important! + * \sa j_trace_enter() */ J_TRACE("backend_kv_put", "%p, %s, %p, %u", batch, key, (gconstpointer)value, value_len); ret = backend->kv.backend_put(backend->data, batch, key, value, value_len); } @@ -639,6 +693,9 @@ j_backend_kv_delete(JBackend* backend, gpointer batch, gchar const* key) g_return_val_if_fail(key != NULL, FALSE); { + /** This macro call is used for access recording + * The name and argument order is therfore important! + * \sa j_trace_enter() */ J_TRACE("backend_kv_delete", "%p, %s", batch, key); ret = backend->kv.backend_delete(backend->data, batch, key); } @@ -661,6 +718,9 @@ j_backend_kv_get(JBackend* backend, gpointer batch, gchar const* key, gpointer* g_return_val_if_fail(value_len != NULL, FALSE); { + /** This macro call is used for access recording + * The name and argument order is therfore important! + * \sa j_trace_enter() */ J_TRACE("backend_kv_get", "%p, %s, %p, %p", batch, key, (gpointer)value, (gpointer)value_len); ret = backend->kv.backend_get(backend->data, batch, key, value, value_len); } @@ -681,6 +741,9 @@ j_backend_kv_get_all(JBackend* backend, gchar const* namespace, gpointer* iterat g_return_val_if_fail(iterator != NULL, FALSE); { + /** This macro call is used for access recording + * The name and argument order is therfore important! + * \sa j_trace_enter() */ J_TRACE("backend_kv_get_all", "%s, %p", namespace, (gpointer)iterator); ret = backend->kv.backend_get_all(backend->data, namespace, iterator); } @@ -702,6 +765,9 @@ j_backend_kv_get_by_prefix(JBackend* backend, gchar const* namespace, gchar cons g_return_val_if_fail(iterator != NULL, FALSE); { + /** This macro call is used for access recording + * The name and argument order is therfore important! + * \sa j_trace_enter() */ J_TRACE("backend_kv_get_by_prefix", "%s, %s, %p", namespace, prefix, (gpointer)iterator); ret = backend->kv.backend_get_by_prefix(backend->data, namespace, prefix, iterator); } @@ -723,6 +789,9 @@ j_backend_kv_iterate(JBackend* backend, gpointer iterator, gchar const** key, gc g_return_val_if_fail(value_len != NULL, FALSE); { + /** This macro call is used for access recording + * The name and argument order is therfore important! + * \sa j_trace_enter() */ J_TRACE("backend_kv_iterate", "%p, %p, %p, %p", iterator, (gpointer)key, (gpointer)value, (gpointer)value_len); ret = backend->kv.backend_iterate(backend->data, iterator, key, value, value_len); } @@ -742,6 +811,9 @@ j_backend_db_init(JBackend* backend, gchar const* path) g_return_val_if_fail(path != NULL, FALSE); { + /** This macro call is used for access recording + * The name and argument order is therfore important! + * \sa j_trace_enter() */ J_TRACE("backend_db_init", "%s", path); ret = backend->db.backend_init(path, &(backend->data)); } @@ -758,6 +830,9 @@ j_backend_db_fini(JBackend* backend) g_return_if_fail(backend->type == J_BACKEND_TYPE_DB); { + /** This macro call is used for access recording + * The name and argument order is therfore important! + * \sa j_trace_enter() */ J_TRACE("backend_db_fini", NULL); backend->db.backend_fini(backend->data); } @@ -778,6 +853,9 @@ j_backend_db_batch_start(JBackend* backend, gchar const* namespace, JSemantics* g_return_val_if_fail(error == NULL || *error == NULL, FALSE); { + /** This macro call is used for access recording + * The name and argument order is therfore important! + * \sa j_trace_enter() */ J_TRACE("backend_db_batch_start", "%s, %p, %p, %p", namespace, (gpointer)semantics, (gpointer)batch, (gpointer)error); ret = backend->db.backend_batch_start(backend->data, namespace, semantics, batch, error); } @@ -798,6 +876,9 @@ j_backend_db_batch_execute(JBackend* backend, gpointer batch, GError** error) g_return_val_if_fail(error == NULL || *error == NULL, FALSE); { + /** This macro call is used for access recording + * The name and argument order is therfore important! + * \sa j_trace_enter() */ J_TRACE("backend_db_batch_execute", "%p, %p", batch, (gpointer)error); ret = backend->db.backend_batch_execute(backend->data, batch, error); } @@ -820,6 +901,9 @@ j_backend_db_schema_create(JBackend* backend, gpointer batch, gchar const* name, g_return_val_if_fail(error == NULL || *error == NULL, FALSE); { + /** This macro call is used for access recording + * The name and argument order is therfore important! + * \sa j_trace_enter() */ J_TRACE("backend_db_schema_create", "%p, %s, %u, %p, %p", batch, name, schema->len, (gconstpointer)schema, (gpointer)error); ret = backend->db.backend_schema_create(backend->data, batch, name, schema, error); } @@ -841,6 +925,9 @@ j_backend_db_schema_get(JBackend* backend, gpointer batch, gchar const* name, bs g_return_val_if_fail(error == NULL || *error == NULL, FALSE); { + /** This macro call is used for access recording + * The name and argument order is therfore important! + * \sa j_trace_enter() */ J_TRACE("backend_db_schema_get", "%p, %s, %p, %p", batch, name, (gpointer)schema, (gpointer)error); ret = backend->db.backend_schema_get(backend->data, batch, name, schema, error); } @@ -862,6 +949,9 @@ j_backend_db_schema_delete(JBackend* backend, gpointer batch, gchar const* name, g_return_val_if_fail(error == NULL || *error == NULL, FALSE); { + /** This macro call is used for access recording + * The name and argument order is therfore important! + * \sa j_trace_enter() */ J_TRACE("backend_db_schema_delete", "%p, %s, %p", batch, name, (gpointer)error); ret = backend->db.backend_schema_delete(backend->data, batch, name, error); } @@ -885,6 +975,9 @@ j_backend_db_insert(JBackend* backend, gpointer batch, gchar const* name, bson_t g_return_val_if_fail(error == NULL || *error == NULL, FALSE); { + /** This macro call is used for access recording + * The name and argument order is therfore important! + * \sa j_trace_enter() */ J_TRACE("backend_db_insert", "%p, %s, %u, %p, %p, %p", batch, name, metadata->len, (gconstpointer)metadata, (gpointer)id, (gpointer)error); ret = backend->db.backend_insert(backend->data, batch, name, metadata, id, error); } @@ -908,6 +1001,9 @@ j_backend_db_update(JBackend* backend, gpointer batch, gchar const* name, bson_t g_return_val_if_fail(error == NULL || *error == NULL, FALSE); { + /** This macro call is used for access recording + * The name and argument order is therfore important! + * \sa j_trace_enter() */ J_TRACE("backend_db_update", "%p, %s, %u, %p, %p, %p", batch, name, metadata->len, (gconstpointer)selector, (gconstpointer)metadata, (gpointer)error); ret = backend->db.backend_update(backend->data, batch, name, selector, metadata, error); } @@ -929,6 +1025,9 @@ j_backend_db_delete(JBackend* backend, gpointer batch, gchar const* name, bson_t g_return_val_if_fail(error == NULL || *error == NULL, FALSE); { + /** This macro call is used for access recording + * The name and argument order is therfore important! + * \sa j_trace_enter() */ J_TRACE("backend_db_delete", "%p, %s, %u, %p, %p", batch, name, selector->len, (gconstpointer)selector, (gpointer)error); ret = backend->db.backend_delete(backend->data, batch, name, selector, error); } @@ -951,6 +1050,9 @@ j_backend_db_query(JBackend* backend, gpointer batch, gchar const* name, bson_t g_return_val_if_fail(error == NULL || *error == NULL, FALSE); { + /** This macro call is used for access recording + * The name and argument order is therfore important! + * \sa j_trace_enter() */ J_TRACE("backend_db_query", "%p, %s, %u, %p, %p, %p", batch, name, selector->len, (gconstpointer)selector, (gpointer)iterator, (gpointer)error); ret = backend->db.backend_query(backend->data, batch, name, selector, iterator, error); } @@ -972,6 +1074,9 @@ j_backend_db_iterate(JBackend* backend, gpointer iterator, bson_t* metadata, GEr g_return_val_if_fail(error == NULL || *error == NULL, FALSE); { + /** This macro call is used for access recording + * The name and argument order is therfore important! + * \sa j_trace_enter() */ J_TRACE("backend_db_iterate", "%p, %p, %p", iterator, (gpointer)metadata, (gpointer)error); ret = backend->db.backend_iterate(backend->data, iterator, metadata, error); } From 6f59e4cca414ace399e6af09893477ea2312d8ac Mon Sep 17 00:00:00 2001 From: Julian Benda Date: Thu, 16 Feb 2023 19:19:19 +0100 Subject: [PATCH 10/40] Add script --- doc/decission-support.md | 11 ++- scripts/decission-support.sh | 160 +++++++++++++++++++++++++++++++++++ tools/access_replay.c | 8 +- 3 files changed, 171 insertions(+), 8 deletions(-) create mode 100755 scripts/decission-support.sh diff --git a/doc/decission-support.md b/doc/decission-support.md index ede643272..210fbbf28 100644 --- a/doc/decission-support.md +++ b/doc/decission-support.md @@ -44,20 +44,19 @@ combination for server. It will test different backend configurations and creates a directory `evaluation`. The script will replay each access to a backend type, and test every servre -backend available for this type. And will produce multiple -`access-_.csv` files. +backend available for this type. And will produce a `summary.csv` -This directory can then be evaluated for example with `./scripts/decision-support.R`. +This summary can then be evaluated for example with `./scripts/decision-support.R`. A example flow is shown below. -For scripts assumes that the backends are empty at the begging !! +For script assumes that the backends are empty at the begining !! ```sh JULEA_TRACE=access julea-server 2> access-record.csv # run application # stop julea-server (Ctrl+C) -./scripts/decission-support.sh acces-record.csv evaluation -Rscript ./scripts/decission-support.R evaluation +./scripts/decission-support.sh acces-record.csv summary.csv +Rscript ./scripts/decission-support.R summary.csv ``` diff --git a/scripts/decission-support.sh b/scripts/decission-support.sh new file mode 100755 index 000000000..9af393671 --- /dev/null +++ b/scripts/decission-support.sh @@ -0,0 +1,160 @@ +#!/bin/sh + +# JULEA - Flexible storage framework +# Copyright (C) 2023-2023 Julian Benda +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program. If not, see . + +SELF_PATH="$(readlink --canonicalize-existing -- "$0")" +SELF_DIR="${SELF_PATH%/*}" +SELF_BASE="${SELF_PATH##*/}" + +HOST=$(hostname) + +function usage() +{ + echo "Usage: ${SELF_BASE} " + echo "\tfurther details can be found in doc/decission-support.md" + exit 1 +} + +function create_config() +{ + + IFS=';' read -r backend path <<< "$2" + cat > $3 <<- EOM + [core] + max-operation-size=0 + max-inject-size=0 + port=0 + + [clients] + max-connections=0 + stripe-size=0 + + [servers] + object=${HOST}; + kv=${HOST}; + db=${HOST}; + + [object] + backend=${backend} + component=$(if [ "$1" == "object" ]; then echo "server"; else echo "client"; fi) + path=${path} + + [kv] + backend=${backend} + component=$(if [ "$1" == "kv" ]; then echo "server"; else echo "client"; fi) + path=${path} + + [db] + backend=${backend} + component=$(if [ "$1" == "db" ]; then echo "server"; else echo "client"; fi) + path=${path} +EOM +} + +# check input parameters +if [ -z ${1+x} ] || [ -z ${2+x} ]; then + echo "too view arguments provided" + usage + exit 1 +fi + +if ! [ -r "$1" ]; then + echo "file '$1' does not exist, or is not readable!" + usage + exit 1 +fi + +if ! touch "$2"; then + echo "unable to create/modify output file '$2'" + usage + exit 1 +fi + +input_file="$1" +output_file="$2" +tmp_dir="$(mktemp -d)" + +# split record in parts for different backends +db_record="${tmp_dir}/db.csv" +kv_record="${tmp_dir}/kv.csv" +object_record="${tmp_dir}/object.csv" + +awk -F ',' 'BEGIN {OFS=","} { if(NR==1 || $4 == "db") print}' $input_file > $db_record +awk -F ',' 'BEGIN {OFS=","} { if(NR==1 || $4 == "kv") print}' $input_file > $kv_record +awk -F ',' 'BEGIN {OFS=","} { if(NR==1 || $4 == "object") print}' $input_file > $object_record + +tmp_record="${tmp_dir}/access-record.csv" +tmp_config="${tmp_dir}/config" +echo "time,process_uid,program_name,backend,type,path,namespace,name,operation,size,complexity,duration,bson" > $output_file + +fail=0 + +echo "start db" +arr=( + "sqlite;${tmp_dir}/sqlite" + "sqlite;:memory:" + "mysql;127.0.0.1:julea_db:julea_user:julea_pw" +) +for db in ${arr[@]}; do + echo -e "\t round ${db}" + create_config "db" "${db}" "${tmp_config}" + JULEA_CONFIG="${tmp_config}" JULEA_TRACE=access julea-access-replay "${db_record}" 2> "${tmp_record}" + type="$(tr ';' ',' <<< "${db}")" + tail -n +2 "${tmp_record}" | awk -F ',' 'BEGIN {OFS=","} { if(NR>1) { $4 = $4",'"${type}"'"} print }' >> "${output_file}" +done + +echo "start object" +arr=( + "gio;${tmp_dir}/gio" + "posix;${tmp_dir}/posix" +) +for object in ${arr[@]}; do + echo -e "\t round ${object}" + create_config "object" "${object}" "${tmp_config}" + JULEA_CONFIG="${tmp_config}" JULEA_TRACE=access julea-access-replay "${object_record}" 2> "${tmp_record}" + type="$(tr ';' ',' <<< "${object}")" + tail -n +2 "${tmp_record}" | awk -F ',' 'BEGIN {OFS=","} { if(NR>1) { $4 = $4",'"${type}"'"} print }' >> "${output_file}" +done + +echo "start kv" +arr=( + "leveldb;${tmp_dir}/leveldb" + "lmdb;${tmp_dir}/lmdb" + "sqlite;${tmp_dir}/kv_sqlite" + "sqlite;${tmp_dir}/:memory:" + "rocksdb;${tmp_dir}/rocksdb" +) +for kv in ${arr[@]}; do + echo -e "\t round ${kv}" + create_config "kv" "${kv}" "${tmp_config}" + JULEA_CONFIG="${tmp_config}" JULEA_TRACE=access julea-access-replay "${kv_record}" 2> "${tmp_record}" + if [ $? -ne 0 ]; then + echo -e "failed to execute! with:\n" + tail -n +2 "${tmp_record}" + fail=$(($fail + 1)) + continue + fi + type="$(tr ';' ',' <<< "${kv}")" + tail -n +2 "${tmp_record}" | awk -F ',' 'BEGIN {OFS=","} { if(NR>1) { $4 = $4",'"${type}"'"} print }' >> "${output_file}" +done + +if [ $fail -ne 0 ]; then + echo "${fail} configurations failed, the resuming are stored in '${output_file}' an can be used further" +else + echo "end decission support tool" +fi +rm -r "${tmp_dir}" diff --git a/tools/access_replay.c b/tools/access_replay.c index 726e3dee2..a6ce3a86b 100644 --- a/tools/access_replay.c +++ b/tools/access_replay.c @@ -41,6 +41,7 @@ setup_backend(JConfiguration* configuration, JBackendType type, gchar const* por component = j_configuration_get_backend_component(configuration, type); path = j_helper_str_replace(j_configuration_get_backend_path(configuration, type), "{PORT}", port_str); + if (strcmp(component, "server") != 0) { return TRUE; } if (j_backend_load_server(backend, component, type, module, j_backend)) { gboolean res = TRUE; @@ -508,7 +509,7 @@ main(int argc, char** argv) GModule* kv_module = NULL; GModule* object_module = NULL; g_autofree gchar* port_str = NULL; - gboolean res = FALSE; + gint res = 1; gchar* memory_chunk; guint64 memory_chunck_size = 0; @@ -544,14 +545,17 @@ main(int argc, char** argv) port_str = g_strdup_printf("%d", j_configuration_get_port(configuration)); if (!setup_backend(configuration, J_BACKEND_TYPE_OBJECT, port_str, &object_module, &object_backend)) { + g_warning("failed to initealize object backend"); goto end; } if (!setup_backend(configuration, J_BACKEND_TYPE_KV, port_str, &kv_module, &kv_backend)) { + g_warning("failed to initealize kv backend"); goto end; } if (!setup_backend(configuration, J_BACKEND_TYPE_DB, port_str, &db_module, &db_backend)) { + g_warning("failed to initealize db backend"); goto end; } @@ -581,7 +585,7 @@ main(int argc, char** argv) } } } - res = TRUE; + res = 0; end: if (record_file) { From 408ad8efba533bd98a1df20beac09328c97295b8 Mon Sep 17 00:00:00 2001 From: Julian Benda Date: Thu, 16 Feb 2023 20:19:20 +0100 Subject: [PATCH 11/40] Add evalation script valiadation --- scripts/decission-support.R | 60 ++++++++++++++++++++++++++++++++++++ scripts/decission-support.sh | 6 ++-- 2 files changed, 63 insertions(+), 3 deletions(-) create mode 100644 scripts/decission-support.R diff --git a/scripts/decission-support.R b/scripts/decission-support.R new file mode 100644 index 000000000..160a4d912 --- /dev/null +++ b/scripts/decission-support.R @@ -0,0 +1,60 @@ +#!/usr/bin/env /Rscript + +suppressMessages(library(ggplot2)) +suppressMessages(library(dplyr)) + + +first_round <- TRUE +last_frame <- NA + +# test if all records of the same backend kind (object,kv,db) +# have executed the same order of commands +test_validity <- function(data) { + res <- TRUE + data <- subset( + data, + select = -c(time, process_uid, program_name, bson, duration)) # nolint: object_usage_linter, line_length_linter. + for (x in unique(data$backend)) { + first_round <<- TRUE + check <- function(frame, key) { + if (res == FALSE) { + return() + } + frame <- frame %>% subset(select = -c(backend)) # nolint: object_usage_linter, line_length_linter. + if (first_round != TRUE) { + if (nrow(frame) != nrow(last_frame)) { + res <<- FALSE + } else if (!Reduce(f = "&", x = frame == last_frame)) { + res <<- FALSE + } + } + first_round <<- FALSE + last_frame <<- frame + } + data %>% filter(backend == x) %>% group_by(type, path) %>% group_walk(check) # nolint: object_usage_linter, line_length_linter. + } + return(res) +} + + +args <- commandArgs(trailingOnly = TRUE) +print(args) +if (length(args) < 1) { + stop( + paste( + "usage:", + "script", # TODO check if script name can be determined + " <- generated by decission-support.sh"), + call. = FALSE) +} + +data <- read.csv(args[1]) + +res <- test_validity(data) +if (res == FALSE) { + stop( + "Access reports between different configurations are not consistent!", + call. = FALSE) +} + + diff --git a/scripts/decission-support.sh b/scripts/decission-support.sh index 9af393671..297c5fb1b 100755 --- a/scripts/decission-support.sh +++ b/scripts/decission-support.sh @@ -114,7 +114,7 @@ for db in ${arr[@]}; do create_config "db" "${db}" "${tmp_config}" JULEA_CONFIG="${tmp_config}" JULEA_TRACE=access julea-access-replay "${db_record}" 2> "${tmp_record}" type="$(tr ';' ',' <<< "${db}")" - tail -n +2 "${tmp_record}" | awk -F ',' 'BEGIN {OFS=","} { if(NR>1) { $4 = $4",'"${type}"'"} print }' >> "${output_file}" + tail -n +2 "${tmp_record}" | awk -F ',' 'BEGIN {OFS=","} { $4 = $4",'"${type}"'"; print }' >> "${output_file}" done echo "start object" @@ -127,7 +127,7 @@ for object in ${arr[@]}; do create_config "object" "${object}" "${tmp_config}" JULEA_CONFIG="${tmp_config}" JULEA_TRACE=access julea-access-replay "${object_record}" 2> "${tmp_record}" type="$(tr ';' ',' <<< "${object}")" - tail -n +2 "${tmp_record}" | awk -F ',' 'BEGIN {OFS=","} { if(NR>1) { $4 = $4",'"${type}"'"} print }' >> "${output_file}" + tail -n +2 "${tmp_record}" | awk -F ',' 'BEGIN {OFS=","} { $4 = $4",'"${type}"'"; print}' >> "${output_file}" done echo "start kv" @@ -149,7 +149,7 @@ for kv in ${arr[@]}; do continue fi type="$(tr ';' ',' <<< "${kv}")" - tail -n +2 "${tmp_record}" | awk -F ',' 'BEGIN {OFS=","} { if(NR>1) { $4 = $4",'"${type}"'"} print }' >> "${output_file}" + tail -n +2 "${tmp_record}" | awk -F ',' 'BEGIN {OFS=","} { $4 = $4",'"${type}"'"; print }' >> "${output_file}" done if [ $fail -ne 0 ]; then From 10c30a8887f542710a3ddba841ce8cb8ee110bd4 Mon Sep 17 00:00:00 2001 From: Julian Benda Date: Fri, 17 Feb 2023 13:10:04 +0100 Subject: [PATCH 12/40] Add plotting evaluation for decission support summary --- scripts/decission-support.R | 38 ++++++++++++++++++++++++++++++++++++ scripts/decission-support.sh | 8 ++++---- 2 files changed, 42 insertions(+), 4 deletions(-) diff --git a/scripts/decission-support.R b/scripts/decission-support.R index 160a4d912..659226351 100644 --- a/scripts/decission-support.R +++ b/scripts/decission-support.R @@ -2,6 +2,8 @@ suppressMessages(library(ggplot2)) suppressMessages(library(dplyr)) +suppressMessages(library(scales)) +suppressMessages(library(ggpubr)) first_round <- TRUE @@ -23,8 +25,10 @@ test_validity <- function(data) { frame <- frame %>% subset(select = -c(backend)) # nolint: object_usage_linter, line_length_linter. if (first_round != TRUE) { if (nrow(frame) != nrow(last_frame)) { + print("number error, the number of accesses differ between runs") res <<- FALSE } else if (!Reduce(f = "&", x = frame == last_frame)) { + print("value error, the access type between runs differ") res <<- FALSE } } @@ -57,4 +61,38 @@ if (res == FALSE) { call. = FALSE) } +plot_durations <- function(data, key) { + confs <- unique(data %>% select("type", "path")) + map <- apply( + confs, 1, + function(x) paste(x[1], if (sum(confs == x[1]) > 1) x[2] else "")) + names(map) <- paste0(confs$type, confs$path) + data$label <- map[paste0(data$type, data$path)] + label <- labs(y = "time in s", fill = "backend", x = "") + seperate <- + ggplot( + data %>% + group_by(label, operation) %>% + summarise(total_time = sum(duration), cnt = n(), .groups = "drop"), + aes(y = total_time, x = paste(operation, "\n#", cnt), fill = label)) + + geom_col(position = "dodge") + + theme(axis.title.x = element_blank(), + axis.text.x = element_text(angle = 45)) + + label + total <- + ggplot( + data %>% + group_by(label) %>% + summarise(total_time = sum(duration), .groups = "drop"), + aes(y = total_time, x = " all operations", fill = label)) + + geom_col(position = "dodge") + + label + plot <- ggarrange(seperate, total, + widths = c(4, 1), ncol = 2, nrow = 1, + common.legend = TRUE, legend = "bottom") + ggsave(paste(key[1], "time.svg", sep = "_"), plot, + unit = "in", width = 11, height = 6, bg = "white") +} + +data %>% group_by(backend) %>% group_walk(plot_durations) diff --git a/scripts/decission-support.sh b/scripts/decission-support.sh index 297c5fb1b..f7aa6cce9 100755 --- a/scripts/decission-support.sh +++ b/scripts/decission-support.sh @@ -113,7 +113,7 @@ for db in ${arr[@]}; do echo -e "\t round ${db}" create_config "db" "${db}" "${tmp_config}" JULEA_CONFIG="${tmp_config}" JULEA_TRACE=access julea-access-replay "${db_record}" 2> "${tmp_record}" - type="$(tr ';' ',' <<< "${db}")" + type="$(tr ';' ',' <<< "${db}" | sed 's/'$(basename ${tmp_dir})'\///')" tail -n +2 "${tmp_record}" | awk -F ',' 'BEGIN {OFS=","} { $4 = $4",'"${type}"'"; print }' >> "${output_file}" done @@ -126,7 +126,7 @@ for object in ${arr[@]}; do echo -e "\t round ${object}" create_config "object" "${object}" "${tmp_config}" JULEA_CONFIG="${tmp_config}" JULEA_TRACE=access julea-access-replay "${object_record}" 2> "${tmp_record}" - type="$(tr ';' ',' <<< "${object}")" + type="$(tr ';' ',' <<< "${object}" | sed 's/'$(basename ${tmp_dir})'\///')" tail -n +2 "${tmp_record}" | awk -F ',' 'BEGIN {OFS=","} { $4 = $4",'"${type}"'"; print}' >> "${output_file}" done @@ -135,7 +135,7 @@ arr=( "leveldb;${tmp_dir}/leveldb" "lmdb;${tmp_dir}/lmdb" "sqlite;${tmp_dir}/kv_sqlite" - "sqlite;${tmp_dir}/:memory:" + "sqlite;:memory:" "rocksdb;${tmp_dir}/rocksdb" ) for kv in ${arr[@]}; do @@ -148,7 +148,7 @@ for kv in ${arr[@]}; do fail=$(($fail + 1)) continue fi - type="$(tr ';' ',' <<< "${kv}")" + type="$(tr ';' ',' <<< "${kv}" | sed 's/'$(basename ${tmp_dir})'\///')" tail -n +2 "${tmp_record}" | awk -F ',' 'BEGIN {OFS=","} { $4 = $4",'"${type}"'"; print }' >> "${output_file}" done From effe650615950df5a52ed77029036f50e41bef57 Mon Sep 17 00:00:00 2001 From: Julian Benda Date: Fri, 17 Feb 2023 13:47:12 +0100 Subject: [PATCH 13/40] start adding tables --- scripts/decission-support.R | 38 ++++++++++++++++++++++++------------- 1 file changed, 25 insertions(+), 13 deletions(-) diff --git a/scripts/decission-support.R b/scripts/decission-support.R index 659226351..e3e11b1c0 100644 --- a/scripts/decission-support.R +++ b/scripts/decission-support.R @@ -62,32 +62,44 @@ if (res == FALSE) { } plot_durations <- function(data, key) { + # generate shorted backend label confs <- unique(data %>% select("type", "path")) map <- apply( confs, 1, function(x) paste(x[1], if (sum(confs == x[1]) > 1) x[2] else "")) names(map) <- paste0(confs$type, confs$path) - data$label <- map[paste0(data$type, data$path)] + data$backend <- map[paste0(data$type, data$path)] label <- labs(y = "time in s", fill = "backend", x = "") - seperate <- - ggplot( - data %>% - group_by(label, operation) %>% - summarise(total_time = sum(duration), cnt = n(), .groups = "drop"), - aes(y = total_time, x = paste(operation, "\n#", cnt), fill = label)) + + writeLines(paste("Backend: ", key[1])) + # analyse for runtime per operation type + sepperate_data <- data %>% + group_by(backend, operation) %>% + summarise(duration = sum(duration), cnt = n(), .groups = "drop") + # plotting + seperate <- ggplot(sepperate_data, + aes(y = duration, x = paste(operation, "\n#", cnt), fill = backend)) + geom_col(position = "dodge") + theme(axis.title.x = element_blank(), axis.text.x = element_text(angle = 45)) + label - total <- - ggplot( - data %>% - group_by(label) %>% - summarise(total_time = sum(duration), .groups = "drop"), - aes(y = total_time, x = " all operations", fill = label)) + + # table + # print(knitr::kable(head(sepperate_data))) + + # analyse for total runtime + total_data <- data %>% + group_by(backend) %>% + summarise(duration = sum(duration), .groups = "drop") + # plotting + total <- ggplot(total_data, + aes(y = duration, x = "all operations", fill = backend)) + geom_col(position = "dodge") + label + # table + print(knitr::kable(head(total_data))) + writeLines("\n\n") + + # combining plots and store them plot <- ggarrange(seperate, total, widths = c(4, 1), ncol = 2, nrow = 1, common.legend = TRUE, legend = "bottom") From 4436e35e5e3d1f522d8a994d47e80a644cc50497 Mon Sep 17 00:00:00 2001 From: Julian Benda Date: Mon, 20 Feb 2023 13:19:43 +0100 Subject: [PATCH 14/40] Adding plotting and table code in R script --- doc/decission-support.md | 7 +++- scripts/decission-support.R | 67 ++++++++++++++++++++++++++++++------- 2 files changed, 61 insertions(+), 13 deletions(-) mode change 100644 => 100755 scripts/decission-support.R diff --git a/doc/decission-support.md b/doc/decission-support.md index 210fbbf28..1facdbca8 100644 --- a/doc/decission-support.md +++ b/doc/decission-support.md @@ -49,13 +49,18 @@ backend available for this type. And will produce a `summary.csv` This summary can then be evaluated for example with `./scripts/decision-support.R`. A example flow is shown below. -For script assumes that the backends are empty at the begining !! +Important notice: +* the script assums a empty backend start state +* the script places backends and temporary files in `/tmp/` support for differnce devices is pending +* a mysql/mariadb instance must be provided, with the user julea_user, pw: julea_pw which needs access to the database julea_db ```sh JULEA_TRACE=access julea-server 2> access-record.csv # run application # stop julea-server (Ctrl+C) ./scripts/decission-support.sh acces-record.csv summary.csv +Rscript ./scripts/decission-support.R summary.csv html > summary.html +# or only ci output Rscript ./scripts/decission-support.R summary.csv ``` diff --git a/scripts/decission-support.R b/scripts/decission-support.R old mode 100644 new mode 100755 index e3e11b1c0..57cbd34ec --- a/scripts/decission-support.R +++ b/scripts/decission-support.R @@ -5,6 +5,7 @@ suppressMessages(library(dplyr)) suppressMessages(library(scales)) suppressMessages(library(ggpubr)) +options(knitr.kable.NA = "") first_round <- TRUE last_frame <- NA @@ -25,10 +26,10 @@ test_validity <- function(data) { frame <- frame %>% subset(select = -c(backend)) # nolint: object_usage_linter, line_length_linter. if (first_round != TRUE) { if (nrow(frame) != nrow(last_frame)) { - print("number error, the number of accesses differ between runs") + warning("number error, the number of accesses differ between runs") res <<- FALSE } else if (!Reduce(f = "&", x = frame == last_frame)) { - print("value error, the access type between runs differ") + warning("value error, the access type between runs differ") res <<- FALSE } } @@ -42,17 +43,18 @@ test_validity <- function(data) { args <- commandArgs(trailingOnly = TRUE) -print(args) if (length(args) < 1) { stop( paste( "usage:", "script", # TODO check if script name can be determined - " <- generated by decission-support.sh"), + " <- generated by decission-support.sh", + "[(print|html)] <- output as simple print(default) or as html", sep="\n"), call. = FALSE) } data <- read.csv(args[1]) +as_html <- length(args) >= 2 & args[2] == "html" res <- test_validity(data) if (res == FALSE) { @@ -61,7 +63,15 @@ if (res == FALSE) { call. = FALSE) } -plot_durations <- function(data, key) { +plot_durations <- function(data, key,...,format="simple") { + # check and convert duration unit as sensible + unit <- "s" + if (max(data$duration) < 10.) { + data$duration <- data$duration * 1000. + unit <- "ms" + } + label <- labs(y = paste0("time in ", unit), fill = "backend", x = "") + # generate shorted backend label confs <- unique(data %>% select("type", "path")) map <- apply( @@ -70,8 +80,6 @@ plot_durations <- function(data, key) { names(map) <- paste0(confs$type, confs$path) data$backend <- map[paste0(data$type, data$path)] - label <- labs(y = "time in s", fill = "backend", x = "") - writeLines(paste("Backend: ", key[1])) # analyse for runtime per operation type sepperate_data <- data %>% group_by(backend, operation) %>% @@ -84,7 +92,17 @@ plot_durations <- function(data, key) { axis.text.x = element_text(angle = 45)) + label # table - # print(knitr::kable(head(sepperate_data))) + sepperate_data <- sepperate_data %>% + group_by(operation) %>% + group_modify(function(df, keys) { + df$speed_up_rel <- max(df$duration) / df$duration + df$speed_up_abs <- max(df$duration) - df$duration + df <- df %>% arrange(duration) %>% add_row(.before=0) + return(df)}) %>% + ungroup() + sepperate_data$operation <- reorder(factor(sepperate_data$operation), sepperate_data$speed_up_abs, FUN=max, decreasing=TRUE, na.rm = TRUE) + sepperate_data <- arrange(sepperate_data, operation) + sepperate_data[! is.na(sepperate_data$duration),]$operation = NA # analyse for total runtime total_data <- data %>% @@ -96,15 +114,40 @@ plot_durations <- function(data, key) { geom_col(position = "dodge") + label # table - print(knitr::kable(head(total_data))) - writeLines("\n\n") + total_data$speed_up_rec <- max(total_data$duration) / total_data$duration + total_data$speed_up_abs <- max(total_data$duration) - total_data$duration + + # output table + plot_file_name <- paste(key[1], "time.svg", sep = "_") + table_sepperate_str <- knitr::kable(sepperate_data, format, digits=2) + table_total_str <- knitr::kable(total_data, format, digits=2) + if (format == "html") { + cat("

",as.character(key[1]),"

", sep = "") + cat("
",table_sepperate_str,"", table_total_str,"
\"barplot
", sep = "") + } else { + cat(as.character(key[1]), paste0("plot: ", plot_file_name), "", table_total_str, "", table_sepperate_str, "", "", sep="\n") + } # combining plots and store them plot <- ggarrange(seperate, total, widths = c(4, 1), ncol = 2, nrow = 1, common.legend = TRUE, legend = "bottom") - ggsave(paste(key[1], "time.svg", sep = "_"), plot, + ggsave(plot_file_name, plot, unit = "in", width = 11, height = 6, bg = "white") } -data %>% group_by(backend) %>% group_walk(plot_durations) +# print html header +if (as_html) { + cat("
    ") + for ( backend in unique(data$backend)) { + cat("
  • ", backend,"
    • ", sep="") + } + cat("
") +} + +data %>% group_by(backend) %>% group_walk(plot_durations, format=if(as_html) "html" else "simple") + +# print html closing +if (as_html) { + cat("") +} From cddc2749a200029adaf0beb14b6b113ea22ed0bb Mon Sep 17 00:00:00 2001 From: Julian Benda Date: Tue, 21 Feb 2023 15:41:08 +0100 Subject: [PATCH 15/40] Report design changes --- doc/decission-support.md | 14 +++++++++++++- scripts/decission-support.R | 28 +++++++++++++++++----------- scripts/decission-support.sh | 3 +++ 3 files changed, 33 insertions(+), 12 deletions(-) diff --git a/doc/decission-support.md b/doc/decission-support.md index 1facdbca8..07e0d923f 100644 --- a/doc/decission-support.md +++ b/doc/decission-support.md @@ -58,7 +58,19 @@ Important notice: JULEA_TRACE=access julea-server 2> access-record.csv # run application # stop julea-server (Ctrl+C) -./scripts/decission-support.sh acces-record.csv summary.csv +./scripts/decission-support.py acces-record.csv summary.csv config.json +{ + "object": [ + {"backend": "posix", "path": "/tmp//"}, + {"backend": "posix", "path": "/mnt/abc"} + ], + "kv": [ + {""} + ], + "object": [ + {""} + ] +} Rscript ./scripts/decission-support.R summary.csv html > summary.html # or only ci output Rscript ./scripts/decission-support.R summary.csv diff --git a/scripts/decission-support.R b/scripts/decission-support.R index 57cbd34ec..45b5f503f 100755 --- a/scripts/decission-support.R +++ b/scripts/decission-support.R @@ -95,12 +95,13 @@ plot_durations <- function(data, key,...,format="simple") { sepperate_data <- sepperate_data %>% group_by(operation) %>% group_modify(function(df, keys) { - df$speed_up_rel <- max(df$duration) / df$duration - df$speed_up_abs <- max(df$duration) - df$duration + df$speed_up <- max(df$duration) / df$duration + df$time_diff <- max(df$duration) - df$duration df <- df %>% arrange(duration) %>% add_row(.before=0) return(df)}) %>% ungroup() - sepperate_data$operation <- reorder(factor(sepperate_data$operation), sepperate_data$speed_up_abs, FUN=max, decreasing=TRUE, na.rm = TRUE) + sepperate_data$operation <- reorder(factor(sepperate_data$operation), sepperate_data$time_diff, FUN=max, decreasing=TRUE, na.rm = TRUE) + sepperate_data <- arrange(sepperate_data, operation) sepperate_data[! is.na(sepperate_data$duration),]$operation = NA @@ -114,16 +115,19 @@ plot_durations <- function(data, key,...,format="simple") { geom_col(position = "dodge") + label # table - total_data$speed_up_rec <- max(total_data$duration) / total_data$duration - total_data$speed_up_abs <- max(total_data$duration) - total_data$duration + total_data$speed_up <- max(total_data$duration) / total_data$duration + total_data$time_diff <- max(total_data$duration) - total_data$duration + total_data <- arrange(total_data, duration) + colnames(total_data)[which(names(total_data) == "duration")] <- "duration (ms)" + colnames(sepperate_data)[which(names(sepperate_data) == "duration")] <- "duration (ms)" # output table plot_file_name <- paste(key[1], "time.svg", sep = "_") - table_sepperate_str <- knitr::kable(sepperate_data, format, digits=2) - table_total_str <- knitr::kable(total_data, format, digits=2) + table_sepperate_str <- knitr::kable(sepperate_data, format, digits=2, table.attr = "class=\"fancy\"") + table_total_str <- knitr::kable(total_data, format, digits=2, table.attr = "class=\"fancy\"") if (format == "html") { - cat("

",as.character(key[1]),"

", sep = "") - cat("
",table_sepperate_str,"", table_total_str,"
\"barplot
", sep = "") + cat("

",as.character(key[1]),"

", sep = "") + cat("
",table_sepperate_str,"", table_total_str,"
", sep = "") } else { cat(as.character(key[1]), paste0("plot: ", plot_file_name), "", table_total_str, "", table_sepperate_str, "", "", sep="\n") } @@ -138,11 +142,13 @@ plot_durations <- function(data, key,...,format="simple") { # print html header if (as_html) { - cat("
    ") + cat("") + cat("
    ", sep="") + cat("
") } data %>% group_by(backend) %>% group_walk(plot_durations, format=if(as_html) "html" else "simple") diff --git a/scripts/decission-support.sh b/scripts/decission-support.sh index f7aa6cce9..3c45d1996 100755 --- a/scripts/decission-support.sh +++ b/scripts/decission-support.sh @@ -111,8 +111,11 @@ arr=( ) for db in ${arr[@]}; do echo -e "\t round ${db}" + create_config "db" "${db}" "${tmp_config}" + JULEA_CONFIG="${tmp_config}" JULEA_TRACE=access julea-access-replay "${db_record}" 2> "${tmp_record}" + type="$(tr ';' ',' <<< "${db}" | sed 's/'$(basename ${tmp_dir})'\///')" tail -n +2 "${tmp_record}" | awk -F ',' 'BEGIN {OFS=","} { $4 = $4",'"${type}"'"; print }' >> "${output_file}" done From e63cc75d4fa87c7396df3ffa42e0d051cbdc3b2e Mon Sep 17 00:00:00 2001 From: Julian Benda Date: Mon, 27 Feb 2023 15:34:46 +0100 Subject: [PATCH 16/40] Changed decission support execution script to python --- doc/decission-support.md | 1 + example/decission-support-config.json | 56 +++++++ lib/core/jtrace.c | 48 +++++- scripts/decission-support.R | 31 +++- scripts/decission-support.py | 208 ++++++++++++++++++++++++++ tools/access_replay.c | 41 ++--- 6 files changed, 360 insertions(+), 25 deletions(-) create mode 100644 example/decission-support-config.json create mode 100755 scripts/decission-support.py diff --git a/doc/decission-support.md b/doc/decission-support.md index 07e0d923f..ebd7ee7b2 100644 --- a/doc/decission-support.md +++ b/doc/decission-support.md @@ -53,6 +53,7 @@ Important notice: * the script assums a empty backend start state * the script places backends and temporary files in `/tmp/` support for differnce devices is pending * a mysql/mariadb instance must be provided, with the user julea_user, pw: julea_pw which needs access to the database julea_db + * this instance must be manual resseted as forhe time, later a internal `julea-sevrer --clean` will be used to autonamtically do this ```sh JULEA_TRACE=access julea-server 2> access-record.csv diff --git a/example/decission-support-config.json b/example/decission-support-config.json new file mode 100644 index 000000000..6ccc2a91d --- /dev/null +++ b/example/decission-support-config.json @@ -0,0 +1,56 @@ +{ + "object": [ + { + "backend": "posix", + "path": "/tmp//" + }, + { + "backend": "posix", + "path": "/mnt/stick//" + }, + { + "backend": "gio", + "path": "/tmp//" + } + ], + "db": [ + { + "backend": "sqlite", + "path": "/tmp//sqlite" + }, + { + "backend": "sqlite", + "path": ":memory:" + }, + { + "backend": "sqlite", + "path": "/mnt/stick//sqlite" + }, + { + "backend": "mysql", + "path": "127.0.0.1:julea_db:julea_user:julea_pw" + } + ], + "kv": [ + { + "backend": "leveldb", + "path": "/tmp//" + }, + { + "backend": "lmdb", + "path": "/tmp//" + }, + { + "backend": "sqlite", + "path": "/tmp//sqlite" + }, + { + "backend": "sqlite", + "path": ":memory:" + }, + { + "backend": "rocksdb", + "path": "/tmp//" + } + ] +} diff --git a/lib/core/jtrace.c b/lib/core/jtrace.c index e9755aecb..6b6ff5afa 100644 --- a/lib/core/jtrace.c +++ b/lib/core/jtrace.c @@ -21,6 +21,7 @@ **/ #include +#include #include #include @@ -75,6 +76,8 @@ struct Access guint32 uid; const char* program_name; const char* backend; + const char* type; + const char* path; const char* namespace; const char* name; const char* operation; @@ -128,18 +131,25 @@ struct JTraceThread const void* utility_ptr; struct { + char* type; + char* config_path; char* namespace; } db; struct { + char* type; + char* config_path; char* namespace; char* name; } kv; struct { + char* type; + char* config_path; char* namespace; char* path; } object; + } access; }; @@ -185,7 +195,7 @@ G_LOCK_DEFINE_STATIC(j_trace_summary); static void j_trace_access_print_header(void) { - g_printerr("time,process_uid,program_name,backend,namespace,name,operation,size,complexity,duration,bson\n"); + g_printerr("time,process_uid,program_name,backend,type,path,namespace,name,operation,size,complexity,duration,bson\n"); } /** * Creates a new trace thread. @@ -229,6 +239,7 @@ j_trace_thread_new(GThread* thread) trace_thread->thread_name = g_strdup_printf("Thread %d", thread_id); } + #ifdef HAVE_OTF if (j_trace_flags & J_TRACE_OTF) { @@ -263,6 +274,16 @@ j_trace_thread_free(JTraceThread* trace_thread) g_return_if_fail(trace_thread != NULL); + if (j_trace_flags & J_TRACE_ACCESS) + { + g_free(trace_thread->access.db.type); + g_free(trace_thread->access.db.config_path); + g_free(trace_thread->access.kv.type); + g_free(trace_thread->access.kv.config_path); + g_free(trace_thread->access.object.path); + g_free(trace_thread->access.object.config_path); + } + #ifdef HAVE_OTF if (j_trace_flags & J_TRACE_OTF) { @@ -566,12 +587,14 @@ j_trace_fini(void) static void j_trace_access_print(const Access* row, guint64 duration) { - g_printerr("%" G_GUINT64_FORMAT ".%06" G_GUINT64_FORMAT ",%u,%s,%s,%s,%s,%s,%" G_GUINT64_FORMAT ",%u,%" G_GUINT64_FORMAT ".%06" G_GUINT64_FORMAT ",\"%s\"\n", + g_printerr("%" G_GUINT64_FORMAT ".%06" G_GUINT64_FORMAT ",%u,%s,%s,%s,%s,%s,%s,%s,%" G_GUINT64_FORMAT ",%u,%" G_GUINT64_FORMAT ".%06" G_GUINT64_FORMAT ",\"%s\"\n", row->timestamp / G_USEC_PER_SEC, row->timestamp % G_USEC_PER_SEC, row->uid, row->program_name, row->backend, + row->type, + row->path, row->namespace, row->name, row->operation, @@ -733,6 +756,21 @@ j_trace_enter(gchar const* name, gchar const* format, ...) else if (strncmp(name, "backend_", 8) == 0) { int type = 0; + if (trace_thread->access.db.type == NULL) /// \TODO more precise checking? + { + JConfiguration* config = j_configuration_new(); + + trace_thread->access.db.type = strdup(j_configuration_get_backend(config, J_BACKEND_TYPE_DB)); + trace_thread->access.db.config_path = strdup(j_configuration_get_backend_path(config, J_BACKEND_TYPE_DB)); + + trace_thread->access.kv.type = strdup(j_configuration_get_backend(config, J_BACKEND_TYPE_KV)); + trace_thread->access.kv.config_path = strdup(j_configuration_get_backend_path(config, J_BACKEND_TYPE_KV)); + + trace_thread->access.object.type = strdup(j_configuration_get_backend(config, J_BACKEND_TYPE_OBJECT)); + trace_thread->access.object.config_path = strdup(j_configuration_get_backend_path(config, J_BACKEND_TYPE_OBJECT)); + + j_configuration_unref(config); + } name += 8; if (strncmp(name, "kv_", 3) == 0) { @@ -762,6 +800,8 @@ j_trace_enter(gchar const* name, gchar const* format, ...) row->backend = "kv"; row->operation = name; row->namespace = trace_thread->access.kv.namespace; + row->type = trace_thread->access.kv.type; + row->path = trace_thread->access.kv.config_path; row->name = trace_thread->access.kv.name; if (strcmp(name, "batch_start") == 0) { @@ -819,6 +859,8 @@ j_trace_enter(gchar const* name, gchar const* format, ...) name += 3; row->backend = "db"; row->operation = name; + row->type = trace_thread->access.db.type; + row->path = trace_thread->access.db.config_path; row->namespace = trace_thread->access.db.namespace; if (strcmp(name, "batch_start") == 0) { @@ -905,6 +947,8 @@ j_trace_enter(gchar const* name, gchar const* format, ...) row->backend = "object"; row->operation = name; row->namespace = trace_thread->access.object.namespace; + row->type = trace_thread->access.object.type; + row->path = trace_thread->access.object.config_path; row->name = trace_thread->access.object.path; if (strcmp(name, "create") == 0) { diff --git a/scripts/decission-support.R b/scripts/decission-support.R index 45b5f503f..54333a8af 100755 --- a/scripts/decission-support.R +++ b/scripts/decission-support.R @@ -1,6 +1,7 @@ -#!/usr/bin/env /Rscript +#!/usr/bin/env Rscript suppressMessages(library(ggplot2)) +suppressMessages(library(plotly)) suppressMessages(library(dplyr)) suppressMessages(library(scales)) suppressMessages(library(ggpubr)) @@ -41,7 +42,8 @@ test_validity <- function(data) { return(res) } - +args <- commandArgs(trailingOnly = FALSE) +script_dir <- dirname(sub("--file=", "", args[grep("--file=", args)])) args <- commandArgs(trailingOnly = TRUE) if (length(args) < 1) { stop( @@ -86,7 +88,8 @@ plot_durations <- function(data, key,...,format="simple") { summarise(duration = sum(duration), cnt = n(), .groups = "drop") # plotting seperate <- ggplot(sepperate_data, - aes(y = duration, x = paste(operation, "\n#", cnt), fill = backend)) + + aes(y = duration, x = paste(operation, "\n#", cnt), fill = backend, + text=sprintf("%s

#%s
duration: %s%s", backend, cnt, duration, unit))) + geom_col(position = "dodge") + theme(axis.title.x = element_blank(), axis.text.x = element_text(angle = 45)) + @@ -111,7 +114,8 @@ plot_durations <- function(data, key,...,format="simple") { summarise(duration = sum(duration), .groups = "drop") # plotting total <- ggplot(total_data, - aes(y = duration, x = "all operations", fill = backend)) + + aes(y = duration, x = "all operations", fill = backend, + text=sprintf("%s

duration: %s%s", backend, duration, unit))) + geom_col(position = "dodge") + label # table @@ -123,11 +127,15 @@ plot_durations <- function(data, key,...,format="simple") { colnames(sepperate_data)[which(names(sepperate_data) == "duration")] <- "duration (ms)" # output table plot_file_name <- paste(key[1], "time.svg", sep = "_") + plot_html_file_name <- paste(key[1], "time.html", sep = "_") table_sepperate_str <- knitr::kable(sepperate_data, format, digits=2, table.attr = "class=\"fancy\"") table_total_str <- knitr::kable(total_data, format, digits=2, table.attr = "class=\"fancy\"") if (format == "html") { cat("

",as.character(key[1]),"

", sep = "") - cat("
",table_sepperate_str,"", table_total_str,"
", sep = "") + cat("
",table_sepperate_str,"", table_total_str,"
", + #"", + "", + "
", sep = "") } else { cat(as.character(key[1]), paste0("plot: ", plot_file_name), "", table_total_str, "", table_sepperate_str, "", "", sep="\n") } @@ -138,12 +146,23 @@ plot_durations <- function(data, key,...,format="simple") { common.legend = TRUE, legend = "bottom") ggsave(plot_file_name, plot, unit = "in", width = 11, height = 6, bg = "white") + tmp <- ggplotly(seperate, dynamicTicks="y", tooltip="text") + # disables dupliclated legend + tmp[[1]][[1]] <- lapply(tmp[[1]][[1]], function(x) {x$showlegend <- FALSE; x}) + htmlwidgets::saveWidget( + subplot( + tmp, + ggplotly(total, dynamicTicks="y", tooltip="text"), + widths = c(0.8, 0.2) + ) %>% layout(hoverlabel = list(bgcolor="white"), yaxis = list(autorange=TRUE, title=label$y)), plot_html_file_name, + selfcontained = FALSE, + libdir = "lib") } # print html header if (as_html) { cat("") - cat("
", sep="") + cat("
", sep="") cat("