diff --git a/.docker/Dockerfile b/.docker/Dockerfile index 7a8d2ee4..89d52071 100644 --- a/.docker/Dockerfile +++ b/.docker/Dockerfile @@ -1,3 +1,4 @@ +# syntax=docker/dockerfile:1 ARG ROAD_RUNNER_IMAGE=2024.2.1 ARG CENTRIFUGO_IMAGE=v4 ARG DOLT_IMAGE=1.42.8 @@ -29,7 +30,7 @@ RUN echo $CACHE_BUST # Build RoadRunner with velox. The GitHub token is mounted as a secret # and never stored in image layers. RUN --mount=type=secret,id=gh_token \ - RT_TOKEN=$(cat /run/secrets/gh_token) \ + export GH_TOKEN="$(cat /run/secrets/gh_token 2>/dev/null | tr -d '\n' || true)" && \ vx build -c velox.toml -o /usr/bin/ # Build JS files diff --git a/.docker/velox.toml b/.docker/velox.toml index 902d57f2..14ac136d 100644 --- a/.docker/velox.toml +++ b/.docker/velox.toml @@ -93,4 +93,14 @@ repository = "tcp" [github.plugins.smtp-server] ref = "2.0.0" owner = "buggregator" -repository = "smtp-server" \ No newline at end of file +repository = "smtp-server" + +[github.plugins.var-dumper-server] +ref = "1.0.1" +owner = "buggregator" +repository = "var-dumper-server" + +[github.plugins.profiler-server] +ref = "1.0.0" +owner = "buggregator" +repository = "profiler-server" \ No newline at end of file diff --git a/.rr-prod.yaml b/.rr-prod.yaml index 250033d0..336fb68c 100644 --- a/.rr-prod.yaml +++ b/.rr-prod.yaml @@ -60,10 +60,6 @@ tcp: # Address to listen. addr: ${RR_TCP_MONOLOG_ADDR:-:9913} delimiter: "\n" - var-dumper: - # Address to listen. - addr: ${RR_TCP_VAR_DUMPER_ADDR:-:9912} - delimiter: "\n" # Chunks that RR uses to read the data. In bytes. # If you expect big payloads on a TCP server, to reduce `read` syscalls, # would be a good practice to use a fairly big enough buffer. @@ -77,15 +73,32 @@ kv: driver: memory config: { } +var-dumper: + addr: ${RR_VAR_DUMPER_ADDR:-:9912} + max_message_size: 10485760 + jobs: + pipeline: "var-dumper" + auto_ack: true + jobs: consume: - smtp + - var-dumper + - profiler pipelines: smtp: driver: memory config: priority: 10 prefetch: 10 + var-dumper: + driver: memory + config: + priority: 10 + profiler: + driver: memory + config: + priority: 10 pool: num_workers: ${RR_JOBS_NUM_WORKERS:-1} @@ -95,6 +108,13 @@ smtp: jobs: pipeline: smtp +profiler: + addr: ${RR_PROFILER_ADDR:-:9914} + max_request_size: 52428800 + jobs: + pipeline: "profiler" + auto_ack: true + service: nginx: service_name_in_log: true diff --git a/.rr.yaml b/.rr.yaml index 69c99da1..49ee3f72 100644 --- a/.rr.yaml +++ b/.rr.yaml @@ -34,11 +34,6 @@ tcp: monolog: addr: 127.0.0.1:9913 delimiter: "\n" - var-dumper: - addr: 127.0.0.1:9912 - delimiter: "\n" - smtp: - addr: 127.0.0.1:1025 pool: num_workers: 2 @@ -47,11 +42,35 @@ kv: driver: memory config: { } +var-dumper: + addr: 127.0.0.1:9912 + jobs: + pipeline: "var-dumper" + auto_ack: true + jobs: - consume: [ ] + pipelines: + var-dumper: + driver: memory + config: + priority: 10 + profiler: + driver: memory + config: + priority: 10 + smtp: + driver: memory + config: + priority: 10 + consume: ["var-dumper", "profiler", "smtp"] pool: num_workers: 1 +profiler: + addr: 127.0.0.1:9914 + jobs: + pipeline: "profiler" + auto_ack: true smtp: addr: ${RR_SMTP_ADDR:-:1025} diff --git a/Makefile b/Makefile index bf47a4b3..c94a5dad 100644 --- a/Makefile +++ b/Makefile @@ -43,7 +43,7 @@ pull-latest: docker compose pull; build-server: - docker compose build buggregator-server --no-cache; + DOCKER_BUILDKIT=1 docker compose build buggregator-server --no-cache; # ====== Database ======== diff --git a/app/config/queue.php b/app/config/queue.php index 84dca517..1a422c1b 100644 --- a/app/config/queue.php +++ b/app/config/queue.php @@ -39,6 +39,8 @@ 'registry' => [ 'handlers' => [ 'smtp.email' => EmailHandler::class, + 'vardumper.dump' => \Modules\VarDumper\Interfaces\Jobs\DumpHandler::class, + 'profiler.profile' => \Modules\Profiler\Interfaces\Jobs\ProfileHandler::class, ], 'serializers' => [ WebhookHandler::class => 'symfony-json', diff --git a/app/config/tcp.php b/app/config/tcp.php index 9f01607b..aaa1f09a 100644 --- a/app/config/tcp.php +++ b/app/config/tcp.php @@ -3,19 +3,14 @@ declare(strict_types=1); use App\Application\TCP\ExceptionHandlerInterceptor; -use Modules\VarDumper\Interfaces\TCP\Service as VarDumperService; use Modules\Monolog\Interfaces\TCP\Service as MonologService; return [ 'services' => [ - 'var-dumper' => VarDumperService::class, 'monolog' => MonologService::class, ], 'interceptors' => [ - 'var-dumper' => [ - ExceptionHandlerInterceptor::class, - ], 'monolog' => [ ExceptionHandlerInterceptor::class, ], diff --git a/app/modules/Profiler/Interfaces/Jobs/ProfileHandler.php b/app/modules/Profiler/Interfaces/Jobs/ProfileHandler.php new file mode 100644 index 00000000..484401dd --- /dev/null +++ b/app/modules/Profiler/Interfaces/Jobs/ProfileHandler.php @@ -0,0 +1,128 @@ +profileFactory->create( + name: $payload['app_name'] ?? 'unknown', + tags: $payload['tags'] ?? [], + peaks: new Peaks( + cpu: $peaks['cpu'] ?? 0, + wt: $peaks['wt'] ?? 0, + ct: $peaks['ct'] ?? 0, + mu: $peaks['mu'] ?? 0, + pmu: $peaks['pmu'] ?? 0, + ), + ); + + $this->em->persist($profile)->run(); + + // Store pre-processed edges + $edges = $payload['edges'] ?? []; + $parents = []; + $batchSize = 0; + $order = 0; + + foreach ($edges as $id => $edge) { + $cost = $edge['cost'] ?? []; + $diff = $edge['diff'] ?? []; + $pcts = $edge['percents'] ?? []; + + $edgeEntity = $this->edgeFactory->create( + profileUuid: $profile->getUuid(), + order: $order++, + cost: new Cost( + cpu: $cost['cpu'] ?? 0, + wt: $cost['wt'] ?? 0, + ct: $cost['ct'] ?? 0, + mu: $cost['mu'] ?? 0, + pmu: $cost['pmu'] ?? 0, + ), + diff: new Diff( + cpu: $diff['d_cpu'] ?? 0, + wt: $diff['d_wt'] ?? 0, + ct: $diff['d_ct'] ?? 0, + mu: $diff['d_mu'] ?? 0, + pmu: $diff['d_pmu'] ?? 0, + ), + percents: new Percents( + cpu: $pcts['p_cpu'] ?? 0.0, + wt: $pcts['p_wt'] ?? 0.0, + ct: $pcts['p_ct'] ?? 0.0, + mu: $pcts['p_mu'] ?? 0.0, + pmu: $pcts['p_pmu'] ?? 0.0, + ), + callee: $edge['callee'], + caller: $edge['caller'] ?? null, + parentUuid: isset($edge['parent']) ? ($parents[$edge['parent']] ?? null) : null, + ); + + $this->em->persist($edgeEntity); + $parents[$id] = $edgeEntity->getUuid(); + + $batchSize++; + if ($batchSize >= self::BATCH_SIZE) { + $this->em->run(); + $batchSize = 0; + } + } + + $this->em->run(); + + // Dispatch event for broadcasting and storage in events table + $this->bus->dispatch( + new HandleReceivedEvent( + type: 'profiler', + payload: [ + 'profile_uuid' => (string) $profile->getUuid(), + 'peaks' => $peaks, + 'tags' => $payload['tags'] ?? [], + 'app_name' => $payload['app_name'] ?? 'unknown', + 'hostname' => $payload['hostname'] ?? 'unknown', + 'date' => $payload['date'] ?? 0, + 'total_edges' => \count($edges), + ], + uuid: $profileUuid, + ), + ); + } +} diff --git a/app/modules/VarDumper/Interfaces/TCP/Service.php b/app/modules/VarDumper/Interfaces/Jobs/DumpHandler.php similarity index 71% rename from app/modules/VarDumper/Interfaces/TCP/Service.php rename to app/modules/VarDumper/Interfaces/Jobs/DumpHandler.php index 5dbd224a..2912aa32 100644 --- a/app/modules/VarDumper/Interfaces/TCP/Service.php +++ b/app/modules/VarDumper/Interfaces/Jobs/DumpHandler.php @@ -2,7 +2,7 @@ declare(strict_types=1); -namespace Modules\VarDumper\Interfaces\TCP; +namespace Modules\VarDumper\Interfaces\Jobs; use App\Application\Commands\HandleReceivedEvent; use Modules\VarDumper\Application\Dump\BodyInterface; @@ -12,36 +12,33 @@ use Modules\VarDumper\Application\Dump\MessageParser; use Modules\VarDumper\Application\Dump\ParsedPayload; use Modules\VarDumper\Application\Dump\PrimitiveBody; +use Spiral\Core\InvokerInterface; use Spiral\Cqrs\CommandBusInterface; -use Spiral\RoadRunner\Tcp\Request; -use Spiral\RoadRunner\Tcp\TcpEvent; -use Spiral\RoadRunnerBridge\Tcp\Response\ContinueRead; -use Spiral\RoadRunnerBridge\Tcp\Response\ResponseInterface; -use Spiral\RoadRunnerBridge\Tcp\Service\ServiceInterface; +use Spiral\Queue\JobHandler; use Symfony\Component\VarDumper\Cloner\Data; -final readonly class Service implements ServiceInterface +final class DumpHandler extends JobHandler { public function __construct( - private CommandBusInterface $commandBus, - private DumpIdGeneratorInterface $dumpId, - ) {} + private readonly CommandBusInterface $commandBus, + private readonly DumpIdGeneratorInterface $dumpId, + InvokerInterface $invoker, + ) { + parent::__construct($invoker); + } - public function handle(Request $request): ResponseInterface + public function invoke(array $payload): void { - if ($request->event === TcpEvent::Connected) { - return new ContinueRead(); - } - - $messages = \array_filter(\explode("\n", $request->body)); + // RR VarDumper plugin sends: { event, uuid, payload (base64), context, ... } + $message = $payload['payload'] ?? ''; - foreach ($messages as $message) { - $payload = (new MessageParser())->parse($message); - - $this->fireEvent($payload); + if ($message === '') { + return; } - return new ContinueRead(); + $parsed = (new MessageParser())->parse($message); + + $this->fireEvent($parsed); } private function fireEvent(ParsedPayload $payload): void @@ -58,7 +55,7 @@ private function fireEvent(ParsedPayload $payload): void ); } - private function convertToPrimitive(Data $data): BodyInterface|null + private function convertToPrimitive(Data $data): BodyInterface { if (\in_array($data->getType(), ['string', 'boolean', 'integer', 'double'])) { return new PrimitiveBody( diff --git a/docker-compose.yaml b/docker-compose.yaml index 10c70069..8dd77f96 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -2,7 +2,7 @@ version: '3.1' services: buggregator-reverse-proxy: - image: traefik:v2.9 + image: traefik:v3.6 command: - "--accesslog" - "--api.insecure=true" diff --git a/tests/Feature/Interfaces/TCP/TCPTestCase.php b/tests/Feature/Interfaces/TCP/TCPTestCase.php index 5d4375f1..6694102a 100644 --- a/tests/Feature/Interfaces/TCP/TCPTestCase.php +++ b/tests/Feature/Interfaces/TCP/TCPTestCase.php @@ -5,7 +5,6 @@ namespace Tests\Feature\Interfaces\TCP; use Modules\Monolog\Interfaces\TCP\Service as MonologService; -use Modules\VarDumper\Interfaces\TCP\Service as VarDumperService; use Modules\Smtp\Interfaces\TCP\Service as SmtpService; use Ramsey\Uuid\Uuid; use Ramsey\Uuid\UuidInterface; @@ -25,13 +24,6 @@ public function handleMonologRequest(string $message): ResponseInterface ->handle($this->buildRequest(message: $message)); } - public function handleVarDumperRequest(string $message): ResponseInterface - { - return $this - ->get(VarDumperService::class) - ->handle($this->buildRequest(message: $message)); - } - public function handleSmtpRequest(string $message, TcpEvent $event = TcpEvent::Data): ResponseInterface { return $this diff --git a/tests/Feature/Interfaces/TCP/VarDumper/SymfonyV6Test.php b/tests/Feature/Interfaces/TCP/VarDumper/SymfonyV6Test.php index 16148023..953deebf 100644 --- a/tests/Feature/Interfaces/TCP/VarDumper/SymfonyV6Test.php +++ b/tests/Feature/Interfaces/TCP/VarDumper/SymfonyV6Test.php @@ -5,6 +5,7 @@ namespace Tests\Feature\Interfaces\TCP\VarDumper; use App\Application\Broadcasting\Channel\EventsChannel; +use Modules\VarDumper\Interfaces\Jobs\DumpHandler; use Tests\Feature\Interfaces\TCP\TCPTestCase; final class SymfonyV6Test extends TCPTestCase @@ -13,7 +14,13 @@ public function testSendDump(): void { $payload = 'YToyOntpOjA7TzozOToiU3ltZm9ueVxDb21wb25lbnRcVmFyRHVtcGVyXENsb25lclxEYXRhIjo3OntzOjQ1OiIAU3ltZm9ueVxDb21wb25lbnRcVmFyRHVtcGVyXENsb25lclxEYXRhAGRhdGEiO2E6MTp7aTowO2E6MTp7aTowO3M6MzoiZm9vIjt9fXM6NDk6IgBTeW1mb255XENvbXBvbmVudFxWYXJEdW1wZXJcQ2xvbmVyXERhdGEAcG9zaXRpb24iO2k6MDtzOjQ0OiIAU3ltZm9ueVxDb21wb25lbnRcVmFyRHVtcGVyXENsb25lclxEYXRhAGtleSI7aTowO3M6NDk6IgBTeW1mb255XENvbXBvbmVudFxWYXJEdW1wZXJcQ2xvbmVyXERhdGEAbWF4RGVwdGgiO2k6MjA7czo1NzoiAFN5bWZvbnlcQ29tcG9uZW50XFZhckR1bXBlclxDbG9uZXJcRGF0YQBtYXhJdGVtc1BlckRlcHRoIjtpOi0xO3M6NTQ6IgBTeW1mb255XENvbXBvbmVudFxWYXJEdW1wZXJcQ2xvbmVyXERhdGEAdXNlUmVmSGFuZGxlcyI7aTotMTtzOjQ4OiIAU3ltZm9ueVxDb21wb25lbnRcVmFyRHVtcGVyXENsb25lclxEYXRhAGNvbnRleHQiO2E6MDp7fX1pOjE7YTozOntzOjk6InRpbWVzdGFtcCI7ZDoxNzAxNDk5NDM3LjUzODQ0NztzOjM6ImNsaSI7YToyOntzOjEyOiJjb21tYW5kX2xpbmUiO3M6MzIxOiIvcm9vdC9yZXBvcy9idWdncmVhZ3Rvci9zcGlyYWwtYXBwL3ZlbmRvci9waHB1bml0L3BocHVuaXQvcGhwdW5pdCAtLWNvbmZpZ3VyYXRpb24gL3Jvb3QvcmVwb3MvYnVnZ3JlYWd0b3Ivc3BpcmFsLWFwcC9waHB1bml0LnhtbCAtLWZpbHRlciAvKEludGVyZmFjZXNcXFRDUFxcVmFyRHVtcGVyXFxTeW1mb255VjZUZXN0Ojp0ZXN0U2VuZER1bXApKCAuKik/JC8gLS10ZXN0LXN1ZmZpeCBTeW1mb255VjZUZXN0LnBocCAvcm9vdC9yZXBvcy9idWdncmVhZ3Rvci9zcGlyYWwtYXBwL3Rlc3RzL0ZlYXR1cmUvSW50ZXJmYWNlcy9UQ1AvVmFyRHVtcGVyIC0tdGVhbWNpdHkiO3M6MTA6ImlkZW50aWZpZXIiO3M6ODoiZGVlMTBhZWUiO31zOjY6InNvdXJjZSI7YTo0OntzOjQ6Im5hbWUiO3M6MTc6IlN5bWZvbnlWNlRlc3QucGhwIjtzOjQ6ImZpbGUiO3M6OTE6Ii9yb290L3JlcG9zL2J1Z2dyZWFndG9yL3NwaXJhbC1hcHAvdGVzdHMvRmVhdHVyZS9JbnRlcmZhY2VzL1RDUC9WYXJEdW1wZXIvU3ltZm9ueVY2VGVzdC5waHAiO3M6NDoibGluZSI7aToxMztzOjEyOiJmaWxlX2V4Y2VycHQiO2I6MDt9fX0='; - $this->handleVarDumperRequest($payload); + $handler = $this->get(DumpHandler::class); + $handler->invoke([ + 'event' => 'DUMP_RECEIVED', + 'uuid' => 'test-uuid', + 'payload' => $payload, + 'context' => [], + ]); $this->broadcastig->assertPushed(new EventsChannel('default'), function (array $data) { $this->assertSame('event.received', $data['event']); @@ -28,7 +35,6 @@ public function testSendDump(): void $this->assertNotEmpty($data['data']['uuid']); $this->assertNotEmpty($data['data']['timestamp']); - return true; }); } diff --git a/tests/Feature/Interfaces/TCP/VarDumper/SymfonyV7Test.php b/tests/Feature/Interfaces/TCP/VarDumper/SymfonyV7Test.php index 774222f5..e31e8673 100644 --- a/tests/Feature/Interfaces/TCP/VarDumper/SymfonyV7Test.php +++ b/tests/Feature/Interfaces/TCP/VarDumper/SymfonyV7Test.php @@ -7,6 +7,7 @@ use App\Application\Broadcasting\Channel\EventsChannel; use Modules\VarDumper\Application\Dump\DumpIdGeneratorInterface; use Modules\VarDumper\Exception\InvalidPayloadException; +use Modules\VarDumper\Interfaces\Jobs\DumpHandler; use PHPUnit\Framework\Attributes\DataProvider; use Symfony\Component\VarDumper\Caster\ReflectionCaster; use Symfony\Component\VarDumper\Cloner\VarCloner; @@ -55,7 +56,7 @@ public function testSendDump(mixed $value, string $type, mixed $expected): void $generator->shouldReceive('generate')->andReturn('sf-dump-730421088'); $message = $this->buildPayload(var: $value); - $this->handleVarDumperRequest($message); + $this->handleVarDumperJob($message); if (\is_object($value)) { $expected = \sprintf($expected, \spl_object_id($value)); @@ -81,7 +82,7 @@ public function testSendDump(mixed $value, string $type, mixed $expected): void public function testSendDumpWithCodeHighlighting(): void { $message = $this->buildPayload(var: 'foo', context: ['language' => 'php']); - $this->handleVarDumperRequest($message); + $this->handleVarDumperJob($message); $this->broadcastig->assertPushed(new EventsChannel('default'), function (array $data) { $this->assertSame('event.received', $data['event']); @@ -102,7 +103,7 @@ public function testSendDumpWithProject(): void { $this->createProject('foo'); $message = $this->buildPayload(project: 'foo'); - $this->handleVarDumperRequest($message); + $this->handleVarDumperJob($message); $this->broadcastig->assertPushed(new EventsChannel('foo'), function (array $data) { $this->assertSame('foo', $data['data']['project']); @@ -113,7 +114,7 @@ public function testSendDumpWithProject(): void public function testSendDumpWithNonExistsProject(): void { $message = $this->buildPayload(project: 'foo'); - $this->handleVarDumperRequest($message); + $this->handleVarDumperJob($message); $this->broadcastig->assertNotPushed(new EventsChannel('foo')); $this->broadcastig->assertPushed(new EventsChannel('default'), function (array $data) { @@ -127,7 +128,18 @@ public function testSendInvalidDump(): void $this->expectException(InvalidPayloadException::class); $this->expectExceptionMessage('Unable to decode the message.'); - $this->handleVarDumperRequest('invalid'); + $this->handleVarDumperJob('invalid'); + } + + private function handleVarDumperJob(string $base64Message): void + { + $handler = $this->get(DumpHandler::class); + $handler->invoke([ + 'event' => 'DUMP_RECEIVED', + 'uuid' => 'test-uuid', + 'payload' => \rtrim($base64Message, "\n"), + 'context' => [], + ]); } private function buildPayload(mixed $var = 'string', ?string $project = null, array $context = []): string