Integration tests

.github/workflows/integration_test.yml

@@ -0,0 +1,46 @@
+name: Integration Tests
+on:
+  pull_request:
+    paths-ignore:
+      - "docs/**"
+      - "*.md"
+jobs:
+  integration:
+    name: Integration Tests
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-go@v5
+        with:
+          go-version: 1.22
+      - name: Start Cortex
+        run: |
+          docker run -d --name cortex \
+            -p 9009:9009 \
+            -v ${{ github.workspace }}/integration/cortex-config.yaml:/etc/cortex/config.yaml \
+            cortexproject/cortex:v1.18.1 \
+            -config.file=/etc/cortex/config.yaml \
+            -target=all,alertmanager
+      - name: Wait for Cortex
+        run: |
+          for i in $(seq 1 30); do
+            if wget -qO- http://localhost:9009/ready > /dev/null 2>&1; then
+              echo "Cortex is ready"
+              exit 0
+            fi
+            echo "Waiting for Cortex... ($i)"
+            sleep 2
+          done
+          echo "Cortex failed to start"
+          docker logs cortex
+          exit 1
+      - name: Run Integration Tests
+        env:
+          CORTEX_ADDRESS: http://localhost:9009
+        run: go test -mod=vendor -tags=integration -v -count=1 ./integration/...
+      - name: Cortex Logs
+        if: failure()
+        run: docker logs cortex
+      - name: Cleanup
+        if: always()
+        run: docker stop cortex && docker rm cortex

integration/alertmanager-config.yaml

@@ -0,0 +1,4 @@
+route:
+  receiver: default
+receivers:
+  - name: default

integration/cortex-config.yaml

@@ -0,0 +1,67 @@
+auth_enabled: false
+
+server:
+  http_listen_port: 9009
+  grpc_server_max_recv_msg_size: 104857600
+  grpc_server_max_send_msg_size: 104857600
+  grpc_server_max_concurrent_streams: 1000
+
+distributor:
+  shard_by_all_labels: true
+  pool:
+    health_check_ingesters: true
+
+ingester_client:
+  grpc_client_config:
+    max_recv_msg_size: 104857600
+    max_send_msg_size: 104857600
+    grpc_compression: gzip
+
+ingester:
+  lifecycler:
+    min_ready_duration: 0s
+    final_sleep: 0s
+    num_tokens: 512
+    ring:
+      kvstore:
+        store: inmemory
+      replication_factor: 1
+
+blocks_storage:
+  tsdb:
+    dir: /tmp/cortex/tsdb
+  bucket_store:
+    sync_dir: /tmp/cortex/tsdb-sync
+  backend: filesystem
+  filesystem:
+    dir: /tmp/cortex/data/tsdb
+
+compactor:
+  data_dir: /tmp/cortex/compactor
+  sharding_ring:
+    kvstore:
+      store: inmemory
+
+frontend_worker:
+  match_max_concurrent: true
+
+ruler:
+  enable_api: true
+
+ruler_storage:
+  backend: filesystem
+  filesystem:
+    dir: /tmp/cortex/rules
+
+alertmanager:
+  enable_api: true
+  external_url: http://localhost:9009/alertmanager
+  sharding_ring:
+    kvstore:
+      store: inmemory
+    replication_factor: 1
+
+alertmanager_storage:
+  backend: filesystem
+  filesystem:
+    dir: /tmp/cortex/alerts

integration/integration_test.go

@@ -0,0 +1,127 @@
+//go:build integration
+
+package integration
+
+import (
+	"context"
+	"errors"
+	"os"
+	"testing"
+
+	"github.com/prometheus/prometheus/model/rulefmt"
+	"github.com/stretchr/testify/require"
+	"gopkg.in/yaml.v3"
+
+	"github.com/cortexproject/cortex-tools/pkg/client"
+	"github.com/cortexproject/cortex-tools/pkg/rules/rwrulefmt"
+)
+
+func cortexAddress() string {
+	if addr := os.Getenv("CORTEX_ADDRESS"); addr != "" {
+		return addr
+	}
+	return "http://localhost:9009"
+}
+
+func newClient(t *testing.T) *client.CortexClient {
+	t.Helper()
+	c, err := client.New(client.Config{
+		Address: cortexAddress(),
+		ID:      "fake",
+	})
+	require.NoError(t, err)
+	return c
+}
+
+func ruleNode(record, expr string) rulefmt.RuleNode {
+	return rulefmt.RuleNode{
+		Record: yaml.Node{Kind: yaml.ScalarNode, Value: record},
+		Expr:   yaml.Node{Kind: yaml.ScalarNode, Value: expr},
+	}
+}
+
+func TestRulesLoadListDelete(t *testing.T) {
+	ctx := context.Background()
+	c := newClient(t)
+
+	namespace := "test_namespace"
+	group := rwrulefmt.RuleGroup{}
+	group.Name = "test_rule_group"
+	group.Rules = []rulefmt.RuleNode{
+		ruleNode("summed_up", "sum(up)"),
+	}
+
+	err := c.CreateRuleGroup(ctx, namespace, group)
+	require.NoError(t, err, "CreateRuleGroup should succeed")
+
+	ruleSet, err := c.ListRules(ctx, "")
+	require.NoError(t, err, "ListRules should succeed")
+	require.Contains(t, ruleSet, namespace, "namespace should exist")
+
+	found := false
+	for _, g := range ruleSet[namespace] {
+		if g.Name == "test_rule_group" {
+			found = true
+			break
+		}
+	}
+	require.True(t, found, "rule group should be in list")
+
+	rg, err := c.GetRuleGroup(ctx, namespace, "test_rule_group")
+	require.NoError(t, err, "GetRuleGroup should succeed")
+	require.Equal(t, "test_rule_group", rg.Name)
+	require.Len(t, rg.Rules, 1)
+
+	err = c.DeleteRuleNamespace(ctx, namespace)
+	require.NoError(t, err, "DeleteRuleNamespace should succeed")
+
+	ruleSet, err = c.ListRules(ctx, "")
+	if err != nil {
+		require.True(t, errors.Is(err, client.ErrResourceNotFound), "expected no rules or resource not found, got: %v", err)
+	} else {
+		require.NotContains(t, ruleSet, namespace, "namespace should be deleted")
+	}
+}
+
+func TestRulesMultipleGroups(t *testing.T) {
+	ctx := context.Background()
+	c := newClient(t)
+
+	namespace := "multi_group_namespace"
+	groups := []rwrulefmt.RuleGroup{
+		{RuleGroup: rulefmt.RuleGroup{Name: "group_a", Rules: []rulefmt.RuleNode{ruleNode("metric_a", "sum(up)")}}},
+		{RuleGroup: rulefmt.RuleGroup{Name: "group_b", Rules: []rulefmt.RuleNode{ruleNode("metric_b", "count(up)")}}},
+		{RuleGroup: rulefmt.RuleGroup{Name: "group_c", Rules: []rulefmt.RuleNode{ruleNode("metric_c", "avg(up)")}}},
+	}
+
+	for _, g := range groups {
+		err := c.CreateRuleGroup(ctx, namespace, g)
+		require.NoError(t, err, "CreateRuleGroup %s should succeed", g.Name)
+	}
+
+	ruleSet, err := c.ListRules(ctx, namespace)
+	require.NoError(t, err)
+	require.Contains(t, ruleSet, namespace)
+	require.Len(t, ruleSet[namespace], 3, "should have 3 rule groups")
+
+	err = c.DeleteRuleNamespace(ctx, namespace)
+	require.NoError(t, err)
+}
+
+func TestAlertmanagerLoadGet(t *testing.T) {
+	ctx := context.Background()
+	c := newClient(t)
+
+	amConfig := "route:\n  receiver: default\nreceivers:\n  - name: default\n"
+
+	err := c.CreateAlertmanagerConfig(ctx, amConfig, nil)
+	require.NoError(t, err, "CreateAlertmanagerConfig should succeed")
+
+	cfg, templates, err := c.GetAlertmanagerConfig(ctx)
+	require.NoError(t, err, "GetAlertmanagerConfig should succeed")
+	require.Contains(t, cfg, "receiver: default")
+	require.Empty(t, templates)
+
+	err = c.DeleteAlermanagerConfig(ctx)
+	require.NoError(t, err, "DeleteAlermanagerConfig should succeed")
+}