Proposal: Add initial telemetry metrics feature (backend endpoints, JS, and HTML template)

tcms/settings/common.py

@@ -432,6 +432,7 @@
                             (_("Trends"), reverse_lazy("testing-execution-trends")),
                         ],
                     ),
+                    (_("Metrics"), reverse_lazy("testing-metrics")),
                     (_("TestCase health"), reverse_lazy("test-case-health")),
                 ],
             ),

tcms/static/js/index.js

@@ -19,6 +19,8 @@ import { pageManagementBuildAdminReadyHandler } from '../../management/static/ma
 
 import { pageTelemetryReadyHandler } from '../../telemetry/static/telemetry/js/index'
 
+import { drawTable as pageTelemetryMetricsReadyHandler } from '../../telemetry/static/telemetry/js/testing/metrics'
+
 import { jsonRPC } from './jsonrpc'
 import { initSimpleMDE } from './simplemde_security_override'
 
@@ -52,7 +54,8 @@ const pageHandlers = {
     'page-telemetry-status-matrix': pageTelemetryReadyHandler,
     'page-telemetry-execution-dashboard': pageTelemetryReadyHandler,
     'page-telemetry-execution-trends': pageTelemetryReadyHandler,
-    'page-telemetry-test-case-health': pageTelemetryReadyHandler
+    'page-telemetry-test-case-health': pageTelemetryReadyHandler,
+    'page-telemetry-metrics': pageTelemetryMetricsReadyHandler
 }
 
 $(() => {

tcms/static/js/jsonrpc.js

@@ -22,12 +22,20 @@ export function jsonRPC (rpcMethod, rpcParams, callback, isSync) {
         contentType: 'application/json',
         success: function (result) {
             if (result.error) {
-                alert(result.error.message)
+                // If authentication error and we are on the login page, ignore
+                if ((result.error.code === 401 || result.error.code === 403) && document.body.id === 'login') {
+                    // do nothing
+                }
+                // Optionally handle error here (alert removed)
             } else {
                 callback(result.result)
             }
         },
         error: function (err, status, thrown) {
+            // If authentication error and we are on the login page, ignore
+            if (err && (err.status === 401 || err.status === 403) && document.body.id === 'login') {
+                return
+            }
             console.log('*** jsonRPC ERROR: ' + err + ' STATUS: ' + status + ' ' + thrown)
         }
     })

tcms/telemetry/api.py

@@ -4,8 +4,108 @@
 from modernrpc.auth.basic import http_basic_auth_login_required
 from modernrpc.core import rpc_method
 
-from tcms.testcases.models import TestCase
-from tcms.testruns.models import TestExecution, TestExecutionStatus
+from tcms.testcases.models import TestCase, TestCasePlan
+from tcms.testplans.models import TestPlan
+from tcms.testruns.models import TestExecution, TestExecutionStatus, TestRun
+@http_basic_auth_login_required
+@rpc_method(name="Testing.metrics")
+def metrics(query=None):
+    """
+    .. function:: RPC Testing.metrics(query)
+
+        Return metrics for test runs and executions
+
+        :param query: Field lookups for :class:`tcms.testruns.models.TestRun`
+        :type query: dict
+        :return: List of dicts with metrics per test run
+        :rtype: list(dict)
+    """
+    if not isinstance(query, dict):
+        query = {}
+
+    test_runs = TestRun.objects.filter(**query).select_related('plan').order_by('-start_date')
+    run_ids = [tr.id for tr in test_runs]
+    plan_ids = [tr.plan_id for tr in test_runs if tr.plan_id]
+
+
+    planned_cases = TestCasePlan.objects.filter(plan_id__in=plan_ids).values('plan_id').annotate(count=Count('id'))
+    planned_cases_map = {pc['plan_id']: pc['count'] for pc in planned_cases}
+
+
+
+    executions = TestExecution.objects.filter(run_id__in=run_ids)
+
+
+    status_ids_weighted = set(TestExecutionStatus.objects.exclude(weight=0).values_list('id', flat=True))
+
+
+    executed_cases = executions.filter(status_id__in=status_ids_weighted).values('run_id').annotate(count=Count('case_id', distinct=True))
+    executed_cases_map = {e['run_id']: e['count'] for e in executed_cases}
+
+
+    status_counts = executions.values('run_id', 'status_id').annotate(count=Count('id'))
+    status_count_map = {}
+    for item in status_counts:
+        run_id = item['run_id']
+        status_id = item['status_id']
+        count = item['count']
+        status_count_map.setdefault(run_id, {})[status_id] = count
+
+
+    total_exec = executions.values('run_id').annotate(count=Count('id'))
+    total_exec_map = {e['run_id']: e['count'] for e in total_exec}
+
+
+    statuses = TestExecutionStatus.objects.exclude(weight=0).values('id', 'name')
+
+    results = []
+    statuses = list(TestExecutionStatus.objects.exclude(weight=0).values('id', 'name'))
+    status_names = [status['name'] for status in statuses]
+
+    status_weights = dict(TestExecutionStatus.objects.exclude(weight=0).values_list('id', 'weight'))
+    for tr in test_runs:
+        tp = tr.plan
+        total_planned = planned_cases_map.get(tr.plan_id, 0)
+        executed_cases = executed_cases_map.get(tr.id, 0)
+        status_counts_run = status_count_map.get(tr.id, {})
+        total_exec = total_exec_map.get(tr.id, 0)
+
+        def percent(status_id):
+            if total_exec == 0:
+                return 0.0
+            return round(status_counts_run.get(status_id, 0) / total_exec * 100, 2)
+
+        metrics = {
+            'test_run_id': tr.id,
+            'test_run_name': tr.summary,
+            'start_date': tr.start_date,
+            'stop_date': tr.stop_date,
+            'test_plan_name': tp.name if tp else None,
+            'total_planned_cases': total_planned,
+            'executed_cases': executed_cases,
+        }
+
+
+        for status in statuses:
+            metrics[f"{status['name'].lower()}_percent"] = percent(status['id'])
+
+        metrics['execution_coverage'] = round(executed_cases / total_planned * 100, 2) if total_planned else 0.0
+
+
+
+
+
+        results.append(metrics)
+    status_formulas = []
+    for status in statuses:
+        status_name = status['name'].upper()
+        formula = f"Cobertura de casos de prueba con estado {status_name} = (Número de pruebas con estado {status_name} / Número total de pruebas ejecutadas) × 100"
+        status_formulas.append(formula)
+
+    return {
+        "results": results,
+        "statuses": status_names
+    }
 
 
 @http_basic_auth_login_required

tcms/telemetry/static/telemetry/js/index.js

@@ -19,6 +19,7 @@ import {
     initializePage as executionDashboardInitialize
 } from './testing/execution-dashboard'
 import { drawChart as executionTrendsDrawChart } from './testing/execution-trends'
+import { drawTable as metricsDrawTable } from './testing/metrics'
 import {
     reloadTable as testCaseHealthDrawChart,
     initializePage as testCaseHealthInitialize
@@ -46,20 +47,24 @@ export function pageTelemetryReadyHandler (pageId) {
         'page-telemetry-status-matrix': statusMatrixDrawChart,
         'page-telemetry-execution-dashboard': executionDashboardDrawTable,
         'page-telemetry-execution-trends': executionTrendsDrawChart,
-        'page-telemetry-test-case-health': testCaseHealthDrawChart
+        'page-telemetry-test-case-health': testCaseHealthDrawChart,
+        'page-telemetry-metrics': metricsDrawTable
     }[pageId]
 
     const initializePage = {
         'page-telemetry-testing-breakdown': testingBreakdownInitialize,
         'page-telemetry-status-matrix': statusMatrixInitialize,
         'page-telemetry-execution-dashboard': executionDashboardInitialize,
         'page-telemetry-execution-trends': () => {},
-        'page-telemetry-test-case-health': testCaseHealthInitialize
+        'page-telemetry-test-case-health': testCaseHealthInitialize,
+        'page-telemetry-metrics': () => {}
     }[pageId]
 
     initializePage()
 
-    loadInitialProduct()
+    loadInitialProduct(() => {
+        drawChart()
+    })
 
     document.getElementById('id_product').onchange = () => {
         updateVersionSelectFromProduct()

tcms/telemetry/static/telemetry/js/testing/metrics.js

@@ -0,0 +1,189 @@
+import { jsonRPC } from '../../../../../static/js/jsonrpc'
+import { exportButtons } from '../../../../../static/js/datatables_common'
+
+export function drawTable (query = {}) {
+    $('.js-spinner').show()
+    jsonRPC('Testing.metrics', query, function (response) {
+        $('.js-spinner').hide()
+
+        const data = response.results
+        const statuses = response.statuses
+
+        const $checkboxes = $('#metrics-coverage-checkboxes')
+        $checkboxes.empty()
+        statuses.forEach(status => {
+            const id = `coverage-status-${status}`
+            $checkboxes.append(` <label style="margin-right:12px;"><input type="checkbox" class="coverage-status-checkbox" id="${id}" value="${status}" checked> ${status}</label> `)
+        })
+
+        function calculateAndShowExecutionCoverage () {
+            const selectedStatuses = []
+            $('.coverage-status-checkbox:checked').each(function () {
+                selectedStatuses.push($(this).val())
+            })
+
+            let totalPlanned = 0
+            let totalExecuted = 0
+            data.forEach(row => {
+                totalPlanned += row.total_planned_cases
+
+                const percentSum = selectedStatuses.reduce((acc, status) => acc + (row[`${status.toLowerCase()}_percent`] || 0), 0)
+                const filteredExecuted = Math.round(row.executed_cases * (percentSum / 100))
+                totalExecuted += filteredExecuted
+            })
+
+            console.log('totalPlanned:', totalPlanned, 'totalExecuted:', totalExecuted)
+            const relevantStatuses = selectedStatuses.join(', ')
+            const text = `Test Execution Coverage = (Total number of executed test cases or scripts / Total number of test cases or scripts planned to be executed) x 100. A test case is considered executed if at least one of its executions has a relevant status (${relevantStatuses}).`
+            $('#metrics-execution-coverage').text(text)
+        }
+
+        $('#metrics-help-block-server').show()
+        calculateAndShowExecutionCoverage()
+        $('.coverage-status-checkbox').on('change', calculateAndShowExecutionCoverage)
+
+        const statusFormulas = statuses.map(status => `Test Coverage for status ${status.toUpperCase()} = (Number of tests with status ${status.toUpperCase()} / Total number of executed tests) × 100`)
+        const nota = 'Statuses are dynamically retrieved from the database and may vary depending on system configuration.'
+        $('#metrics-nota').text(nota)
+        const $list = $('#metrics-status-formulas-list')
+        $list.empty()
+        statusFormulas.forEach(function (f) {
+            $list.append($('<li>').text(f))
+        })
+
+        function getSelectedStatuses () {
+            const selectedStatuses = []
+            $('.coverage-status-checkbox:checked').each(function () {
+                selectedStatuses.push($(this).val())
+            })
+            return selectedStatuses
+        }
+
+        function calculateRowCoverage (row, selectedStatuses) {
+            if (!row.executed_cases) return '0.00'
+            const percentSum = selectedStatuses.reduce((acc, status) => acc + (row[`${status.toLowerCase()}_percent`] || 0), 0)
+            const filteredExecuted = Math.round(row.executed_cases * (percentSum / 100))
+            return (row.executed_cases ? (filteredExecuted / row.executed_cases * 100).toFixed(2) : '0.00')
+        }
+
+        const columns = [
+            {
+                data: 'test_run_id',
+                render: function (data, type, full, meta) {
+                    return `<a href="/runs/${data}/">TR-${data}</a>`
+                }
+            },
+            {
+                data: 'test_run_name',
+                render: function (data, type, full, meta) {
+                    return `<a href="/runs/${full.test_run_id}/">${data}</a>`
+                }
+            },
+            { data: 'start_date', title: 'Start Date' },
+            { data: 'stop_date', title: 'Stop Date' },
+            { data: 'test_plan_name', title: 'Test Plan Name' },
+            { data: 'total_planned_cases', title: 'Planned Cases' },
+            { data: 'executed_cases', title: 'Executed Cases' }
+        ]
+
+        statuses.forEach(status => {
+            columns.push({
+                data: `${status.toLowerCase()}_percent`,
+                title: `${status} (%)`
+            })
+        })
+
+        columns.push({
+            data: null,
+            title: 'Execution Coverage (%)',
+            render: function (data, type, row, meta) {
+                const selectedStatuses = getSelectedStatuses()
+                return calculateRowCoverage(row, selectedStatuses)
+            },
+            className: 'execution-coverage-col'
+        })
+
+        let theadHtml = `
+            <th>Test Run ID</th>
+            <th>Test Run Name</th>
+            <th>Start Date</th>
+            <th>Stop Date</th>
+            <th>Test Plan Name</th>
+            <th>Planned Cases</th>
+            <th>Executed Cases</th>
+        `
+        statuses.forEach(status => {
+            theadHtml += `<th>${status} (%)</th>`
+        })
+        theadHtml += '<th>Execution Coverage (%)</th>'
+        $('#resultsTableHead').html(theadHtml)
+
+        const dt = $('#resultsTable').DataTable({
+            destroy: true,
+            pageLength: $('#navbar').data('defaultpagesize'),
+            paging: true,
+            pagingType: 'bootstrap_input',
+            data,
+            select: {
+                className: 'success',
+                style: 'multi',
+                selector: 'td > input'
+            },
+            columns,
+            dom: 'Bptp',
+            buttons: exportButtons,
+            language: {
+                loadingRecords: '<div class="spinner spinner-lg"></div>',
+                emptyTable: 'No records found.',
+                processing: '<div class="spinner spinner-lg"></div>',
+                zeroRecords: 'No records found',
+                paginate: {
+                    previous: 'Previous',
+                    next: 'Next',
+                    first: 'First',
+                    last: 'Last'
+                }
+            },
+            order: [[2, 'desc']]
+        })
+
+        $('.coverage-status-checkbox').on('change', function () {
+            dt.rows().invalidate().draw(false)
+            calcularYMostrarExecutionCoverage()
+        })
+    })
+}
+
+$(document).ready(function () {
+    drawTable()
+
+    $(document).on('change', '#id_product, #id_version, #id_build, #id_test_plan', function () {
+        searchWithFilters()
+    })
+    $(document).on('input', '#id_test_run_summary', function () {
+        searchWithFilters()
+    })
+    $(document).on('change', '#id_after, #id_before', function () {
+        searchWithFilters()
+    })
+})
+
+function searchWithFilters () {
+    const query = {}
+    const products = $('#id_product').val()
+    if (products && products.length > 0) query.plan__product__in = products
+    const versions = $('#id_version').val()
+    if (versions && versions.length > 0) query.plan__product_version__in = versions
+    const builds = $('#id_build').val()
+    if (builds && builds.length > 0) query.build__in = builds
+    const testPlans = $('#id_test_plan').val()
+    if (testPlans && testPlans.length > 0) query.plan__in = testPlans
+    const after = $('#id_after').val()
+    if (after) query.start_date__gte = after
+    const before = $('#id_before').val()
+    if (before) query.start_date__lte = before
+    const summary = $('#id_test_run_summary').val()
+    if (summary) query.summary__icontains = summary
+
+    drawTable(query)
+}