diff --git a/.github/workflows/pulsar-ci.yaml b/.github/workflows/pulsar-ci.yaml index f7dbc755264d6..b9227ccf45915 100644 --- a/.github/workflows/pulsar-ci.yaml +++ b/.github/workflows/pulsar-ci.yaml @@ -190,19 +190,6 @@ jobs: group: BROKER_GROUP_1 - name: Brokers - Broker Group 2 group: BROKER_GROUP_2 - - name: Brokers - Broker Group 3 - group: BROKER_GROUP_3 - - name: Brokers - Client Api - group: BROKER_CLIENT_API - - name: Brokers - Client Impl - group: BROKER_CLIENT_IMPL - - name: Proxy - group: PROXY - - name: Pulsar IO - group: PULSAR_IO - timeout: 75 - - name: Pulsar Client - group: CLIENT steps: - name: checkout @@ -305,6 +292,10 @@ jobs: with: action: wait + - name: Setup tmate session + if: failure() + uses: mxschmitt/action-tmate@v3 + unit-tests-upload-coverage: name: CI - Unit - Upload Coverage diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt index 6b3455127b423..9205e072f7efe 100644 --- a/distribution/server/src/assemble/LICENSE.bin.txt +++ b/distribution/server/src/assemble/LICENSE.bin.txt @@ -343,35 +343,37 @@ The Apache Software License, Version 2.0 - org.apache.logging.log4j-log4j-slf4j-impl-2.18.0.jar - org.apache.logging.log4j-log4j-web-2.18.0.jar * Java Native Access JNA - - net.java.dev.jna-jna-5.12.1.jar - net.java.dev.jna-jna-jpms-5.12.1.jar - net.java.dev.jna-jna-platform-jpms-5.12.1.jar * BookKeeper - - org.apache.bookkeeper-bookkeeper-common-4.15.4.jar - - org.apache.bookkeeper-bookkeeper-common-allocator-4.15.4.jar - - org.apache.bookkeeper-bookkeeper-proto-4.15.4.jar - - org.apache.bookkeeper-bookkeeper-server-4.15.4.jar - - org.apache.bookkeeper-bookkeeper-tools-framework-4.15.4.jar - - org.apache.bookkeeper-circe-checksum-4.15.4.jar - - org.apache.bookkeeper-cpu-affinity-4.15.4.jar - - org.apache.bookkeeper-statelib-4.15.4.jar - - org.apache.bookkeeper-stream-storage-api-4.15.4.jar - - org.apache.bookkeeper-stream-storage-common-4.15.4.jar - - org.apache.bookkeeper-stream-storage-java-client-4.15.4.jar - - org.apache.bookkeeper-stream-storage-java-client-base-4.15.4.jar - - org.apache.bookkeeper-stream-storage-proto-4.15.4.jar - - org.apache.bookkeeper-stream-storage-server-4.15.4.jar - - org.apache.bookkeeper-stream-storage-service-api-4.15.4.jar - - org.apache.bookkeeper-stream-storage-service-impl-4.15.4.jar - - org.apache.bookkeeper.http-http-server-4.15.4.jar - - org.apache.bookkeeper.http-vertx-http-server-4.15.4.jar - - org.apache.bookkeeper.stats-bookkeeper-stats-api-4.15.4.jar - - org.apache.bookkeeper.stats-prometheus-metrics-provider-4.15.4.jar - - org.apache.distributedlog-distributedlog-common-4.15.4.jar - - org.apache.distributedlog-distributedlog-core-4.15.4-tests.jar - - org.apache.distributedlog-distributedlog-core-4.15.4.jar - - org.apache.distributedlog-distributedlog-protocol-4.15.4.jar - - org.apache.bookkeeper.stats-codahale-metrics-provider-4.15.4.jar + - org.apache.bookkeeper-bookkeeper-common-4.16.0.jar + - org.apache.bookkeeper-bookkeeper-common-allocator-4.16.0.jar + - org.apache.bookkeeper-bookkeeper-proto-4.16.0.jar + - org.apache.bookkeeper-bookkeeper-server-4.16.0.jar + - org.apache.bookkeeper-bookkeeper-tools-framework-4.16.0.jar + - org.apache.bookkeeper-circe-checksum-4.16.0.jar + - org.apache.bookkeeper-cpu-affinity-4.16.0.jar + - org.apache.bookkeeper-statelib-4.16.0.jar + - org.apache.bookkeeper-stream-storage-api-4.16.0.jar + - org.apache.bookkeeper-stream-storage-common-4.16.0.jar + - org.apache.bookkeeper-stream-storage-java-client-4.16.0.jar + - org.apache.bookkeeper-stream-storage-java-client-base-4.16.0.jar + - org.apache.bookkeeper-stream-storage-proto-4.16.0.jar + - org.apache.bookkeeper-stream-storage-server-4.16.0.jar + - org.apache.bookkeeper-stream-storage-service-api-4.16.0.jar + - org.apache.bookkeeper-stream-storage-service-impl-4.16.0.jar + - org.apache.bookkeeper.http-http-server-4.16.0.jar + - org.apache.bookkeeper.http-vertx-http-server-4.16.0.jar + - org.apache.bookkeeper.stats-bookkeeper-stats-api-4.16.0.jar + - org.apache.bookkeeper.stats-prometheus-metrics-provider-4.16.0.jar + - org.apache.distributedlog-distributedlog-common-4.16.0.jar + - org.apache.distributedlog-distributedlog-core-4.16.0-tests.jar + - org.apache.distributedlog-distributedlog-core-4.16.0.jar + - org.apache.distributedlog-distributedlog-protocol-4.16.0.jar + - org.apache.bookkeeper.stats-codahale-metrics-provider-4.16.0.jar + - org.apache.bookkeeper-bookkeeper-slogger-api-4.16.0.jar + - org.apache.bookkeeper-bookkeeper-slogger-slf4j-4.16.0.jar + - org.apache.bookkeeper-native-io-4.16.0.jar * Apache HTTP Client - org.apache.httpcomponents-httpclient-4.5.13.jar - org.apache.httpcomponents-httpcore-4.4.15.jar diff --git a/distribution/shell/src/assemble/LICENSE.bin.txt b/distribution/shell/src/assemble/LICENSE.bin.txt index 90896790b1fba..0ac08caa1c01d 100644 --- a/distribution/shell/src/assemble/LICENSE.bin.txt +++ b/distribution/shell/src/assemble/LICENSE.bin.txt @@ -390,9 +390,9 @@ The Apache Software License, Version 2.0 - log4j-web-2.18.0.jar * BookKeeper - - bookkeeper-common-allocator-4.15.4.jar - - cpu-affinity-4.15.4.jar - - circe-checksum-4.15.4.jar + - bookkeeper-common-allocator-4.16.0.jar + - cpu-affinity-4.16.0.jar + - circe-checksum-4.16.0.jar * AirCompressor - aircompressor-0.20.jar * AsyncHttpClient diff --git a/pom.xml b/pom.xml index 3f33069b8c252..5367b211c6ef9 100644 --- a/pom.xml +++ b/pom.xml @@ -126,7 +126,7 @@ flexible messaging model and an intuitive client API. 1.21 - 4.15.4 + 4.16.0 3.8.1 1.5.0 1.10.0 @@ -2455,5 +2455,13 @@ flexible messaging model and an intuitive client API. false + + bk-staging + bk-staging + https://repository.apache.org/content/repositories/orgapachebookkeeper-1082/ + + false + + diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/metrics/LongAdderCounter.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/metrics/LongAdderCounter.java index ff9d9302456d1..8ade2bc883f9a 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/metrics/LongAdderCounter.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/metrics/LongAdderCounter.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.broker.stats.prometheus.metrics; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.LongAdder; import org.apache.bookkeeper.stats.Counter; @@ -50,10 +51,16 @@ public void dec() { } @Override - public void add(long delta) { + public void addCount(long delta) { counter.add(delta); } + @Override + public void addLatency(long eventLatency, TimeUnit unit) { + long valueMillis = unit.toMillis(eventLatency); + counter.add(valueMillis); + } + @Override public Long get() { return counter.sum(); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/SameThreadOrderedSafeExecutor.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/SameThreadOrderedSafeExecutor.java index 5f6a32a61bd25..258188a31f5d0 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/SameThreadOrderedSafeExecutor.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/SameThreadOrderedSafeExecutor.java @@ -21,7 +21,6 @@ import io.netty.util.concurrent.DefaultThreadFactory; import org.apache.bookkeeper.common.util.OrderedExecutor; -import org.apache.bookkeeper.common.util.SafeRunnable; import org.apache.bookkeeper.stats.NullStatsLogger; public class SameThreadOrderedSafeExecutor extends OrderedExecutor { @@ -46,12 +45,12 @@ public void execute(Runnable r) { } @Override - public void executeOrdered(int orderingKey, SafeRunnable r) { + public void executeOrdered(int orderingKey, Runnable r) { r.run(); } @Override - public void executeOrdered(long orderingKey, SafeRunnable r) { + public void executeOrdered(long orderingKey, Runnable r) { r.run(); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/PrometheusMetricsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/PrometheusMetricsTest.java index 13e67762ace6f..e6573ce88971d 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/PrometheusMetricsTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/PrometheusMetricsTest.java @@ -1204,24 +1204,7 @@ public void testManagedLedgerBookieClientStats() throws Exception { System.out.println(e.getKey() + ": " + e.getValue()) ); - List cm = (List) metrics.get(keyNameBySubstrings(metrics, - "pulsar_managedLedger_client", "bookkeeper_ml_scheduler_completed_tasks")); - assertEquals(cm.size(), 1); - assertEquals(cm.get(0).tags.get("cluster"), "test"); - - cm = (List) metrics.get( - keyNameBySubstrings(metrics, - "pulsar_managedLedger_client", "bookkeeper_ml_scheduler_queue")); - assertEquals(cm.size(), 1); - assertEquals(cm.get(0).tags.get("cluster"), "test"); - - cm = (List) metrics.get( - keyNameBySubstrings(metrics, - "pulsar_managedLedger_client", "bookkeeper_ml_scheduler_total_tasks")); - assertEquals(cm.size(), 1); - assertEquals(cm.get(0).tags.get("cluster"), "test"); - - cm = (List) metrics.get( + List cm = (List) metrics.get( keyNameBySubstrings(metrics, "pulsar_managedLedger_client", "bookkeeper_ml_scheduler_threads")); assertEquals(cm.size(), 1); diff --git a/pulsar-sql/presto-distribution/LICENSE b/pulsar-sql/presto-distribution/LICENSE index c523fae7606e1..0c73a9908182e 100644 --- a/pulsar-sql/presto-distribution/LICENSE +++ b/pulsar-sql/presto-distribution/LICENSE @@ -424,18 +424,21 @@ The Apache Software License, Version 2.0 - async-http-client-2.12.1.jar - async-http-client-netty-utils-2.12.1.jar * Apache Bookkeeper - - bookkeeper-common-4.15.4.jar - - bookkeeper-common-allocator-4.15.4.jar - - bookkeeper-proto-4.15.4.jar - - bookkeeper-server-4.15.4.jar - - bookkeeper-stats-api-4.15.4.jar - - bookkeeper-tools-framework-4.15.4.jar - - circe-checksum-4.15.4.jar - - codahale-metrics-provider-4.15.4.jar - - cpu-affinity-4.15.4.jar - - http-server-4.15.4.jar - - prometheus-metrics-provider-4.15.4.jar - - codahale-metrics-provider-4.15.4.jar + - bookkeeper-common-4.16.0.jar + - bookkeeper-common-allocator-4.16.0.jar + - bookkeeper-proto-4.16.0.jar + - bookkeeper-server-4.16.0.jar + - bookkeeper-stats-api-4.16.0.jar + - bookkeeper-tools-framework-4.16.0.jar + - circe-checksum-4.16.0.jar + - codahale-metrics-provider-4.16.0.jar + - cpu-affinity-4.16.0.jar + - http-server-4.16.0.jar + - prometheus-metrics-provider-4.16.0.jar + - codahale-metrics-provider-4.16.0.jar + - bookkeeper-slogger-api-4.16.0.jar + - bookkeeper-slogger-slf4j-4.16.0.jar + - native-io-4.16.0.jar * Apache Commons - commons-cli-1.5.0.jar - commons-codec-1.15.jar @@ -476,6 +479,8 @@ The Apache Software License, Version 2.0 - amqp-client-5.5.3.jar * Stream Lib - stream-2.9.5.jar + * High Performance Primitive Collections for Java + - hppc-0.9.1.jar Protocol Buffers License diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarConnectorMetricsTracker.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarConnectorMetricsTracker.java index b586063cc3c9a..12ee2da463c40 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarConnectorMetricsTracker.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarConnectorMetricsTracker.java @@ -185,7 +185,7 @@ public void end_ENTRY_QUEUE_DEQUEUE_WAIT_TIME() { public void register_BYTES_READ(long bytes) { if (statsLogger != null) { bytesReadSum += bytes; - statsLoggerBytesRead.add(bytes); + statsLoggerBytesRead.addCount(bytes); } } @@ -220,7 +220,7 @@ public void end_MESSAGE_QUEUE_ENQUEUE_WAIT_TIME() { public void incr_NUM_MESSAGES_DESERIALIZED_PER_ENTRY() { if (statsLogger != null) { numMessagedDerserializedPerBatch++; - statsLoggerNumMessagesDeserialized.add(1); + statsLoggerNumMessagesDeserialized.addCount(1); } } @@ -295,7 +295,7 @@ public void end_RECORD_DESERIALIZE_TIME() { public void incr_NUM_RECORD_DESERIALIZED() { if (statsLogger != null) { - statsLoggerNumRecordDeserialized.add(1); + statsLoggerNumRecordDeserialized.addCount(1); } } diff --git a/testmocks/src/main/java/org/apache/bookkeeper/client/TestStatsProvider.java b/testmocks/src/main/java/org/apache/bookkeeper/client/TestStatsProvider.java index f015306cb048c..4d08a7f80df5b 100644 --- a/testmocks/src/main/java/org/apache/bookkeeper/client/TestStatsProvider.java +++ b/testmocks/src/main/java/org/apache/bookkeeper/client/TestStatsProvider.java @@ -59,10 +59,16 @@ public void dec() { } @Override - public void add(long delta) { + public void addCount(long delta) { updateMax(val.addAndGet(delta)); } + @Override + public void addLatency(long eventLatency, TimeUnit unit) { + long valueMillis = unit.toMillis(eventLatency); + updateMax(val.addAndGet(valueMillis)); + } + @Override public Long get() { return val.get();