Skip to content

Commit a71c90c

Browse files
committed
Merge remote-tracking branch 'apache/master' into fix-lgtm-alerts
2 parents eaed7f3 + 9140e3f commit a71c90c

File tree

245 files changed

+6017
-1090
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

245 files changed

+6017
-1090
lines changed

conf/bookkeeper.conf

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -590,6 +590,16 @@ zkEnableSecurity=false
590590
## Server parameters
591591
#############################################################################
592592

593+
# The flag enables/disables starting the admin http server. Default value is 'false'.
594+
httpServerEnabled=false
595+
596+
# The http server port to listen on. Default value is 8080.
597+
# Use `8000` as the port to keep it consistent with prometheus stats provider
598+
httpServerPort=8000
599+
600+
# The http server class
601+
httpServerClass=org.apache.bookkeeper.http.vertx.VertxHttpServer
602+
593603
# Configure a list of server components to enable and load on a bookie server.
594604
# This provides the plugin run extra services along with a bookie server.
595605
#

conf/broker.conf

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -567,6 +567,9 @@ exposeTopicLevelMetricsInPrometheus=true
567567
# Enable consumer level metrics. default is false
568568
exposeConsumerLevelMetricsInPrometheus=false
569569

570+
# Classname of Pluggable JVM GC metrics logger that can log GC specific metrics
571+
# jvmGCMetricsLoggerClassName=
572+
570573
### --- Functions --- ###
571574

572575
# Enable Functions Worker Service in Broker

conf/functions_worker.yml

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,8 @@ workerPortTls: 6751
3232

3333
numFunctionPackageReplicas: 1
3434
downloadDirectory: /tmp/pulsar_functions
35+
# Classname of Pluggable JVM GC metrics logger that can log GC specific metrics
36+
# jvmGCMetricsLoggerClassName:
3537

3638
#################################################################
3739
# Function metadata managment (assignment, scheduling, and etc)
@@ -109,6 +111,14 @@ processContainerFactory:
109111
# # if it is not an absolute path, it is relative to `pulsarRootDir`
110112
# extraFunctionDependenciesDir:
111113

114+
## A set of the minimum amount of resources functions must request.
115+
## Support for this depends on function runtime.
116+
## Only kubernetes runtime currently supports this.
117+
# functionInstanceMinResources:
118+
# cpu: 1
119+
# ram: 1073741824
120+
# disk: 10737418240
121+
112122
############################################
113123
# security settings for worker service
114124
############################################

conf/standalone.conf

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -429,6 +429,9 @@ webSocketSessionIdleTimeoutMillis=300000
429429
# Enable topic level metrics
430430
exposeTopicLevelMetricsInPrometheus=true
431431

432+
# Classname of Pluggable JVM GC metrics logger that can log GC specific metrics
433+
# jvmGCMetricsLoggerClassName=
434+
432435
### --- Broker Web Stats --- ###
433436

434437
# Enable topic level metrics

dashboard/README.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,4 +25,4 @@ Pulsar dashboard is a web application that monitors
2525
a complete instance of Pulsar and gives insights
2626
about about all the topics.
2727

28-
See [docs/Dashboard.md](https://github.com/apache/pulsar/blob/master/docs/Dashboard.md) for more information.
28+
See [site2/docs/administration-dashboard.md](https://github.com/apache/pulsar/blob/master/site2/docs/administration-dashboard.md) for more information.

distribution/io/src/assemble/io.xml

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -105,5 +105,11 @@
105105
<outputDirectory>connectors</outputDirectory>
106106
<fileMode>644</fileMode>
107107
</file>
108+
109+
<file>
110+
<source>${basedir}/../../pulsar-io/hbase/target/pulsar-io-hbase-${project.version}.nar</source>
111+
<outputDirectory>connectors</outputDirectory>
112+
<fileMode>644</fileMode>
113+
</file>
108114
</files>
109115
</assembly>

distribution/server/pom.xml

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -157,6 +157,19 @@
157157
<groupId>io.grpc</groupId>
158158
<artifactId>grpc-all</artifactId>
159159
</dependency>
160+
161+
<!-- bookkeeper http server -->
162+
<dependency>
163+
<groupId>org.apache.bookkeeper.http</groupId>
164+
<artifactId>vertx-http-server</artifactId>
165+
<version>${bookkeeper.version}</version>
166+
<exclusions>
167+
<exclusion>
168+
<groupId>io.netty</groupId>
169+
<artifactId>*</artifactId>
170+
</exclusion>
171+
</exclusions>
172+
</dependency>
160173
</dependencies>
161174

162175
<build>

distribution/server/src/assemble/LICENSE.bin.txt

Lines changed: 12 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -382,6 +382,7 @@ The Apache Software License, Version 2.0
382382
- org.apache.bookkeeper-stream-storage-service-api-4.7.3.jar
383383
- org.apache.bookkeeper-stream-storage-service-impl-4.7.3.jar
384384
- org.apache.bookkeeper.http-http-server-4.7.3.jar
385+
- org.apache.bookkeeper.http-vertx-http-server-4.7.3.jar
385386
- org.apache.bookkeeper.stats-bookkeeper-stats-api-4.7.3.jar
386387
- org.apache.bookkeeper.stats-prometheus-metrics-provider-4.7.3.jar
387388
- org.apache.bookkeeper.tests-stream-storage-tests-common-4.7.3.jar
@@ -392,8 +393,8 @@ The Apache Software License, Version 2.0
392393
- org.apache.bookkeeper.stats-codahale-metrics-provider-4.7.3.jar
393394
* LZ4 -- org.lz4-lz4-java-1.5.0.jar
394395
* AsyncHttpClient
395-
- org.asynchttpclient-async-http-client-2.1.0-alpha26.jar
396-
- org.asynchttpclient-async-http-client-netty-utils-2.1.0-alpha26.jar
396+
- org.asynchttpclient-async-http-client-2.7.0.jar
397+
- org.asynchttpclient-async-http-client-netty-utils-2.7.0.jar
397398
* Jetty
398399
- org.eclipse.jetty-jetty-client-9.4.12.v20180830.jar
399400
- org.eclipse.jetty-jetty-continuation-9.4.12.v20180830.jar
@@ -476,6 +477,10 @@ The Apache Software License, Version 2.0
476477
- io.jsonwebtoken-jjwt-jackson-0.10.5.jar
477478
* JavaX Injection
478479
- javax.inject-javax.inject-1.jar
480+
* Vertx
481+
- io.vertx-vertx-auth-common-3.4.1.jar
482+
- io.vertx-vertx-core-3.4.1.jar
483+
- io.vertx-vertx-web-3.4.1.jar
479484

480485
BSD 3-clause "New" or "Revised" License
481486
* Google auth library
@@ -507,6 +512,7 @@ CDDL-1.1 -- licenses/LICENSE-CDDL-1.1.txt
507512
* Java Annotations API
508513
- javax.annotation-javax.annotation-api-1.2.jar
509514
- javax.activation-javax.activation-api-1.2.0.jar
515+
- com.sun.activation-javax.activation-1.2.0.jar
510516
- javax.xml.bind-activation-1.0.2.jar
511517
- javax.xml.bind-jaxb-api-2.3.1.jar
512518
* Java Servlet API -- javax.servlet-javax.servlet-api-3.1.0.jar
@@ -542,13 +548,13 @@ Public Domain
542548
- org.tukaani-xz-1.5.jar
543549

544550
Public Domain (CC0) -- licenses/LICENSE-CC0.txt
545-
* Reactive Streams -- org.reactivestreams-reactive-streams-1.0.0.jar
551+
* Reactive Streams -- org.reactivestreams-reactive-streams-1.0.2.jar
546552

547553
Bouncy Castle License
548554
* Bouncy Castle -- licenses/LICENSE-bouncycastle.txt
549-
- org.bouncycastle-bcpkix-jdk15on-1.55.jar
550-
- org.bouncycastle-bcprov-jdk15on-1.55.jar
551-
- org.bouncycastle-bcprov-ext-jdk15on-1.59.jar
555+
- org.bouncycastle-bcpkix-jdk15on-1.60.jar
556+
- org.bouncycastle-bcprov-jdk15on-1.60.jar
557+
- org.bouncycastle-bcprov-ext-jdk15on-1.60.jar
552558

553559
------------------------
554560

docker/pulsar/scripts/watch-znode.py

Lines changed: 14 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -26,18 +26,21 @@
2626
logging.getLogger('kazoo.client').addHandler(logging.StreamHandler())
2727

2828
def usage():
29-
print("\n%s -z <zookeeper> -p <path> [-w|-c|-e]" % (sys.argv[0]), file=sys.stderr)
30-
print("\nWait for, or create znode", file=sys.stderr)
31-
print("\n-z Specify zookeeper connect string", file=sys.stderr)
32-
print("\n-p Znode path to watch or create", file=sys.stderr)
33-
print("\n-w Watch for path creation", file=sys.stderr)
34-
print("\n-c Create path", file=sys.stderr)
35-
print("\n-e Check if znode exists", file=sys.stderr)
29+
sys.stderr.write('''
30+
%s -z <zookeeper> -p <path> [-w|-c|-e]
31+
32+
Wait for, or create znode
33+
-z Specify zookeeper connect string
34+
-p Znode path to watch or create
35+
-w Watch for path creation
36+
-c Create path
37+
-e Check if znode exists
38+
''' % (sys.argv[0]))
3639

3740
try:
3841
opts, args = getopt.getopt(sys.argv[1:], "z:p:cweh")
3942
except getopt.GetoptError as err:
40-
print(str(err), file=sys.stderr)
43+
sys.stderr.write(str(err) + '\n')
4144
usage()
4245
sys.exit(2)
4346

@@ -67,17 +70,17 @@ def usage():
6770
sys.exit(2)
6871

6972
if not zookeeper:
70-
print("Zookeeper must be specified", file=sys.stderr)
73+
sys.stderr.write("Zookeeper must be specified\n")
7174
usage()
7275
sys.exit(3)
7376

7477
if not znode:
75-
print("Znode must be specified", file=sys.stderr)
78+
sys.stderr.write("Znode must be specified\n")
7679
usage()
7780
sys.exit(4)
7881

7982
if (not watch and not create and not exists):
80-
print("Exactly one of watch (-w), create (-c) or exists (-e) must be specified", file=sys.stderr)
83+
sys.stderr.write("Exactly one of watch (-w), create (-c) or exists (-e) must be specified\n")
8184
usage()
8285
sys.exit(5)
8386

managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java

Lines changed: 32 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -166,6 +166,7 @@ enum State {
166166
NoLedger, // There is no metadata ledger open for writing
167167
Open, // Metadata ledger is ready
168168
SwitchingLedger, // The metadata ledger is being switched
169+
Closing, // The managed cursor is closing
169170
Closed // The managed cursor has been closed
170171
}
171172

@@ -282,6 +283,15 @@ protected void recoverFromLedger(final ManagedCursorInfo info, final VoidCallbac
282283

283284
// Read the last entry in the ledger
284285
long lastEntryInLedger = lh.getLastAddConfirmed();
286+
287+
if (lastEntryInLedger < 0) {
288+
log.warn("[{}] Error reading from metadata ledger {} for consumer {}: No entries in ledger",
289+
ledger.getName(), ledgerId, name);
290+
// Rewind to last cursor snapshot available
291+
initialize(getRollbackPosition(info), callback);
292+
return;
293+
}
294+
285295
lh.asyncReadEntries(lastEntryInLedger, lastEntryInLedger, (rc1, lh1, seq, ctx1) -> {
286296
if (log.isDebugEnabled()) {
287297
log.debug("[{}} readComplete rc={} entryId={}", ledger.getName(), rc1, lh1.getLastAddConfirmed());
@@ -430,7 +440,7 @@ public void readEntriesFailed(ManagedLedgerException exception, Object ctx) {
430440
public void asyncReadEntries(final int numberOfEntriesToRead, final ReadEntriesCallback callback,
431441
final Object ctx) {
432442
checkArgument(numberOfEntriesToRead > 0);
433-
if (STATE_UPDATER.get(this) == State.Closed) {
443+
if (isClosed()) {
434444
callback.readEntriesFailed(new ManagedLedgerException("Cursor was already closed"), ctx);
435445
return;
436446
}
@@ -480,7 +490,7 @@ public void readEntryComplete(Entry entry, Object ctx) {
480490
public void asyncGetNthEntry(int n, IndividualDeletedEntries deletedEntries, ReadEntryCallback callback,
481491
Object ctx) {
482492
checkArgument(n > 0);
483-
if (STATE_UPDATER.get(this) == State.Closed) {
493+
if (isClosed()) {
484494
callback.readEntryFailed(new ManagedLedgerException("Cursor was already closed"), ctx);
485495
return;
486496
}
@@ -545,7 +555,7 @@ public void readEntriesFailed(ManagedLedgerException exception, Object ctx) {
545555
@Override
546556
public void asyncReadEntriesOrWait(int numberOfEntriesToRead, ReadEntriesCallback callback, Object ctx) {
547557
checkArgument(numberOfEntriesToRead > 0);
548-
if (STATE_UPDATER.get(this) == State.Closed) {
558+
if (isClosed()) {
549559
callback.readEntriesFailed(new CursorAlreadyClosedException("Cursor was already closed"), ctx);
550560
return;
551561
}
@@ -619,6 +629,10 @@ public void asyncReadEntriesOrWait(int numberOfEntriesToRead, ReadEntriesCallbac
619629
}
620630
}
621631

632+
private boolean isClosed() {
633+
return state == State.Closed || state == State.Closing;
634+
}
635+
622636
@Override
623637
public boolean cancelPendingReadRequest() {
624638
if (log.isDebugEnabled()) {
@@ -1342,7 +1356,7 @@ public void asyncMarkDelete(final Position position, Map<String, Long> propertie
13421356
checkNotNull(position);
13431357
checkArgument(position instanceof PositionImpl);
13441358

1345-
if (STATE_UPDATER.get(this) == State.Closed) {
1359+
if (isClosed()) {
13461360
callback.markDeleteFailed(new ManagedLedgerException("Cursor was already closed"), ctx);
13471361
return;
13481362
}
@@ -1558,7 +1572,7 @@ public void deleteFailed(ManagedLedgerException exception, Object ctx) {
15581572

15591573
@Override
15601574
public void asyncDelete(Iterable<Position> positions, AsyncCallbacks.DeleteCallback callback, Object ctx) {
1561-
if (state == State.Closed) {
1575+
if (isClosed()) {
15621576
callback.deleteFailed(new ManagedLedgerException("Cursor was already closed"), ctx);
15631577
return;
15641578
}
@@ -1876,6 +1890,14 @@ private boolean shouldPersistUnackRangesToLedger() {
18761890

18771891
private void persistPositionMetaStore(long cursorsLedgerId, PositionImpl position, Map<String, Long> properties,
18781892
MetaStoreCallback<Void> callback, boolean persistIndividualDeletedMessageRanges) {
1893+
if (state == State.Closed) {
1894+
ledger.getExecutor().execute(safeRun(() -> {
1895+
callback.operationFailed(new MetaStoreException(
1896+
new ManagedLedgerException.CursorAlreadyClosedException(name + " cursor already closed")));
1897+
}));
1898+
return;
1899+
}
1900+
18791901
// When closing we store the last mark-delete position in the z-node itself, so we won't need the cursor ledger,
18801902
// hence we write it as -1. The cursor ledger is deleted once the z-node write is confirmed.
18811903
ManagedCursorInfo.Builder info = ManagedCursorInfo.newBuilder() //
@@ -1910,13 +1932,14 @@ public void operationFailed(MetaStoreException e) {
19101932

19111933
@Override
19121934
public void asyncClose(final AsyncCallbacks.CloseCallback callback, final Object ctx) {
1913-
State oldState = STATE_UPDATER.getAndSet(this, State.Closed);
1914-
if (oldState == State.Closed) {
1935+
State oldState = STATE_UPDATER.getAndSet(this, State.Closing);
1936+
if (oldState == State.Closed || oldState == State.Closing) {
19151937
log.info("[{}] [{}] State is already closed", ledger.getName(), name);
19161938
callback.closeComplete(ctx);
19171939
return;
19181940
}
19191941
persistPosition(-1, lastMarkDeleteEntry.newPosition, lastMarkDeleteEntry.properties, callback, ctx);
1942+
STATE_UPDATER.set(this, State.Closed);
19201943
}
19211944

19221945
/**
@@ -2059,7 +2082,7 @@ public void deleteComplete(int rc, Object ctx) {
20592082
});
20602083
}));
20612084
}, Collections.emptyMap());
2062-
2085+
20632086
}
20642087

20652088
private List<LongProperty> buildPropertiesMap(Map<String, Long> properties) {
@@ -2166,7 +2189,7 @@ boolean shouldCloseLedger(LedgerHandle lh) {
21662189
long now = clock.millis();
21672190
if ((lh.getLastAddConfirmed() >= config.getMetadataMaxEntriesPerLedger()
21682191
|| lastLedgerSwitchTimestamp < (now - config.getLedgerRolloverTimeout() * 1000))
2169-
&& STATE_UPDATER.get(this) != State.Closed) {
2192+
&& (STATE_UPDATER.get(this) != State.Closed && STATE_UPDATER.get(this) != State.Closing)) {
21702193
// It's safe to modify the timestamp since this method will be only called from a callback, implying that
21712194
// calls will be serialized on one single thread
21722195
lastLedgerSwitchTimestamp = now;

0 commit comments

Comments
 (0)