Skip to content

Commit 0a6fa4b

Browse files
merlimatmerlimat
authored
Distribute Pulsar IO connectors individually (#3601)
1 parent 6fba72c commit 0a6fa4b

File tree

5 files changed

+52
-92
lines changed

5 files changed

+52
-92
lines changed

‎distribution/io/pom.xml

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -59,8 +59,7 @@
5959
<goal>single</goal>
6060
</goals>
6161
<configuration>
62-
<attach>true</attach>
63-
<tarLongFileMode>posix</tarLongFileMode>
62+
<attach>false</attach>
6463
<finalName>apache-pulsar-io-connectors-${project.version}</finalName>
6564
<descriptors>
6665
<descriptor>src/assemble/io.xml</descriptor>

‎distribution/io/src/assemble/io.xml

Lines changed: 26 additions & 72 deletions
Original file line numberDiff line numberDiff line change
@@ -24,15 +24,9 @@
2424
xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
2525
<id>bin</id>
2626
<formats>
27-
<format>tar.gz</format>
27+
<format>dir</format>
2828
</formats>
29-
<includeBaseDirectory>true</includeBaseDirectory>
30-
<fileSets>
31-
<fileSet>
32-
<directory>${basedir}/../../distribution/io/target/conf</directory>
33-
<outputDirectory>conf</outputDirectory>
34-
</fileSet>
35-
</fileSets>
29+
<includeBaseDirectory>false</includeBaseDirectory>
3630
<files>
3731
<file>
3832
<source>${basedir}/../../LICENSE</source>
@@ -46,70 +40,30 @@
4640
<fileMode>644</fileMode>
4741
</file>
4842

49-
<file>
50-
<source>${basedir}/../../pulsar-io/cassandra/target/pulsar-io-cassandra-${project.version}.nar</source>
51-
<outputDirectory>connectors</outputDirectory>
52-
<fileMode>644</fileMode>
53-
</file>
54-
55-
<file>
56-
<source>${basedir}/../../pulsar-io/twitter/target/pulsar-io-twitter-${project.version}.nar</source>
57-
<outputDirectory>connectors</outputDirectory>
58-
<fileMode>644</fileMode>
59-
</file>
60-
61-
<file>
62-
<source>${basedir}/../../pulsar-io/kafka/target/pulsar-io-kafka-${project.version}.nar</source>
63-
<outputDirectory>connectors</outputDirectory>
64-
<fileMode>644</fileMode>
65-
</file>
66-
67-
<file>
68-
<source>${basedir}/../../pulsar-io/kinesis/target/pulsar-io-kinesis-${project.version}.nar</source>
69-
<outputDirectory>connectors</outputDirectory>
70-
<fileMode>644</fileMode>
71-
</file>
72-
73-
<file>
74-
<source>${basedir}/../../pulsar-io/rabbitmq/target/pulsar-io-rabbitmq-${project.version}.nar</source>
75-
<outputDirectory>connectors</outputDirectory>
76-
<fileMode>644</fileMode>
77-
</file>
78-
79-
<file>
80-
<source>${basedir}/../../pulsar-io/jdbc/target/pulsar-io-jdbc-${project.version}.nar</source>
81-
<outputDirectory>connectors</outputDirectory>
82-
<fileMode>644</fileMode>
83-
</file>
84-
85-
<file>
86-
<source>${basedir}/../../pulsar-io/data-generator/target/pulsar-io-data-generator-${project.version}.nar</source>
87-
<outputDirectory>connectors</outputDirectory>
88-
<fileMode>644</fileMode>
89-
</file>
43+
<!-- Include all connectors.
44+
There doesn't seem to be a straight way to automatically
45+
include all nars and place them in a single directory.
46+
-->
9047

91-
<file>
92-
<source>${basedir}/../../pulsar-io/aerospike/target/pulsar-io-aerospike-${project.version}.nar</source>
93-
<outputDirectory>connectors</outputDirectory>
94-
<fileMode>644</fileMode>
95-
</file>
96-
97-
<file>
98-
<source>${basedir}/../../pulsar-io/elastic-search/target/pulsar-io-elastic-search-${project.version}.nar</source>
99-
<outputDirectory>connectors</outputDirectory>
100-
<fileMode>644</fileMode>
101-
</file>
102-
103-
<file>
104-
<source>${basedir}/../../pulsar-io/kafka-connect-adaptor/target/pulsar-io-kafka-connect-adaptor-${project.version}.nar</source>
105-
<outputDirectory>connectors</outputDirectory>
106-
<fileMode>644</fileMode>
107-
</file>
108-
109-
<file>
110-
<source>${basedir}/../../pulsar-io/hbase/target/pulsar-io-hbase-${project.version}.nar</source>
111-
<outputDirectory>connectors</outputDirectory>
112-
<fileMode>644</fileMode>
113-
</file>
48+
<file><source>${basedir}/../../pulsar-io/cassandra/target/pulsar-io-cassandra-${project.version}.nar</source></file>
49+
<file><source>${basedir}/../../pulsar-io/twitter/target/pulsar-io-twitter-${project.version}.nar</source></file>
50+
<file><source>${basedir}/../../pulsar-io/kafka/target/pulsar-io-kafka-${project.version}.nar</source></file>
51+
<file><source>${basedir}/../../pulsar-io/kinesis/target/pulsar-io-kinesis-${project.version}.nar</source></file>
52+
<file><source>${basedir}/../../pulsar-io/rabbitmq/target/pulsar-io-rabbitmq-${project.version}.nar</source></file>
53+
<file><source>${basedir}/../../pulsar-io/jdbc/target/pulsar-io-jdbc-${project.version}.nar</source></file>
54+
<file><source>${basedir}/../../pulsar-io/data-generator/target/pulsar-io-data-generator-${project.version}.nar</source></file>
55+
<file><source>${basedir}/../../pulsar-io/aerospike/target/pulsar-io-aerospike-${project.version}.nar</source></file>
56+
<file><source>${basedir}/../../pulsar-io/elastic-search/target/pulsar-io-elastic-search-${project.version}.nar</source></file>
57+
<file><source>${basedir}/../../pulsar-io/kafka-connect-adaptor/target/pulsar-io-kafka-connect-adaptor-${project.version}.nar</source></file>
58+
<file><source>${basedir}/../../pulsar-io/hbase/target/pulsar-io-hbase-${project.version}.nar</source></file>
59+
<file><source>${basedir}/../../pulsar-io/kinesis/target/pulsar-io-kinesis-${project.version}.nar</source></file>
60+
<file><source>${basedir}/../../pulsar-io/hdfs2/target/pulsar-io-hdfs2-${project.version}.nar</source></file>
61+
<file><source>${basedir}/../../pulsar-io/hdfs3/target/pulsar-io-hdfs3-${project.version}.nar</source></file>
62+
<file><source>${basedir}/../../pulsar-io/file/target/pulsar-io-file-${project.version}.nar</source></file>
63+
<file><source>${basedir}/../../pulsar-io/data-generator/target/pulsar-io-data-generator-${project.version}.nar</source></file>
64+
<file><source>${basedir}/../../pulsar-io/canal/target/pulsar-io-canal-${project.version}.nar</source></file>
65+
<file><source>${basedir}/../../pulsar-io/netty/target/pulsar-io-netty-${project.version}.nar</source></file>
66+
<file><source>${basedir}/../../pulsar-io/debezium/target/pulsar-io-debezium-${project.version}.nar</source></file>
67+
<file><source>${basedir}/../../pulsar-io/mongo/target/pulsar-io-mongo-${project.version}.nar</source></file>
11468
</files>
11569
</assembly>

‎docker/pulsar-all/Dockerfile

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -19,11 +19,10 @@
1919

2020
FROM apachepulsar/pulsar:latest
2121

22-
ARG PULSAR_IO_TARBALL
22+
ARG PULSAR_IO_DIR
2323
ARG PULSAR_OFFLOADER_TARBALL
2424

25-
ADD ${PULSAR_IO_TARBALL} /
26-
RUN mv /apache-pulsar-io-connectors-*/connectors /pulsar/connectors
25+
ADD ${PULSAR_IO_DIR} /pulsar/connectors
2726

2827
ADD ${PULSAR_OFFLOADER_TARBALL} /
2928
RUN mv /apache-pulsar-offloaders-*/offloaders /pulsar/offloaders

‎docker/pulsar-all/pom.xml

Lines changed: 19 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,6 @@
2626
<version>2.3.0-SNAPSHOT</version>
2727
</parent>
2828
<modelVersion>4.0.0</modelVersion>
29-
<groupId>org.apache.pulsar</groupId>
3029
<artifactId>pulsar-all-docker-image</artifactId>
3130
<name>Apache Pulsar :: Docker Images :: Pulsar Latest Version (Include All Components)</name>
3231
<packaging>pom</packaging>
@@ -36,8 +35,7 @@
3635
<groupId>org.apache.pulsar</groupId>
3736
<artifactId>pulsar-io-distribution</artifactId>
3837
<version>${project.parent.version}</version>
39-
<classifier>bin</classifier>
40-
<type>tar.gz</type>
38+
<type>pom</type>
4139
<scope>provided</scope>
4240
</dependency>
4341
<dependency>
@@ -65,21 +63,30 @@
6563
<build>
6664
<plugins>
6765
<plugin>
68-
<groupId>org.apache.maven.plugins</groupId>
69-
<artifactId>maven-dependency-plugin</artifactId>
66+
<artifactId>maven-resources-plugin</artifactId>
7067
<executions>
7168
<execution>
72-
<id>copy-io-tarball</id>
69+
<id>copy-resources</id>
70+
<phase>generate-resources</phase>
7371
<goals>
74-
<goal>copy-dependencies</goal>
72+
<goal>copy-resources</goal>
7573
</goals>
76-
<phase>generate-resources</phase>
7774
<configuration>
78-
<outputDirectory>${project.build.directory}/</outputDirectory>
79-
<includeArtifactIds>pulsar-io-distribution</includeArtifactIds>
80-
<excludeTransitive>true</excludeTransitive>
75+
<outputDirectory>${basedir}/target/apache-pulsar-io-connectors-${project.version}-bin</outputDirectory>
76+
<resources>
77+
<resource>
78+
<directory>${basedir}/../../distribution/io/target/apache-pulsar-io-connectors-${project.version}-bin</directory>
79+
<filtering>false</filtering>
80+
</resource>
81+
</resources>
8182
</configuration>
8283
</execution>
84+
</executions>
85+
</plugin>
86+
<plugin>
87+
<groupId>org.apache.maven.plugins</groupId>
88+
<artifactId>maven-dependency-plugin</artifactId>
89+
<executions>
8390
<execution>
8491
<id>copy-offloader-tarball</id>
8592
<goals>
@@ -122,7 +129,7 @@
122129
<pullNewerImage>false</pullNewerImage>
123130
<tag>${project.version}</tag>
124131
<buildArgs>
125-
<PULSAR_IO_TARBALL>target/pulsar-io-distribution-${project.version}-bin.tar.gz</PULSAR_IO_TARBALL>
132+
<PULSAR_IO_DIR>target/apache-pulsar-io-connectors-${project.version}-bin</PULSAR_IO_DIR>
126133
<PULSAR_OFFLOADER_TARBALL>target/pulsar-offloader-distribution-${project.version}-bin.tar.gz</PULSAR_OFFLOADER_TARBALL>
127134
</buildArgs>
128135
</configuration>

‎src/stage-release.sh

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -27,16 +27,17 @@ fi
2727

2828
DEST_PATH=$1
2929

30-
pushd $(dirname "$0")
30+
pushd pushd $(dirname "$0")
3131
PULSAR_PATH=$(git rev-parse --show-toplevel)
3232
VERSION=`./get-project-version.py`
3333
popd
3434

3535
cp $PULSAR_PATH/distribution/server/target/apache-pulsar-$VERSION-src.tar.gz $DEST_PATH
3636
cp $PULSAR_PATH/distribution/server/target/apache-pulsar-$VERSION-bin.tar.gz $DEST_PATH
37-
cp $PULSAR_PATH/distribution/io/target/apache-pulsar-io-connectors-$VERSION-bin.tar.gz $DEST_PATH
3837
cp $PULSAR_PATH/distribution/offloaders/target/apache-pulsar-offloaders-$VERSION-bin.tar.gz $DEST_PATH
3938

39+
cp -r $PULSAR_PATH/distribution/io/target/apache-pulsar-io-connectors-$VERSION-bin $DEST_PATH/connectors
40+
4041
mkdir $DEST_PATH/RPMS
4142
cp -r $PULSAR_PATH/pulsar-client-cpp/pkg/rpm/RPMS/x86_64/* $DEST_PATH/RPMS
4243

@@ -45,4 +46,4 @@ cp -r $PULSAR_PATH/pulsar-client-cpp/pkg/deb/BUILD/DEB/* $DEST_PATH/DEB
4546

4647
# Sign all files
4748
cd $DEST_PATH
48-
find . -type f | xargs $PULSAR_PATH/src/sign-release.sh
49+
find . -type f | grep -v LICENSE | grep -v README | xargs $PULSAR_PATH/src/sign-release.sh

0 commit comments

Comments
 (0)