Skip to content

Fix broker request id generator to avoid generating same id (#16661) #38028

Fix broker request id generator to avoid generating same id (#16661)

Fix broker request id generator to avoid generating same id (#16661) #38028

Workflow file for this run

#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
name: Pinot Tests
on:
push:
branches:
- master
paths-ignore:
- "contrib/**"
- "docs/**"
- "docker/**"
- "kubernetes/**"
- "licenses/**"
- "licenses-binary/**"
- "**.md"
pull_request:
branches:
- master
paths-ignore:
- "contrib/**"
- "docs/**"
- "docker/**"
- "kubernetes/**"
- "licenses/**"
- "licenses-binary/**"
- "**.md"
jobs:
linter-test:
if: github.repository == 'apache/pinot'
runs-on: ubuntu-latest
name: Pinot Linter Test Set
steps:
- uses: actions/checkout@v4
- name: Set up JDK 11
uses: actions/setup-java@v4
with:
java-version: 11
distribution: 'temurin'
cache: 'maven'
- name: Install pinot-dependency-verifier into repo
run: |
mvn clean install \
-pl pinot-dependency-verifier \
-am \
-DskipTests
- name: Linter Test
env:
DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }}
MAVEN_OPTS: >
-Xmx2G -DskipShade -DfailIfNoTests=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=25
-Dmaven.wagon.http.retryHandler.count=30 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false
-B -ntp
-XX:+IgnoreUnrecognizedVMOptions
--add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED
--add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED
--add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED
--add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED
--add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED
run: .github/workflows/scripts/.pinot_linter.sh
binary-compat-check:
if: github.repository == 'apache/pinot'
runs-on: ubuntu-latest
name: Pinot Binary Compatibility Check
steps:
- uses: actions/checkout@v4
- name: Set up JDK 11
uses: actions/setup-java@v4
with:
java-version: 11
distribution: 'temurin'
cache: 'maven'
# Step that does that actual cache save and restore
- uses: actions/cache@v4
env:
SEGMENT_DOWNLOAD_TIMEOUT_MINS: 10
with:
path: ~/.m2/repository
key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
restore-keys: |
${{ runner.os }}-maven-
# Build the PR branch's module to get a new JAR to compare
- name: Build SPI on PR branch
run: |
for mod in pinot-spi pinot-segment-spi; do
mvn clean package -T1C -pl $mod -am -DskipTests -Dproject.build.finalName="$mod"
done
- name: Checkout master into a subfolder
uses: actions/checkout@v4
with:
ref: ${{ github.event.pull_request.base.sha }}
path: master-head
# Build the same module on master to produce the baseline JAR
- name: Build SPI on master
working-directory: master-head
run: |
for mod in pinot-spi pinot-segment-spi; do
mvn clean package -T1C -pl $mod -am -DskipTests -Dproject.build.finalName="$mod"
done
# Download the japicmp standalone JAR which will be used to compare APIs
- name: Download japicmp CLI
run: |
JAPICMP_VER=0.23.1
curl -fSL \
-o japicmp.jar \
"https://repo1.maven.org/maven2/com/github/siom79/japicmp/japicmp/${JAPICMP_VER}/japicmp-${JAPICMP_VER}-jar-with-dependencies.jar"
# Locate the old JAR (from master) and the new JAR (PR) and run japicmp to compare
- name: Compare API
run: |
for mod in pinot-spi pinot-segment-spi; do
OLD=$(ls master-head/$mod/target/$mod-*.jar | grep -v '\-tests.jar' | head -1)
NEW=$(ls $mod/target/$mod-*.jar | grep -v '\-tests.jar' | head -1)
java -jar japicmp.jar \
--old "$OLD" \
--new "$NEW" \
--error-on-binary-incompatibility \
--only-incompatible \
--ignore-missing-classes
done
unit-test:
if: github.repository == 'apache/pinot'
runs-on: ubuntu-latest
strategy:
# Changed to false in order to improve coverage using unsafe buffers
fail-fast: false
matrix:
testset: [ 1, 2 ]
java: [ 11, 21 ]
distribution: [ "temurin" ]
name: Pinot Unit Test Set ${{ matrix.testset }} (${{matrix.distribution}}-${{matrix.java}})
steps:
- uses: actions/checkout@v4
- name: Set up JDK ${{ matrix.java }}-${{ matrix.distribution }}
uses: actions/setup-java@v4
with:
java-version: ${{ matrix.java }}
distribution: ${{ matrix.distribution }}
cache: 'maven'
# Step that does that actual cache save and restore
- uses: actions/cache@v4
env:
SEGMENT_DOWNLOAD_TIMEOUT_MINS: 10
with:
path: ~/.m2/repository
key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
restore-keys: |
${{ runner.os }}-maven-
- name: Build Project
env:
RUN_INTEGRATION_TESTS: false
RUN_TEST_SET: ${{ matrix.testset }}
DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }}
MAVEN_OPTS: >
-Xmx2G -DskipShade -DfailIfNoTests=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=25
-Dmaven.wagon.http.retryHandler.count=30 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false
-B -ntp
-XX:+IgnoreUnrecognizedVMOptions
--add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED
--add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED
--add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED
--add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED
--add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED
--add-opens=java.base/java.nio=ALL-UNNAMED
--add-opens=java.base/sun.nio.ch=ALL-UNNAMED
--add-opens=java.base/java.lang=ALL-UNNAMED
--add-opens=java.base/java.util=ALL-UNNAMED
--add-opens=java.base/java.lang.reflect=ALL-UNNAMED
run: .github/workflows/scripts/pr-tests/.pinot_tests_build.sh
- name: Unit Test
env:
RUN_INTEGRATION_TESTS: false
RUN_TEST_SET: ${{ matrix.testset }}
DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }}
MAVEN_OPTS: >
-Xmx2G -DskipShade -DfailIfNoTests=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=25
-Dmaven.wagon.http.retryHandler.count=30 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false
-B -ntp
-XX:+IgnoreUnrecognizedVMOptions
--add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED
--add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED
--add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED
--add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED
--add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED
--add-opens=java.base/java.nio=ALL-UNNAMED
--add-opens=java.base/sun.nio.ch=ALL-UNNAMED
--add-opens=java.base/java.lang=ALL-UNNAMED
--add-opens=java.base/java.util=ALL-UNNAMED
--add-opens=java.base/java.lang.reflect=ALL-UNNAMED
run: .github/workflows/scripts/pr-tests/.pinot_tests_unit.sh
- name: Upload coverage to Codecov
uses: codecov/codecov-action@v5
continue-on-error: true
timeout-minutes: 5
with:
flags: unittests,unittests${{ matrix.testset }},${{matrix.distribution}},java-${{matrix.java}}
name: codecov-unit-tests
token: ${{ secrets.CODECOV_TOKEN }}
fail_ci_if_error: false
verbose: true
- name: Upload test results to Codecov
if: ${{ !cancelled() }}
uses: codecov/test-results-action@v1
timeout-minutes: 5
with:
flags: unittests,unittests${{ matrix.testset }},${{matrix.distribution}},java-${{matrix.java}}
name: codecov-unit-tests
token: ${{ secrets.CODECOV_TOKEN }}
fail_ci_if_error: false
verbose: true
integration-test:
if: github.repository == 'apache/pinot'
runs-on: ubuntu-latest
strategy:
# Changed to false in order to improve coverage using unsafe buffers
fail-fast: false
matrix:
testset: [ 1, 2 ]
java: [ 11, 21 ]
distribution: [ "temurin" ]
name: Pinot Integration Test Set ${{ matrix.testset }} (${{matrix.distribution}}-${{matrix.java}})
steps:
- uses: actions/checkout@v4
- name: Set up JDK ${{ matrix.java }}-${{ matrix.distribution }}
uses: actions/setup-java@v4
with:
java-version: ${{ matrix.java }}
distribution: ${{ matrix.distribution }}
cache: 'maven'
# Step that does that actual cache save and restore
- uses: actions/cache@v4
env:
SEGMENT_DOWNLOAD_TIMEOUT_MINS: 10
with:
path: ~/.m2/repository
key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
restore-keys: |
${{ runner.os }}-maven-
- name: Build Project
env:
RUN_INTEGRATION_TESTS: true
DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }}
MAVEN_OPTS: >
-Xmx2G -DskipShade -DfailIfNoTests=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=25
-Dmaven.wagon.http.retryHandler.count=30 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false
-B -ntp
-XX:+IgnoreUnrecognizedVMOptions
--add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED
--add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED
--add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED
--add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED
--add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED
run: .github/workflows/scripts/pr-tests/.pinot_tests_build.sh
- name: Integration Test
env:
RUN_INTEGRATION_TESTS: true
RUN_TEST_SET: ${{ matrix.testset }}
DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }}
MAVEN_OPTS: >
-Xmx2G -DskipShade -DfailIfNoTests=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=25
-Dmaven.wagon.http.retryHandler.count=30 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false
-B -ntp
-XX:+IgnoreUnrecognizedVMOptions
--add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED
--add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED
--add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED
--add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED
--add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED
run: .github/workflows/scripts/pr-tests/.pinot_tests_integration.sh
- name: Upload coverage to Codecov
uses: codecov/codecov-action@v5
continue-on-error: true
timeout-minutes: 5
with:
flags: integration,integration${{ matrix.testset }},${{matrix.distribution}},java-${{matrix.java}}
name: codecov-integration-tests
token: ${{ secrets.CODECOV_TOKEN }}
fail_ci_if_error: false
verbose: true
- name: Upload test results to Codecov
if: ${{ !cancelled() }}
uses: codecov/test-results-action@v1
continue-on-error: true
timeout-minutes: 5
with:
flags: integration,integration${{ matrix.testset }},${{matrix.distribution}},java-${{matrix.java}}
name: codecov-integration-tests
token: ${{ secrets.CODECOV_TOKEN }}
fail_ci_if_error: false
verbose: true
- name: Custom Integration Test
if : ${{ matrix.testset == 1 }}
env:
RUN_INTEGRATION_TESTS: true
RUN_TEST_SET: ${{ matrix.testset }}
DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }}
MAVEN_OPTS: >
-Xmx2G -DskipShade -DfailIfNoTests=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=25
-Dmaven.wagon.http.retryHandler.count=30 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false
-B -ntp
-XX:+IgnoreUnrecognizedVMOptions
--add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED
--add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED
--add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED
--add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED
--add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED
run: .github/workflows/scripts/pr-tests/.pinot_tests_custom_integration.sh
- name: Upload coverage to Codecov
if : ${{ matrix.testset == 1 }}
uses: codecov/codecov-action@v5
continue-on-error: true
timeout-minutes: 5
with:
flags: integration,custom-integration${{ matrix.testset }},${{matrix.distribution}},java-${{matrix.java}}
name: codecov-custom-integration-tests
token: ${{ secrets.CODECOV_TOKEN }}
fail_ci_if_error: false
verbose: true
- name: Upload test results to Codecov
if: ${{ !cancelled() }}
uses: codecov/test-results-action@v1
timeout-minutes: 5
with:
flags: integration,custom-integration${{ matrix.testset }},${{matrix.distribution}},java-${{matrix.java}}
name: codecov-custom-integration-tests
token: ${{ secrets.CODECOV_TOKEN }}
fail_ci_if_error: false
verbose: true
compatibility-verifier:
if: github.repository == 'apache/pinot'
runs-on: ubuntu-latest
strategy:
# Changed to false in order to improve coverage using unsafe buffers
fail-fast: false
matrix:
test_suite: [ "compatibility-verifier/sample-test-suite" ]
old_commit: [
"release-1.2.0", "release-1.3.0", "master"
]
name: Pinot Compatibility Regression Testing against ${{ matrix.old_commit }} on ${{ matrix.test_suite }}
steps:
- uses: actions/checkout@v4
- name: Set up JDK 11
uses: actions/setup-java@v4
with:
java-version: 11
distribution: 'temurin'
cache: 'maven'
- name: Setup node
uses: actions/setup-node@v4
with:
node-version: v16.15.0
cache: 'npm'
cache-dependency-path: pinot-controller/src/main/resources/package-lock.json
- name: Install npm
run: |
npm install -g [email protected]
npm --version
# Step that does that actual cache save and restore
- uses: actions/cache@v4
env:
SEGMENT_DOWNLOAD_TIMEOUT_MINS: 10
with:
path: ~/.m2/repository
key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
restore-keys: |
${{ runner.os }}-maven-
- name: Pinot Compatibility Regression Testing
env:
OLD_COMMIT: ${{ matrix.old_commit }}
WORKING_DIR: /tmp/compatibility-verifier
TEST_SUITE: ${{ matrix.test_suite }}
MAVEN_OPTS: >
-Xmx2G -DskipShade -DfailIfNoTests=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=25
-Dmaven.wagon.http.retryHandler.count=30 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false
-B -ntp
-XX:+IgnoreUnrecognizedVMOptions
--add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED
--add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED
--add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED
--add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED
--add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED
run: .github/workflows/scripts/.pinot_compatibility_verifier.sh
multi-stage-compatibility-verifier:
if: github.repository == 'apache/pinot'
runs-on: ubuntu-latest
strategy:
# Changed to false in order to improve coverage using unsafe buffers
fail-fast: false
matrix:
test_suite: [ "compatibility-verifier/multi-stage-query-engine-test-suite" ]
old_commit: [
"release-1.2.0", "release-1.3.0", "master"
]
name: Pinot Multi-Stage Query Engine Compatibility Regression Testing against ${{ matrix.old_commit }} on ${{ matrix.test_suite }}
steps:
- uses: actions/checkout@v4
- name: Set up JDK 11
uses: actions/setup-java@v4
with:
java-version: 11
distribution: 'temurin'
cache: 'maven'
- name: Setup node
uses: actions/setup-node@v4
with:
node-version: v16.15.0
cache: 'npm'
cache-dependency-path: pinot-controller/src/main/resources/package-lock.json
- name: Install npm
run: |
npm install -g [email protected]
npm --version
# Step that does that actual cache save and restore
- uses: actions/cache@v4
env:
SEGMENT_DOWNLOAD_TIMEOUT_MINS: 10
with:
path: ~/.m2/repository
key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
restore-keys: |
${{ runner.os }}-maven-
- name: Pinot Multi-Stage Query Engine Compatibility Regression Testing
env:
OLD_COMMIT: ${{ matrix.old_commit }}
WORKING_DIR: /tmp/multi-stage-compatibility-verifier
TEST_SUITE: ${{ matrix.test_suite }}
MAVEN_OPTS: >
-Xmx2G -DskipShade -DfailIfNoTests=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=25
-Dmaven.wagon.http.retryHandler.count=30 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false
-B -ntp
-XX:+IgnoreUnrecognizedVMOptions
--add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED
--add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED
--add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED
--add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED
--add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED
run: .github/workflows/scripts/.pinot_compatibility_verifier.sh
quickstarts:
if: github.repository == 'apache/pinot'
runs-on: ubuntu-latest
strategy:
# Changed to false in order to improve coverage using unsafe buffers
fail-fast: false
matrix:
java: [ 11, 21 ]
distribution: [ "temurin" ]
name: Pinot Quickstart on JDK ${{ matrix.java }}-${{ matrix.distribution }}
steps:
- uses: actions/checkout@v4
- name: Set up JDK ${{ matrix.java }}
uses: actions/setup-java@v4
with:
java-version: ${{ matrix.java }}
distribution: ${{ matrix.distribution }}
cache: 'maven'
# Step that does that actual cache save and restore
- uses: actions/cache@v4
env:
SEGMENT_DOWNLOAD_TIMEOUT_MINS: 10
DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }}
with:
path: ~/.m2/repository
key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
restore-keys: |
${{ runner.os }}-maven-
- name: Quickstart on JDK ${{ matrix.java }}
run: .github/workflows/scripts/.pinot_quickstart.sh