Add support for running and comparing benchmarks in CI.

Signed-off-by: Harkishen-Singh <[email protected]>

Author: Harkishen-Singh

.github/workflows/benchmark.yaml

@@ -0,0 +1,100 @@
+name: Benchmarks
+
+on:
+  pull_request:
+    branches: [ '**' ]
+
+env:
+  PROMSCALE_LOGGING: false
+  PROMSCALE_BENCHMARK: true
+  golang-version: 1.18.4
+
+jobs:
+  benchmark_current:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout current branch
+        uses: actions/checkout@v3
+
+      - name: Set up Go ${{ env.golang-version }}
+        uses: actions/[email protected]
+        with:
+          go-version: ${{ env.golang-version }}
+        id: go
+
+      - name: Benchmark
+        run: go test -bench=. -run=^$ -benchmem -cpu=1 ./... | tee current.txt
+
+      - name: Upload result
+        uses: actions/upload-artifact@v3
+        with:
+          name: current
+          path: current.txt
+
+  benchmark_master:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout master branch
+        uses: actions/checkout@v3
+        with:
+          ref: 'master'
+
+      - name: Set up Go ${{ env.golang-version }}
+        uses: actions/[email protected]
+        with:
+          go-version: ${{ env.golang-version }}
+        id: go
+
+      - name: Benchmark
+        run: go test -bench=. -run=^$ -benchmem -cpu=1 ./... | tee master.txt
+
+      - name: Upload result
+        uses: actions/upload-artifact@v3
+        with:
+          name: master
+          path: master.txt
+
+  compare_and_publish_results:
+    runs-on: ubuntu-latest
+    needs: [benchmark_current, benchmark_master]
+    steps:
+      - name: Set up Go ${{ env.golang-version }}
+        uses: actions/[email protected]
+        with:
+          go-version: ${{ env.golang-version }}
+        id: go
+
+      - name: Download result for benchmark_current
+        uses: actions/download-artifact@v3
+        with:
+          name: current
+
+      - name: Download result for benchmark_master
+        uses: actions/download-artifact@v3
+        with:
+          name: master
+
+      - name: Install benchstat
+        run: go install -a golang.org/x/perf/cmd/benchstat@latest
+
+      - name: Compare
+        id: compare
+        run: |
+          OUTPUT=$(benchstat -delta-test=none master.txt current.txt)
+          echo "${OUTPUT}"
+          OUTPUT="${OUTPUT//$'\n'/'%0A'}"
+          OUTPUT="${OUTPUT//$'\r'/'%0D'}"
+          echo "::set-output name=result::$OUTPUT"
+
+      - name: Publish results as comment
+        uses: marocchino/sticky-pull-request-comment@v2
+        with:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          header: compare
+          message: |
+            @${{ github.event.pull_request.user.login }}
+            ## Benchmark Result for **${{ github.sha }}**
+            ### Master vs PR
+            ```
+            ${{ steps.compare.outputs.result }}
+            ```

pkg/log/log.go

@@ -9,6 +9,7 @@ import (
 	"flag"
 	"fmt"
 	"os"
+	"strconv"
 	"sync"
 	"time"
 
@@ -43,10 +44,24 @@ func ParseFlags(fs *flag.FlagSet, cfg *Config) *Config {
 	return cfg
 }
 
+func shouldLog() bool {
+	value := os.Getenv("PROMSCALE_LOGGING")
+	enabled, err := strconv.ParseBool(value)
+	if err != nil {
+		// If we cannot parse, then assume to continue logging.
+		return true
+	}
+	return enabled
+}
+
 // InitDefault is used to start the logger with sane defaults before we can configure it.
 // It's useful in instances where we want to log stuff before the configuration
 // has been successfully parsed. Calling Init function later on overrides this.
 func InitDefault() {
+	if !shouldLog() {
+		logger = log.NewNopLogger()
+		return
+	}
 	l := level.NewFilter(
 		log.NewLogfmtLogger(log.NewSyncWriter(os.Stderr)),
 		level.AllowInfo(),
@@ -57,6 +72,10 @@ func InitDefault() {
 // Init starts logging given the configuration. By default, it uses logfmt format
 // and minimum logging level.
 func Init(cfg Config) error {
+	if !shouldLog() {
+		logger = log.NewNopLogger()
+		return nil
+	}
 	var l log.Logger
 	switch cfg.Format {
 	case "logfmt", "":

pkg/promql/bench_test.go

@@ -16,6 +16,7 @@ package promql
 import (
 	"context"
 	"fmt"
+	"os"
 	"strconv"
 	"strings"
 	"testing"
@@ -26,7 +27,15 @@ import (
 	"github.com/prometheus/prometheus/storage"
 )
 
+func ignore() bool {
+	value := os.Getenv("PROMSCALE_BENCHMARK")
+	return value != ""
+}
+
 func BenchmarkRangeQuery(b *testing.B) {
+	if ignore() {
+		return
+	}
 	teststorage := NewTestStorage(b)
 	defer teststorage.Close()
 	opts := EngineOpts{
@@ -233,6 +242,9 @@ func BenchmarkRangeQuery(b *testing.B) {
 }
 
 func BenchmarkParser(b *testing.B) {
+	if ignore() {
+		return
+	}
 	cases := []string{
 		"a",
 		"metric",