XGrammar Benchmark #129

	name: XGrammar Benchmark
	on:
	workflow_dispatch:
	schedule:
	- cron: '0 0 * * *'

	jobs:
	run_benchmark:
	name: Run XGrammar Benchmark
	if: github.ref == 'refs/heads/main'&& github.repository_owner == 'mlc-ai'
	runs-on: [self-hosted, Linux, X64]
	steps:
	- name: Checkout
	uses: actions/checkout@v4
	with:
	submodules: recursive

	- name: Set up Python 3.11
	uses: actions/setup-python@v5
	with:
	python-version: 3.11

	- name: Build xgrammar from source
	run: \|
	python -m pip install --upgrade pip
	pip install .

	- name: Install dependencies
	run: \|
	pip install torch transformers datasets tqdm requests

	- name: Run benchmark
	id: benchmark
	env:
	HF_TOKEN: ${{ secrets.HF_TOKEN }}
	run: \|
	python examples/benchmark/cibench_grammar_compile_mask_gen.py --num_iters 3 --num_warmup 2 --datasets all \| tee benchmark_output.txt

	- name: Upload benchmark results
	uses: actions/upload-artifact@v4
	with:
	name: benchmark-results
	path: benchmark_output.txt

Provide feedback