Skip to content
This repository was archived by the owner on Jun 4, 2025. It is now read-only.

Commit a943157

Browse files
dsikkadsikka
authored
Merge branch 'main' into compressed-tensors-quantizer
2 parents 3cb4415 + 75b7485 commit a943157

File tree

308 files changed

+10736
-2077
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

308 files changed

+10736
-2077
lines changed

‎.circleci/config.yml

Lines changed: 47 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -47,13 +47,13 @@ jobs:
4747

4848
- run:
4949
name: "Retrieve Artifact Paths"
50-
env:
51-
CIRCLE_TOKEN: ${{ secrets.CI_ARTIFACT_TOKEN }}
50+
# [reference] https://circleci.com/docs/api/v2/index.html#operation/getJobArtifacts
51+
# `CIRCLE_TOKEN` is defined as an environment variables set within a context, see `https://circleci.com/docs/contexts/`
5252
command: |
5353
project_slug="gh/${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME}"
5454
job_number=${CIRCLE_BUILD_NUM}
5555
url="https://circleci.com/api/v2/project/${project_slug}/${job_number}/artifacts"
56-
curl -o test_preparation/artifacts.json ${url}
56+
curl -o test_preparation/artifacts.json ${url} --header "Circle-Token: $CIRCLE_TOKEN"
5757
- run:
5858
name: "Prepare pipeline parameters"
5959
command: |
@@ -82,22 +82,49 @@ jobs:
8282
parallelism: 1
8383
steps:
8484
- checkout
85-
- run: uv pip install -e .
86-
- run: |
87-
mkdir test_preparation
88-
echo -n "tests" > test_preparation/test_list.txt
89-
echo -n "all" > test_preparation/examples_test_list.txt
90-
echo -n "tests/repo_utils" > test_preparation/test_repo_utils.txt
85+
- run: uv pip install -U -e .
86+
- run: echo 'export "GIT_COMMIT_MESSAGE=$(git show -s --format=%s)"' >> "$BASH_ENV" && source "$BASH_ENV"
87+
- run: mkdir -p test_preparation
88+
- run: python utils/tests_fetcher.py --fetch_all | tee tests_fetched_summary.txt
89+
- run: python utils/tests_fetcher.py --filter_tests
90+
- run: export "GIT_COMMIT_MESSAGE=$(git show -s --format=%s)" && echo $GIT_COMMIT_MESSAGE && python .circleci/create_circleci_config.py --fetcher_folder test_preparation
9191
- run: |
92-
echo -n "tests" > test_list.txt
93-
python utils/tests_fetcher.py --filter_tests
94-
mv test_list.txt test_preparation/filtered_test_list.txt
95-
- run: python .circleci/create_circleci_config.py --fetcher_folder test_preparation
96-
- run: cp test_preparation/generated_config.yml test_preparation/generated_config.txt
92+
if [ ! -s test_preparation/generated_config.yml ]; then
93+
echo "No tests to run, exiting early!"
94+
circleci-agent step halt
95+
fi
96+
97+
- store_artifacts:
98+
path: test_preparation
99+
100+
- run:
101+
name: "Retrieve Artifact Paths"
102+
env:
103+
CIRCLE_TOKEN: ${{ secrets.CI_ARTIFACT_TOKEN }}
104+
command: |
105+
project_slug="gh/${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME}"
106+
job_number=${CIRCLE_BUILD_NUM}
107+
url="https://circleci.com/api/v2/project/${project_slug}/${job_number}/artifacts"
108+
curl -o test_preparation/artifacts.json ${url}
109+
- run:
110+
name: "Prepare pipeline parameters"
111+
command: |
112+
python utils/process_test_artifacts.py
113+
114+
# To avoid too long generated_config.yaml on the continuation orb, we pass the links to the artifacts as parameters.
115+
# Otherwise the list of tests was just too big. Explicit is good but for that it was a limitation.
116+
# We used:
117+
118+
# https://circleci.com/docs/api/v2/index.html#operation/getJobArtifacts : to get the job artifacts
119+
# We could not pass a nested dict, which is why we create the test_file_... parameters for every single job
120+
97121
- store_artifacts:
98-
path: test_preparation/generated_config.txt
122+
path: test_preparation/transformed_artifacts.json
123+
- store_artifacts:
124+
path: test_preparation/artifacts.json
99125
- continuation/continue:
100-
configuration_path: test_preparation/generated_config.yml
126+
parameters: test_preparation/transformed_artifacts.json
127+
configuration_path: test_preparation/generated_config.yml
101128

102129
check_code_quality:
103130
working_directory: ~/transformers
@@ -163,7 +190,10 @@ workflows:
163190
- check_circleci_user
164191
- check_code_quality
165192
- check_repository_consistency
166-
- fetch_tests
193+
- fetch_tests:
194+
# [reference] https://circleci.com/docs/contexts/
195+
context:
196+
- TRANSFORMERS_CONTEXT
167197

168198
nightly:
169199
when: <<pipeline.parameters.nightly>>

‎.circleci/create_circleci_config.py

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -312,6 +312,15 @@ def job_name(self):
312312
)
313313

314314

315+
non_model_job = CircleCIJob(
316+
"non_model",
317+
docker_image=[{"image": "huggingface/transformers-torch-light"}],
318+
marker="not generate",
319+
parallelism=6,
320+
pytest_num_workers=8,
321+
)
322+
323+
315324
# We also include a `dummy.py` file in the files to be doc-tested to prevent edge case failure. Otherwise, the pytest
316325
# hangs forever during test collection while showing `collecting 0 items / 21 errors`. (To see this, we have to remove
317326
# the bash output redirection.)
@@ -336,7 +345,7 @@ def job_name(self):
336345
pytest_num_workers=1,
337346
)
338347

339-
REGULAR_TESTS = [torch_and_tf_job, torch_and_flax_job, torch_job, tf_job, flax_job, hub_job, onnx_job, tokenization_job, processor_job, generate_job] # fmt: skip
348+
REGULAR_TESTS = [torch_and_tf_job, torch_and_flax_job, torch_job, tf_job, flax_job, hub_job, onnx_job, tokenization_job, processor_job, generate_job, non_model_job] # fmt: skip
340349
EXAMPLES_TESTS = [examples_torch_job, examples_tensorflow_job]
341350
PIPELINE_TESTS = [pipelines_torch_job, pipelines_tf_job]
342351
REPO_UTIL_TESTS = [repo_utils_job]

‎.github/workflows/build_documentation.yml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
name: Build documentation
22

33
on:
4+
workflow_dispatch:
45
push:
56
branches:
67
- main

‎.github/workflows/model_jobs.yml

Lines changed: 27 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,8 @@ jobs:
4141
fail-fast: false
4242
matrix:
4343
folders: ${{ fromJson(inputs.folder_slices)[inputs.slice_id] }}
44-
runs-on: ['${{ inputs.machine_type }}', nvidia-gpu, t4, '${{ inputs.runner }}']
44+
runs-on:
45+
group: '${{ inputs.machine_type }}'
4546
container:
4647
image: ${{ inputs.docker }}
4748
options: --gpus all --shm-size "16gb" --ipc host -v /mnt/cache/.cache/huggingface:/mnt/cache/
@@ -97,25 +98,42 @@ jobs:
9798
working-directory: /transformers
9899
run: pip freeze
99100

101+
- name: Set `machine_type` for report and artifact names
102+
working-directory: /transformers
103+
shell: bash
104+
run: |
105+
echo "${{ inputs.machine_type }}"
106+
107+
if [ "${{ inputs.machine_type }}" = "aws-g4dn-2xlarge-cache" ]; then
108+
machine_type=single-gpu
109+
elif [ "${{ inputs.machine_type }}" = "aws-g4dn-12xlarge-cache" ]; then
110+
machine_type=multi-gpu
111+
else
112+
machine_type=${{ inputs.machine_type }}
113+
fi
114+
115+
echo "$machine_type"
116+
echo "machine_type=$machine_type" >> $GITHUB_ENV
117+
100118
- name: Run all tests on GPU
101119
working-directory: /transformers
102-
run: python3 -m pytest -rsfE -v --make-reports=${{ inputs.machine_type }}_run_models_gpu_${{ matrix.folders }}_test_reports tests/${{ matrix.folders }}
120+
run: python3 -m pytest -rsfE -v --make-reports=${{ env.machine_type }}_run_models_gpu_${{ matrix.folders }}_test_reports tests/${{ matrix.folders }}
103121

104122
- name: Failure short reports
105123
if: ${{ failure() }}
106124
continue-on-error: true
107-
run: cat /transformers/reports/${{ inputs.machine_type }}_run_models_gpu_${{ matrix.folders }}_test_reports/failures_short.txt
125+
run: cat /transformers/reports/${{ env.machine_type }}_run_models_gpu_${{ matrix.folders }}_test_reports/failures_short.txt
108126

109127
- name: Run test
110128
shell: bash
111129
run: |
112-
mkdir -p /transformers/reports/${{ inputs.machine_type }}_run_models_gpu_${{ matrix.folders }}_test_reports
113-
echo "hello" > /transformers/reports/${{ inputs.machine_type }}_run_models_gpu_${{ matrix.folders }}_test_reports/hello.txt
114-
echo "${{ inputs.machine_type }}_run_models_gpu_${{ matrix.folders }}_test_reports"
130+
mkdir -p /transformers/reports/${{ env.machine_type }}_run_models_gpu_${{ matrix.folders }}_test_reports
131+
echo "hello" > /transformers/reports/${{ env.machine_type }}_run_models_gpu_${{ matrix.folders }}_test_reports/hello.txt
132+
echo "${{ env.machine_type }}_run_models_gpu_${{ matrix.folders }}_test_reports"
115133
116-
- name: "Test suite reports artifacts: ${{ inputs.machine_type }}_run_models_gpu_${{ env.matrix_folders }}_test_reports"
134+
- name: "Test suite reports artifacts: ${{ env.machine_type }}_run_models_gpu_${{ env.matrix_folders }}_test_reports"
117135
if: ${{ always() }}
118136
uses: actions/upload-artifact@v4
119137
with:
120-
name: ${{ inputs.machine_type }}_run_models_gpu_${{ env.matrix_folders }}_test_reports
121-
path: /transformers/reports/${{ inputs.machine_type }}_run_models_gpu_${{ matrix.folders }}_test_reports
138+
name: ${{ env.machine_type }}_run_models_gpu_${{ env.matrix_folders }}_test_reports
139+
path: /transformers/reports/${{ env.machine_type }}_run_models_gpu_${{ matrix.folders }}_test_reports

‎.github/workflows/self-scheduled-caller.yml

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,9 @@ name: Self-hosted runner (scheduled)
22

33

44
on:
5+
repository_dispatch:
6+
schedule:
7+
- cron: "17 2 * * *"
58
push:
69
branches:
710
- run_scheduled_ci*

0 commit comments

Comments
 (0)