Merge remote-tracking branch 'origin/develop' into DEV-1249

Author: makseq

.github/workflows/build_pypi.yml

@@ -146,11 +146,11 @@ jobs:
                 core.setOutput("pipy-artifact-url", artifact.url);
                 core.setOutput("pipy-artifact-digests-sha256", artifact.digests.sha256);
                 clearInterval(intervalId)
-              } else if (currentAttempt > MAX_ATTEMPTS) {
+              } else if (currentAttempt >= MAX_ATTEMPTS) {
                 clearInterval(intervalId)
                 throw Error('Max attempts exceeded')
               }
-            }, 6 * 1000)
+            }, 60 * 1000 )
 
       - name: Attach artifacts to release
         if: inputs.release-id

.github/workflows/cicd_pipeline.yml

@@ -148,6 +148,15 @@ jobs:
     with:
       ref: ${{ github.event.pull_request.head.ref || github.ref }}
 
+  migrations:
+    name: "Tests"
+    needs:
+      - changed_files
+    if: needs.changed_files.outputs.src == 'true'
+    uses: heartexlabs/label-studio/.github/workflows/test_migrations.yml@develop
+    with:
+      head_sha: ${{ github.event.pull_request.head.sha || github.event.after }}
+    secrets: inherit
 
   draft-release:
     name: "Draft Release"
@@ -194,11 +203,15 @@ jobs:
               process.exit()
             }
 
-            const {data: tags} = await github.rest.repos.listTags({
+            const tags = await github.paginate(
+              github.rest.repos.listTags,
+              {
                 owner,
                 repo,
                 per_page: 100
-            });
+              },
+              (response) => response.data
+            );
             console.log(`Tags:`)
             console.log(tags.map(e => e.name))
             const matchedTags = tags.filter(e => e.name.indexOf(version) !== -1)
@@ -231,10 +244,15 @@ jobs:
             console.log(`Previous version: ${previousTag}`)
 
             console.log('Find or Create a Draft release')
-            const {data: releases} = await github.rest.repos.listReleases({
+            const releases = await github.paginate(
+              github.rest.repos.listReleases,
+              {
                 owner,
                 repo,
-            });
+                per_page: 100
+              },
+              (response) => response.data
+            );
             let release = releases.find(e => target_commitish.endsWith(e.target_commitish) && e.draft)
             if (release) {
                 console.log(`Draft release already exist ${release.html_url}`)

.github/workflows/cut-off-release-branch.yml

@@ -60,9 +60,9 @@ jobs:
           release_branch="${{ env.RELEASE_BRANCH_PREFIX }}/${first}.${second}.${third}"
           next_develop_version="${first}.${second}.$(($third + 1))dev"
           
-          echo "release_branch=${release_branch}"
-          echo "release_version=${release_version}"
-          echo "next_develop_version=${next_develop_version}"
+          echo "release_branch=${release_branch}" >> $GITHUB_OUTPUT
+          echo "release_version=${release_version}" >> $GITHUB_OUTPUT
+          echo "next_develop_version=${next_develop_version}" >> $GITHUB_OUTPUT
 
       - name: Cut dependencies release branches
         uses: actions/github-script@v6
@@ -154,7 +154,7 @@ jobs:
           git add ${{ env.PYTHON_VERSION_FILE }}
           git commit --message "ci: cut release ${{ steps.calculate_branch_name_and_version.outputs.release_version }}"
 
-          echo "commit=$(git rev-parse HEAD)"
+          echo "commit=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT
           git push origin HEAD:refs/heads/${{ steps.calculate_branch_name_and_version.outputs.release_branch }}
 
       - uses: actions/cache@v3
@@ -174,15 +174,15 @@ jobs:
           git checkout '${{ github.event.repository.default_branch }}'
           
           branch='bump-version-${{ steps.calculate_branch_name_and_version.outputs.next_develop_version }}'
-          echo "branch=${branch}"
+          echo "branch=${branch}" >> $GITHUB_OUTPUT
           git checkout -b "${branch}"
           
           sed -i "s/^__version__[ ]*=.*/__version__ = '${{ steps.calculate_branch_name_and_version.outputs.next_develop_version }}'/g" ${{ env.PYTHON_VERSION_FILE }}
           
           git add ${{ env.PYTHON_VERSION_FILE }}
           git commit --message "chore: Bump version to ${{ steps.calculate_branch_name_and_version.outputs.next_develop_version }}"
 
-          echo "commit=$(git rev-parse HEAD)"
+          echo "commit=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT
           git push origin HEAD:refs/heads/${branch}
 
       - name: Create PR to 'develop'

.github/workflows/docker-release-promote.yml

@@ -40,13 +40,18 @@ jobs:
           github-token: ${{ secrets.GIT_PAT }}
           script: |
             const {repo, owner} = context.repo;
-            const {data: checks} = await github.rest.checks.listForRef({
-              owner,
-              repo,
-              ref: 'tags/${{ inputs.release_tag }}',
-              status: "completed",
-            })
-            const check = checks.check_runs.find(e => e.name === '${{ env.DOCKER_IMAGE_TAG_CHECK_NAME }}')
+            const check_runs = await github.paginate(
+              github.rest.checks.listForRef,
+              {
+                owner,
+                repo,
+                ref: 'tags/${{ inputs.release_tag }}',
+                status: "completed",
+                per_page: 100
+              },
+              (response) => response.data
+            );
+            const check = check_runs.find(e => e.name === '${{ env.DOCKER_IMAGE_TAG_CHECK_NAME }}')
             const details = JSON.parse(check.output.summary)
             console.log(details)
             core.setOutput("image_version", details.image_version);
@@ -83,11 +88,15 @@ jobs:
                 return +a[1] - b[1]
             }
             
-            const {data: tags} = await github.rest.repos.listTags({
-              owner,
-              repo,
-              per_page: 100
-            });
+            const tags = await github.paginate(
+              github.rest.repos.listTags,
+              {
+                owner,
+                repo,
+                per_page: 100
+              },
+              (response) => response.data
+            );
             const rawTags = tags.map(e => e.name)
             const filteredTags = rawTags.filter(e => e.match(regexp))
             const sortedTags = filteredTags
@@ -284,4 +293,4 @@ jobs:
               ${{ env.PREFLIGHT_REPO }} \
               check container "${image}" \
               --submit
-          done <<< '${{ steps.generate-tags.outputs.redhat-tags }},'
+          done <<< '${{ steps.generate-tags.outputs.redhat-tags }},'

.github/workflows/test_migrations.yml

@@ -0,0 +1,86 @@
+name: "Tests"
+
+on:
+  workflow_call:
+    inputs:
+      head_sha:
+        required: true
+        type: string
+
+env:
+  NODE: '14'
+  CACHE_NAME_PREFIX: v1
+
+jobs:
+  migrations:
+    name: "migrations"
+    runs-on: ubuntu-20.04
+    timeout-minutes: 20
+    env:
+      DJANGO_SETTINGS_MODULE: core.settings.label_studio
+      COVERAGE_PROCESS_START: 1
+      LOG_DIR: pytest_logs
+      collect_analytics: true
+      DEBUG_CONTEXTLOG: true
+      LABEL_STUDIO_TEST_ENVIRONMENT: false
+      SENTRY_ENVIRONMENT: tests-ubuntu-sqlite
+      SENTRY_RATE: 0
+      DJANGO_DB: sqlite
+      JSON_LOG: 0
+      # SENTRY_DSN:
+
+    steps:
+      - uses: hmarr/[email protected]
+
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ inputs.ref }}
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.9'
+
+      - name: Install OS dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install libsasl2-dev python3-dev libldap2-dev libssl-dev libxml2-dev libxslt-dev
+
+      - uses: actions/cache@v3
+        name: Configure pip cache
+        id: pip-cache
+        with:
+          # Cache the Python package environment, excluding pip and setuptools and label_studio* installed by setup-python
+          path: |
+            ~/.cache/pip
+            ${{ env.pythonLocation }}/bin/*
+            ${{ env.pythonLocation }}/include
+            ${{ env.pythonLocation }}/lib/python*/site-packages/*
+            !${{ env.pythonLocation }}/bin/pip*
+            !${{ env.pythonLocation }}/lib/python*/site-packages/pip*
+            !${{ env.pythonLocation }}/lib/python*/site-packages/setuptools*
+            !${{ env.pythonLocation }}/lib/python*/site-packages/label_studio*
+          key: ${{ runner.os }}-pip-${{ env.pythonLocation }}-${{ hashFiles('**/requirements.txt') }}-${{ hashFiles('**/requirements-test.txt') }}
+          restore-keys: |
+            ${{ env.CACHE_NAME_PREFIX }}-${{ runner.os }}-pip-django-${{ env.pythonLocation }}-
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip setuptools
+          pip install --upgrade cython
+          pip install -U pip==20.2
+          pip install -r deploy/requirements.txt -r deploy/requirements-test.txt
+          pip install -e .
+
+      - name: Test migrations
+        run: |
+          output=$(python label_studio/manage.py makemigrations)
+          if ! grep 'No changes detected' <<< "${output}"; then
+            error="${output}"
+            error="${error//'%'/'%25'}"
+            error="${error//$'\n'/'%0A'}"
+            error="${error//$'\r'/'%0D'}"
+            echo "::error::${error}"
+            exit 1
+          fi

.github/workflows/tests.yml

@@ -9,11 +9,12 @@ on:
 
 env:
   NODE: '14'
+  CACHE_NAME_PREFIX: v1
 
 jobs:
   run_pytest_sqlite:
     name: LS SQLite Ubuntu
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     strategy:
       fail-fast: false
       matrix:
@@ -58,7 +59,7 @@ jobs:
         name: Configure pip cache
         id: pip-cache
         with:
-          # Cache the Python package environment, excluding pip and setuptools installed by setup-python
+          # Cache the Python package environment, excluding pip and setuptools and label_studio* installed by setup-python
           path: |
             ~/.cache/pip
             ${{ env.pythonLocation }}/bin/*
@@ -67,9 +68,10 @@ jobs:
             !${{ env.pythonLocation }}/bin/pip*
             !${{ env.pythonLocation }}/lib/python*/site-packages/pip*
             !${{ env.pythonLocation }}/lib/python*/site-packages/setuptools*
+            !${{ env.pythonLocation }}/lib/python*/site-packages/label_studio*
           key: ${{ runner.os }}-pip-${{ env.pythonLocation }}-${{ hashFiles('**/requirements.txt') }}-${{ hashFiles('**/requirements-test.txt') }}
           restore-keys: |
-            ${{ runner.os }}-pip-${{ env.pythonLocation }}-
+            ${{ env.CACHE_NAME_PREFIX }}-${{ runner.os }}-pip-django-${{ env.pythonLocation }}-
 
       - name: Install dependencies
         run: |
@@ -121,7 +123,7 @@ jobs:
 
   run_pytest_postgresql:
     name: LS PostgreSQL Ubuntu
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     strategy:
       fail-fast: false
       matrix:
@@ -198,7 +200,7 @@ jobs:
         name: Configure pip cache
         id: pip-cache
         with:
-          # Cache the Python package environment, excluding pip and setuptools installed by setup-python
+          # Cache the Python package environment, excluding pip and setuptools and label_studio* installed by setup-python
           path: |
             ~/.cache/pip
             ${{ env.pythonLocation }}/bin/*
@@ -207,9 +209,10 @@ jobs:
             !${{ env.pythonLocation }}/bin/pip*
             !${{ env.pythonLocation }}/lib/python*/site-packages/pip*
             !${{ env.pythonLocation }}/lib/python*/site-packages/setuptools*
+            !${{ env.pythonLocation }}/lib/python*/site-packages/label_studio*
           key: ${{ runner.os }}-pip-${{ env.pythonLocation }}-${{ hashFiles('**/requirements.txt') }}-${{ hashFiles('**/requirements-test.txt') }}
           restore-keys: |
-            ${{ runner.os }}-pip-${{ env.pythonLocation }}-
+            ${{ env.CACHE_NAME_PREFIX }}-${{ runner.os }}-pip-django-${{ env.pythonLocation }}-
 
       - name: Install pip dependencies
         run: |

.github/workflows/upstream_repo_sync.yml

@@ -12,6 +12,7 @@ env:
   NODE: 14
   CACHE_NAME_PREFIX: v1
   STATIC_DIST: 'label_studio/frontend/dist'
+  RELEASE_BRANCH_PREFIX: "ls-release/"
 
 jobs:
   open:
@@ -140,7 +141,7 @@ jobs:
           
           branch_name='${{ steps.get-pr.outputs.branch-name }}'
           
-          if grep -q '^ls-release/.*$' <<< '${{ github.event.client_payload.base_branch_name }}'; then
+          if grep -q '^${{ env.RELEASE_BRANCH_PREFIX }}.*$' <<< '${{ github.event.client_payload.base_branch_name }}'; then
             git checkout '${{ github.event.client_payload.base_branch_name }}'
           else
             git checkout '${{ github.event.repository.default_branch }}'
@@ -175,6 +176,7 @@ jobs:
         uses: actions/github-script@v6
         env:
           TITLE: ${{ github.event.client_payload.title }}
+          RELEASE_BRANCH_PREFIX: "${{ env.RELEASE_BRANCH_PREFIX }}"
         with:
           github-token: ${{ secrets.GIT_PAT }}
           script: |
@@ -200,7 +202,7 @@ jobs:
               }
             } else {
               let base = '${{ github.event.repository.default_branch }}'
-              if ( '${{ github.event.client_payload.base_branch_name }}'.startsWith('lse-release/') ) {
+              if ( '${{ github.event.client_payload.base_branch_name }}'.startsWith(process.env.RELEASE_BRANCH_PREFIX) ) {
                 base = '${{ github.event.client_payload.base_branch_name }}'
               }
               const createPullResponse = await github.rest.pulls.create({
@@ -238,12 +240,16 @@ jobs:
             let openPRs = []  
             for (let submodule of submodules) {
               core.info(`Checking ${ submodule.owner }/${ submodule.repo }`)
-              const { data: listAllOpenPulls } = await github.rest.pulls.list({
-                owner: submodule.owner,
-                repo: submodule.repo,
-                status: 'open',
-                per_page: 100
-              });
+              const listAllOpenPulls = await github.paginate(
+                github.rest.pulls.list,
+                {
+                  owner: submodule.owner,
+                  repo: submodule.repo,
+                  status: 'open',
+                  per_page: 100
+                },
+                (response) => response.data
+              );
               
               const listOpenPulls = listAllOpenPulls.filter(e => e.head.ref.toLowerCase() === branchNameLowerCase)