From d2e1cef55911c468d5e6c9de4248f8059a135dea Mon Sep 17 00:00:00 2001 From: Feanil Patel Date: Sat, 4 Apr 2026 00:42:59 -0400 Subject: [PATCH 1/4] build: collect per-test timing data. Run pytest with extra reporting enabled to generate files with per-test durations. The file is uploaded as a CI artifact so timing data can be downloaded and used to drive optimal shard rebalancing. Co-Authored-By: Claude Sonnet 4.6 --- .github/workflows/unit-tests.yml | 11 ++++++++++- requirements/edx/development.txt | 3 +++ requirements/edx/testing.in | 1 + requirements/edx/testing.txt | 3 +++ 4 files changed, 17 insertions(+), 1 deletion(-) diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 7b7b14a43e88..c2c0b6a89962 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -119,7 +119,16 @@ jobs: - name: run tests shell: bash run: | - python -Wd -m pytest -p no:randomly --ds=${{ env.settings_path }} ${{ env.unit_test_paths }} --cov=. + python -Wd -m pytest -p no:randomly --ds=${{ env.settings_path }} ${{ env.unit_test_paths }} --cov=. \ + --report-log=reports/pytest-report-${{ matrix.shard_name }}.jsonl + + - name: Upload pytest timing report + if: always() + uses: actions/upload-artifact@v7 + with: + name: pytest-report-${{ matrix.shard_name }}-${{ matrix.python-version }}-${{ matrix.django-version }}-${{ matrix.mongo-version }}-${{ matrix.os-version }} + path: reports/pytest-report-${{ matrix.shard_name }}.jsonl + overwrite: true - name: rename warnings json file if: success() diff --git a/requirements/edx/development.txt b/requirements/edx/development.txt index 905a754e88a9..a0c173f2a5a5 100644 --- a/requirements/edx/development.txt +++ b/requirements/edx/development.txt @@ -1724,6 +1724,7 @@ pytest==8.2.0 # pytest-json-report # pytest-metadata # pytest-randomly + # pytest-reportlog # pytest-xdist pytest-attrib==0.1.3 # via -r requirements/edx/testing.txt @@ -1739,6 +1740,8 @@ pytest-metadata==3.1.1 # pytest-json-report pytest-randomly==4.0.1 # via -r requirements/edx/testing.txt +pytest-reportlog==1.0.0 + # via -r requirements/edx/testing.txt pytest-xdist[psutil]==3.8.0 # via -r requirements/edx/testing.txt python-dateutil==2.9.0.post0 diff --git a/requirements/edx/testing.in b/requirements/edx/testing.in index 9784cb6cc946..b84fd39e3cab 100644 --- a/requirements/edx/testing.in +++ b/requirements/edx/testing.in @@ -38,6 +38,7 @@ pytest-django # Django support for pytest pytest-json-report # Output json formatted warnings after running pytest pytest-metadata # To prevent 'make upgrade' failure, dependency of pytest-json-report pytest-randomly # pytest plugin to randomly order tests +pytest-reportlog # Per-test timing data including setup/teardown (used for shard rebalancing) pytest-xdist[psutil] # Parallel execution of tests on multiple CPU cores or hosts singledispatch # Backport of functools.singledispatch from Python 3.4+, used in tests of XBlock rendering testfixtures # Provides a LogCapture utility used by several tests diff --git a/requirements/edx/testing.txt b/requirements/edx/testing.txt index 88e137db23cb..0678bd84800f 100644 --- a/requirements/edx/testing.txt +++ b/requirements/edx/testing.txt @@ -1310,6 +1310,7 @@ pytest==8.2.0 # pytest-json-report # pytest-metadata # pytest-randomly + # pytest-reportlog # pytest-xdist pytest-attrib==0.1.3 # via -r requirements/edx/testing.in @@ -1325,6 +1326,8 @@ pytest-metadata==3.1.1 # pytest-json-report pytest-randomly==4.0.1 # via -r requirements/edx/testing.in +pytest-reportlog==1.0.0 + # via -r requirements/edx/testing.in pytest-xdist[psutil]==3.8.0 # via -r requirements/edx/testing.in python-dateutil==2.9.0.post0 From 00c5c5a325dc3f7b7a2b80aab5fa70de49570730 Mon Sep 17 00:00:00 2001 From: Feanil Patel Date: Sat, 4 Apr 2026 23:36:14 -0400 Subject: [PATCH 2/4] build: rebalance unit test shards to reduce critical path Redistribute test paths across 9 shards (down from 16) using a greedy bin-packing optimiser driven by real per-test timing data from pytest-reportlog. Predicted critical path: ~18.7m (down from ~29m). Key changes: - Rename shard groups to reflect semantic meaning: lms-*, shared-with-lms-*, shared-with-cms-*, cms-* (openedx/common/xmodule paths explicitly separated from lms-only and cms-only paths) - Split lms/djangoapps/discussion/ into its 4 subdirectories so the heavy rest_api/ shard (15.7m) can be distributed across bins independently - Remove outdated comment referencing unit-tests-gh-hosted.yml Co-Authored-By: Claude Sonnet 4.6 --- .github/workflows/unit-test-shards.json | 194 ++++++++++-------------- .github/workflows/unit-tests.yml | 19 +-- 2 files changed, 88 insertions(+), 125 deletions(-) diff --git a/.github/workflows/unit-test-shards.json b/.github/workflows/unit-test-shards.json index 2c1ffd4744b7..02a83a4ceb3a 100644 --- a/.github/workflows/unit-test-shards.json +++ b/.github/workflows/unit-test-shards.json @@ -2,168 +2,169 @@ "lms-1": { "settings": "lms.envs.test", "paths": [ - "lms/djangoapps/branding/", - "lms/djangoapps/bulk_email/", - "lms/djangoapps/bulk_enroll/", - "lms/djangoapps/bulk_user_retirement/", - "lms/djangoapps/ccx/", - "lms/djangoapps/certificates/", - "lms/djangoapps/commerce/" + "lms/djangoapps/discussion/rest_api/" ] }, "lms-2": { "settings": "lms.envs.test", "paths": [ + "lms/djangoapps/ccx/", + "lms/djangoapps/commerce/", "lms/djangoapps/course_api/", - "lms/djangoapps/course_blocks/", - "lms/djangoapps/course_goals/", - "lms/djangoapps/course_home_api/", "lms/djangoapps/course_wiki/", - "lms/djangoapps/coursewarehistoryextended/", - "lms/djangoapps/debug/" + "lms/djangoapps/discussion/notification_prefs/", + "lms/djangoapps/instructor_task/", + "lms/djangoapps/ora_staff_grader/", + "lms/djangoapps/survey/", + "lms/djangoapps/teams/", + "lms/djangoapps/verify_student/" ] }, "lms-3": { "settings": "lms.envs.test", "paths": [ - "lms/djangoapps/courseware/" + "lms/djangoapps/branding/", + "lms/djangoapps/bulk_enroll/", + "lms/djangoapps/courseware/", + "lms/djangoapps/instructor_analytics/", + "lms/djangoapps/learner_dashboard/", + "lms/djangoapps/lti_provider/", + "lms/djangoapps/program_enrollments/" ] }, "lms-4": { "settings": "lms.envs.test", "paths": [ - "lms/djangoapps/discussion/", - "lms/djangoapps/edxnotes/", - "lms/djangoapps/experiments/" - ] - }, - "lms-5": { - "settings": "lms.envs.test", - "paths": [ - "lms/djangoapps/gating/", + "lms/djangoapps/bulk_user_retirement/", + "lms/djangoapps/course_blocks/", + "lms/djangoapps/course_goals/", + "lms/djangoapps/course_home_api/", + "lms/djangoapps/coursewarehistoryextended/", + "lms/djangoapps/discussion/tests/", + "lms/djangoapps/experiments/", "lms/djangoapps/grades/", "lms/djangoapps/instructor/", - "lms/djangoapps/instructor_analytics/" + "lms/djangoapps/mfe_config_api/", + "lms/djangoapps/staticbook/", + "lms/lib/" ] }, - "lms-6": { + "lms-5": { "settings": "lms.envs.test", "paths": [ - "lms/djangoapps/instructor_task/", - "lms/djangoapps/learner_dashboard/", + "lms/djangoapps/bulk_email/", + "lms/djangoapps/certificates/", + "lms/djangoapps/debug/", + "lms/djangoapps/discussion/django_comment_client/", + "lms/djangoapps/edxnotes/", + "lms/djangoapps/gating/", "lms/djangoapps/learner_home/", "lms/djangoapps/lms_initialization/", "lms/djangoapps/lms_xblock/", - "lms/djangoapps/lti_provider/", "lms/djangoapps/mailing/", "lms/djangoapps/mobile_api/", "lms/djangoapps/monitoring/", - "lms/djangoapps/ora_staff_grader/", - "lms/djangoapps/program_enrollments/", "lms/djangoapps/rss_proxy/", "lms/djangoapps/static_template_view/", - "lms/djangoapps/staticbook/", "lms/djangoapps/support/", - "lms/djangoapps/survey/", - "lms/djangoapps/teams/", "lms/djangoapps/tests/", "lms/djangoapps/user_tours/", - "lms/djangoapps/verify_student/", - "lms/djangoapps/mfe_config_api/", "lms/envs/", - "lms/lib/", "lms/tests.py" ] }, - "openedx-1-with-lms": { + "shared-with-lms-1": { "settings": "lms.envs.test", "paths": [ - "openedx/core/djangoapps/ace_common/", - "openedx/core/djangoapps/cors_csrf/", + "common/djangoapps/", "openedx/core/djangoapps/agreements/", "openedx/core/djangoapps/api_admin/", + "openedx/core/djangoapps/authz/", + "openedx/core/djangoapps/cache_toolbox/", + "openedx/core/djangoapps/ccxcon/", + "openedx/core/djangoapps/content/", + "openedx/core/djangoapps/content_libraries/", + "openedx/core/djangoapps/course_apps/", + "openedx/core/djangoapps/credentials/", + "openedx/core/djangoapps/credit/", + "openedx/core/djangoapps/dark_lang/", + "openedx/core/djangoapps/django_comment_common/", + "openedx/core/djangoapps/embargo/", + "openedx/core/djangoapps/header_control/", + "openedx/core/djangoapps/heartbeat/", + "openedx/core/djangoapps/models/", + "openedx/core/djangoapps/notifications/", + "openedx/core/djangoapps/oauth_dispatch/", + "openedx/core/djangoapps/safe_sessions/", + "openedx/core/djangoapps/schedules/", + "openedx/core/djangoapps/user_api/", + "openedx/core/djangoapps/util/", + "openedx/core/djangoapps/video_pipeline/", + "openedx/core/djangoapps/waffle_utils/", + "openedx/core/djangoapps/xblock/", + "openedx/core/tests/", + "openedx/features/", + "openedx/tests/" + ] + }, + "shared-with-lms-2": { + "settings": "lms.envs.test", + "paths": [ + "openedx/core/djangoapps/ace_common/", "openedx/core/djangoapps/auth_exchange/", "openedx/core/djangoapps/bookmarks/", - "openedx/core/djangoapps/cache_toolbox/", "openedx/core/djangoapps/catalog/", - "openedx/core/djangoapps/ccxcon/", "openedx/core/djangoapps/commerce/", "openedx/core/djangoapps/common_initialization/", "openedx/core/djangoapps/common_views/", "openedx/core/djangoapps/config_model_utils/", - "openedx/core/djangoapps/content/", - "openedx/core/djangoapps/content_libraries/", "openedx/core/djangoapps/contentserver/", "openedx/core/djangoapps/cookie_metadata/", - "openedx/core/djangoapps/course_apps/", + "openedx/core/djangoapps/cors_csrf/", "openedx/core/djangoapps/course_date_signals/", "openedx/core/djangoapps/course_groups/", + "openedx/core/djangoapps/course_live/", "openedx/core/djangoapps/courseware_api/", "openedx/core/djangoapps/crawlers/", - "openedx/core/djangoapps/credentials/", - "openedx/core/djangoapps/credit/", - "openedx/core/djangoapps/course_live/", - "openedx/core/djangoapps/dark_lang/", "openedx/core/djangoapps/debug/", "openedx/core/djangoapps/discussions/", - "openedx/core/djangoapps/django_comment_common/", - "openedx/core/djangoapps/embargo/", "openedx/core/djangoapps/enrollments/", - "openedx/core/djangoapps/external_user_ids/" - ] - }, - "openedx-2-with-lms": { - "settings": "lms.envs.test", - "paths": [ + "openedx/core/djangoapps/external_user_ids/", "openedx/core/djangoapps/geoinfo/", - "openedx/core/djangoapps/header_control/", - "openedx/core/djangoapps/heartbeat/", "openedx/core/djangoapps/lang_pref/", - "openedx/core/djangoapps/models/", "openedx/core/djangoapps/monkey_patch/", - "openedx/core/djangoapps/notifications/", - "openedx/core/djangoapps/oauth_dispatch/", "openedx/core/djangoapps/olx_rest_api/", "openedx/core/djangoapps/password_policy/", "openedx/core/djangoapps/plugin_api/", "openedx/core/djangoapps/plugins/", "openedx/core/djangoapps/profile_images/", "openedx/core/djangoapps/programs/", - "openedx/core/djangoapps/safe_sessions/", - "openedx/core/djangoapps/schedules/", "openedx/core/djangoapps/service_status/", "openedx/core/djangoapps/session_inactivity_timeout/", "openedx/core/djangoapps/signals/", "openedx/core/djangoapps/site_configuration/", "openedx/core/djangoapps/system_wide_roles/", "openedx/core/djangoapps/theming/", - "openedx/core/djangoapps/user_api/", "openedx/core/djangoapps/user_authn/", - "openedx/core/djangoapps/util/", "openedx/core/djangoapps/verified_track_content/", "openedx/core/djangoapps/video_config/", - "openedx/core/djangoapps/video_pipeline/", - "openedx/core/djangoapps/waffle_utils/", - "openedx/core/djangoapps/xblock/", "openedx/core/djangoapps/xmodule_django/", "openedx/core/djangoapps/zendesk_proxy/", - "openedx/core/djangoapps/authz/", "openedx/core/djangolib/", "openedx/core/lib/", - "openedx/core/tests/", - "openedx/features/", "openedx/testing/", - "openedx/tests/" + "xmodule/" ] }, - "openedx-1-with-cms": { + "shared-with-cms-1": { "settings": "cms.envs.test", "paths": [ + "common/djangoapps/", "openedx/core/djangoapps/ace_common/", - "openedx/core/djangoapps/cors_csrf/", "openedx/core/djangoapps/agreements/", "openedx/core/djangoapps/api_admin/", "openedx/core/djangoapps/auth_exchange/", + "openedx/core/djangoapps/authz/", "openedx/core/djangoapps/bookmarks/", "openedx/core/djangoapps/cache_toolbox/", "openedx/core/djangoapps/catalog/", @@ -175,8 +176,10 @@ "openedx/core/djangoapps/content/", "openedx/core/djangoapps/content_libraries/", "openedx/core/djangoapps/content_staging/", + "openedx/core/djangoapps/content_tagging/", "openedx/core/djangoapps/contentserver/", "openedx/core/djangoapps/cookie_metadata/", + "openedx/core/djangoapps/cors_csrf/", "openedx/core/djangoapps/course_apps/", "openedx/core/djangoapps/course_date_signals/", "openedx/core/djangoapps/course_groups/", @@ -190,13 +193,7 @@ "openedx/core/djangoapps/django_comment_common/", "openedx/core/djangoapps/embargo/", "openedx/core/djangoapps/enrollments/", - "openedx/core/djangoapps/external_user_ids/" - ] - }, - "openedx-2-with-cms": { - "settings": "cms.envs.test", - "paths": [ - "openedx/core/djangoapps/content_tagging/", + "openedx/core/djangoapps/external_user_ids/", "openedx/core/djangoapps/geoinfo/", "openedx/core/djangoapps/header_control/", "openedx/core/djangoapps/heartbeat/", @@ -228,9 +225,9 @@ "openedx/core/djangoapps/xblock/", "openedx/core/djangoapps/xmodule_django/", "openedx/core/djangoapps/zendesk_proxy/", - "openedx/core/djangoapps/authz/", "openedx/core/lib/", - "openedx/tests/" + "openedx/tests/", + "xmodule/" ] }, "cms-1": { @@ -238,44 +235,15 @@ "paths": [ "cms/djangoapps/api/", "cms/djangoapps/cms_user_tasks/", + "cms/djangoapps/contentstore/", "cms/djangoapps/course_creators/", "cms/djangoapps/export_course_metadata/", - "cms/djangoapps/modulestore_migrator/", "cms/djangoapps/models/", + "cms/djangoapps/modulestore_migrator/", "cms/djangoapps/pipeline_js/", "cms/djangoapps/xblock_config/", "cms/envs/", "cms/lib/" ] - }, - "cms-2": { - "settings": "cms.envs.test", - "paths": [ - "cms/djangoapps/contentstore/" - ] - }, - "common-with-lms": { - "settings": "lms.envs.test", - "paths": [ - "common/djangoapps/" - ] - }, - "common-with-cms": { - "settings": "cms.envs.test", - "paths": [ - "common/djangoapps/" - ] - }, - "xmodule-with-lms": { - "settings": "lms.envs.test", - "paths": [ - "xmodule/" - ] - }, - "xmodule-with-cms": { - "settings": "cms.envs.test", - "paths": [ - "xmodule/" - ] } } diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index c2c0b6a89962..9eacc25b5f4d 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -25,25 +25,20 @@ jobs: - "3.12" django-version: - "pinned" - # When updating the shards, remember to make the same changes in - # .github/workflows/unit-tests-gh-hosted.yml shard_name: - "lms-1" - "lms-2" - "lms-3" - "lms-4" - "lms-5" - - "lms-6" - - "openedx-1-with-lms" - - "openedx-2-with-lms" - - "openedx-1-with-cms" - - "openedx-2-with-cms" + - "shared-with-lms-1" + - "shared-with-lms-2" + # Note: The shared-with-cms-1 shard is currently a subset of both + # shared-with-lms-1 and shared-with-lms-2. Some shared tests are + # not run -with-cms at all. + # https://github.com/openedx/openedx-platform/issues/38355 + - "shared-with-cms-1" - "cms-1" - - "cms-2" - - "common-with-lms" - - "common-with-cms" - - "xmodule-with-lms" - - "xmodule-with-cms" mongo-version: - "7.0" os-version: From e9f093abc2bac48165de21546817a1edee76ad38 Mon Sep 17 00:00:00 2001 From: Feanil Patel Date: Fri, 10 Apr 2026 09:56:43 -0400 Subject: [PATCH 3/4] =?UTF-8?q?build:=20split=20cms-1=20shard=20=E2=80=94?= =?UTF-8?q?=20move=20contentstore/=20to=20new=20cms-2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit contentstore/ is large enough that the cms-1 runner was being killed mid-run in CI (OOM or runner-level timeout). Splitting it into its own shard keeps each job under the ~20-25 min target. No changes needed to gha_unit_tests_collector.py — it already classifies any shard whose first path starts with "cms/" as a CMS shard. Co-Authored-By: Claude Sonnet 4.6 --- .github/workflows/unit-test-shards.json | 7 ++++++- .github/workflows/unit-tests.yml | 1 + 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/unit-test-shards.json b/.github/workflows/unit-test-shards.json index 02a83a4ceb3a..0f26c1be11b0 100644 --- a/.github/workflows/unit-test-shards.json +++ b/.github/workflows/unit-test-shards.json @@ -235,7 +235,6 @@ "paths": [ "cms/djangoapps/api/", "cms/djangoapps/cms_user_tasks/", - "cms/djangoapps/contentstore/", "cms/djangoapps/course_creators/", "cms/djangoapps/export_course_metadata/", "cms/djangoapps/models/", @@ -245,5 +244,11 @@ "cms/envs/", "cms/lib/" ] + }, + "cms-2": { + "settings": "cms.envs.test", + "paths": [ + "cms/djangoapps/contentstore/" + ] } } diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 9eacc25b5f4d..d2ea23a9e889 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -39,6 +39,7 @@ jobs: # https://github.com/openedx/openedx-platform/issues/38355 - "shared-with-cms-1" - "cms-1" + - "cms-2" mongo-version: - "7.0" os-version: From 4f1e0c845473b2e027e35c22615d060630158442 Mon Sep 17 00:00:00 2001 From: Feanil Patel Date: Tue, 14 Apr 2026 10:55:51 -0400 Subject: [PATCH 4/4] build: only collect per-test timing data on master pushes The --report-log flag adds overhead (writing a JSONL file for every test) that's only useful for rebalancing work. Skip it entirely on PR runs by conditionally setting the flag via an env var; also gate the upload step on master so artifacts aren't created unnecessarily. Co-Authored-By: Claude Sonnet 4.6 --- .github/workflows/unit-tests.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index d2ea23a9e889..2d94a818ef6f 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -111,15 +111,20 @@ jobs: shell: bash run: | echo "unit_test_paths=$(python scripts/unit_test_shards_parser.py --shard-name=${{ matrix.shard_name }} )" >> $GITHUB_ENV + if [[ "${{ github.ref }}" == "refs/heads/master" ]]; then + echo "report_log_arg=--report-log=reports/pytest-report-${{ matrix.shard_name }}.jsonl" >> $GITHUB_ENV + else + echo "report_log_arg=" >> $GITHUB_ENV + fi - name: run tests shell: bash run: | python -Wd -m pytest -p no:randomly --ds=${{ env.settings_path }} ${{ env.unit_test_paths }} --cov=. \ - --report-log=reports/pytest-report-${{ matrix.shard_name }}.jsonl + ${{ env.report_log_arg }} - name: Upload pytest timing report - if: always() + if: github.ref == 'refs/heads/master' uses: actions/upload-artifact@v7 with: name: pytest-report-${{ matrix.shard_name }}-${{ matrix.python-version }}-${{ matrix.django-version }}-${{ matrix.mongo-version }}-${{ matrix.os-version }}