Skip to content

FIX]: Fix false-positive SUCCESS status for commit tests with missing comparison rows #1119

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
x15sr71 wants to merge 1 commit into
base: master
Choose a base branch
from
Open

FIX]: Fix false-positive SUCCESS status for commit tests with missing comparison rows #1119

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
40 changes: 12 additions & 28 deletions mod_ci/controllers.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,6 @@
from pymysql.err import IntegrityError
from sqlalchemy import and_, func, select
from sqlalchemy.sql import label
from sqlalchemy.sql.functions import count
from werkzeug.utils import secure_filename

from database import DeclEnum, create_session
Expand Down Expand Up @@ -2276,14 +2275,16 @@ def start_ci():
return json.dumps({'msg': 'EOL'})


def update_build_badge(status, test) -> None:
def update_build_badge(status, test, test_results=None) -> None:
"""
Build status badge for current test to be displayed on sample-platform.

:param status: current testing status
:type status: str
:param test: current commit that is tested
:type test: Test
:param test_results: pre-computed results from get_test_results; if None, fetched internally
:type test_results: list | None
:return: null
:rtype: null
"""
Expand All @@ -2294,7 +2295,8 @@ def update_build_badge(status, test) -> None:
shutil.copyfile(original_location, build_status_location)
g.log.info('Build badge updated successfully!')

test_results = get_test_results(test)
if test_results is None:
test_results = get_test_results(test)
test_ids_to_update = []
for category_results in test_results:
test_ids_to_update.extend([test['test'].id for test in category_results['tests'] if not test['error']])
Expand Down Expand Up @@ -2429,33 +2431,15 @@ def progress_type_request(log, test, test_id, request) -> bool:
message = 'Tests aborted due to an error; please check'

elif status == TestStatus.completed:
# Determine if success or failure
# It fails if any of these happen:
# - A crash (unexpected exit code)
# - A not None value on the "got" of a TestResultFile (
# meaning the hashes do not match)
crashes = g.db.query(count(TestResult.exit_code)).filter(
and_(
TestResult.test_id == test.id,
TestResult.exit_code != TestResult.expected_rc
)).scalar()
results_zero_rc = select(RegressionTest.id).filter(
RegressionTest.expected_rc == 0
test_results = get_test_results(test)
has_failures = any(
t['error']
for category in test_results
for t in category['tests']
)
results = g.db.query(count(TestResultFile.got)).filter(
and_(
TestResultFile.test_id == test.id,
TestResultFile.regression_test_id.in_(
results_zero_rc.select()
),
TestResultFile.got.isnot(None)
)
).scalar()
log.debug(f'[Test: {test.id}] Test completed: {crashes} crashes, {results} results')
if crashes > 0 or results > 0:
if has_failures:
state = Status.FAILURE
message = 'Not all tests completed successfully, please check'

else:
state = Status.SUCCESS
message = 'Tests completed'
Expand All @@ -2468,7 +2452,7 @@ def progress_type_request(log, test, test_id, request) -> bool:
message = 'All tests passed'
else:
message = 'Not all tests completed successfully, please check'
update_build_badge(state, test)
update_build_badge(state, test, test_results)

else:
message = progress.message
Expand Down
Loading