diff --git a/kernelboard/api/leaderboard.py b/kernelboard/api/leaderboard.py index 36c391a..c5b9ea0 100644 --- a/kernelboard/api/leaderboard.py +++ b/kernelboard/api/leaderboard.py @@ -159,6 +159,13 @@ def _get_query(): FROM leaderboard.submission s JOIN leaderboard.runs r ON r.submission_id = s.id WHERE s.leaderboard_id = %(leaderboard_id)s + AND s.status <> 'hacked' + AND NOT EXISTS ( + SELECT 1 + FROM leaderboard.submission_job_status sjs + WHERE sjs.submission_id = s.id + AND sjs.status = 'hacked' + ) GROUP BY s.user_id, r.runner ), @@ -181,6 +188,13 @@ def _get_query(): AND r.score IS NOT NULL AND r.passed AND s.leaderboard_id = %(leaderboard_id)s + AND s.status <> 'hacked' + AND NOT EXISTS ( + SELECT 1 + FROM leaderboard.submission_job_status sjs + WHERE sjs.submission_id = s.id + AND sjs.status = 'hacked' + ) AND EXISTS ( SELECT 1 FROM leaderboard.runs sr @@ -280,6 +294,13 @@ def get_custom_trend(leaderboard_id: int): AND r.score IS NOT NULL AND r.passed = true AND NOT r.secret + AND s.status <> 'hacked' + AND NOT EXISTS ( + SELECT 1 + FROM leaderboard.submission_job_status sjs + WHERE sjs.submission_id = s.id + AND sjs.status = 'hacked' + ) AND EXISTS ( SELECT 1 FROM leaderboard.runs sr @@ -418,6 +439,13 @@ def get_user_trend(leaderboard_id: int): AND r.score IS NOT NULL AND r.passed = true AND NOT r.secret + AND s.status <> 'hacked' + AND NOT EXISTS ( + SELECT 1 + FROM leaderboard.submission_job_status sjs + WHERE sjs.submission_id = s.id + AND sjs.status = 'hacked' + ) AND EXISTS ( SELECT 1 FROM leaderboard.runs sr @@ -521,6 +549,13 @@ def get_fastest_trend(leaderboard_id: int): AND r.score IS NOT NULL AND r.passed = true AND NOT r.secret + AND s.status <> 'hacked' + AND NOT EXISTS ( + SELECT 1 + FROM leaderboard.submission_job_status sjs + WHERE sjs.submission_id = s.id + AND sjs.status = 'hacked' + ) AND EXISTS ( SELECT 1 FROM leaderboard.runs sr @@ -618,6 +653,13 @@ def search_users(leaderboard_id: int): FROM leaderboard.user_info u JOIN leaderboard.submission s ON s.user_id = u.id WHERE s.leaderboard_id = %s + AND s.status <> 'hacked' + AND NOT EXISTS ( + SELECT 1 + FROM leaderboard.submission_job_status sjs + WHERE sjs.submission_id = s.id + AND sjs.status = 'hacked' + ) AND u.user_name ILIKE %s ORDER BY u.user_name LIMIT %s @@ -629,6 +671,13 @@ def search_users(leaderboard_id: int): FROM leaderboard.user_info u JOIN leaderboard.submission s ON s.user_id = u.id WHERE s.leaderboard_id = %s + AND s.status <> 'hacked' + AND NOT EXISTS ( + SELECT 1 + FROM leaderboard.submission_job_status sjs + WHERE sjs.submission_id = s.id + AND sjs.status = 'hacked' + ) ORDER BY u.user_name LIMIT %s """ diff --git a/kernelboard/api/leaderboard_summaries.py b/kernelboard/api/leaderboard_summaries.py index 961c58d..5406fce 100644 --- a/kernelboard/api/leaderboard_summaries.py +++ b/kernelboard/api/leaderboard_summaries.py @@ -16,7 +16,7 @@ leaderboard_summaries_bp = Blueprint("leaderboard_summaries_bp", __name__, url_prefix="/leaderboard-summaries") # Redis cache key prefix for ended leaderboard top_users -CACHE_KEY_PREFIX = "lb_top_users:" +CACHE_KEY_PREFIX = "lb_top_users:v2:" # ============================================================================= @@ -370,6 +370,13 @@ def _get_query_for_ids(): AND r.score IS NOT NULL AND r.passed AND s.leaderboard_id IN %s + AND s.status <> 'hacked' + AND NOT EXISTS ( + SELECT 1 + FROM leaderboard.submission_job_status sjs + WHERE sjs.submission_id = s.id + AND sjs.status = 'hacked' + ) AND EXISTS ( SELECT 1 FROM leaderboard.runs sr @@ -483,6 +490,13 @@ def _get_query(): WHERE NOT r.secret AND r.score IS NOT NULL AND r.passed + AND s.status <> 'hacked' + AND NOT EXISTS ( + SELECT 1 + FROM leaderboard.submission_job_status sjs + WHERE sjs.submission_id = s.id + AND sjs.status = 'hacked' + ) AND EXISTS ( SELECT 1 FROM leaderboard.runs sr diff --git a/kernelboard/index.py b/kernelboard/index.py index a45ee78..81ca1ac 100644 --- a/kernelboard/index.py +++ b/kernelboard/index.py @@ -86,6 +86,13 @@ def index(): WHERE NOT r.secret AND r.score IS NOT NULL AND r.passed + AND s.status <> 'hacked' + AND NOT EXISTS ( + SELECT 1 + FROM leaderboard.submission_job_status sjs + WHERE sjs.submission_id = s.id + AND sjs.status = 'hacked' + ) AND EXISTS ( SELECT 1 FROM leaderboard.runs sr diff --git a/kernelboard/leaderboard.py b/kernelboard/leaderboard.py index 2f8f6ed..21f32d3 100644 --- a/kernelboard/leaderboard.py +++ b/kernelboard/leaderboard.py @@ -53,6 +53,13 @@ def leaderboard(leaderboard_id: int): AND r.score IS NOT NULL AND r.passed AND s.leaderboard_id = %(leaderboard_id)s + AND s.status <> 'hacked' + AND NOT EXISTS ( + SELECT 1 + FROM leaderboard.submission_job_status sjs + WHERE sjs.submission_id = s.id + AND sjs.status = 'hacked' + ) AND EXISTS ( SELECT 1 FROM leaderboard.runs sr diff --git a/ranking_worker.py b/ranking_worker.py index d118f17..4873a4d 100644 --- a/ranking_worker.py +++ b/ranking_worker.py @@ -130,6 +130,13 @@ def ensure_snapshot_table(conn): WHERE NOT r.secret AND r.score IS NOT NULL AND r.passed + AND s.status <> 'hacked' + AND NOT EXISTS ( + SELECT 1 + FROM leaderboard.submission_job_status sjs + WHERE sjs.submission_id = s.id + AND sjs.status = 'hacked' + ) AND EXISTS ( SELECT 1 FROM leaderboard.runs sr diff --git a/tests/api/test_leaderboard_api.py b/tests/api/test_leaderboard_api.py index 6c3d0cf..02f39f5 100644 --- a/tests/api/test_leaderboard_api.py +++ b/tests/api/test_leaderboard_api.py @@ -150,3 +150,120 @@ def test_failed_secret_benchmark_hides_public_leaderboard_run(client, app): assert "hidden_secret_fail.py" not in ranked_files assert "hidden_missing_secret.py" not in ranked_files assert "visible_public_pass.py" in ranked_files + + +def test_hacked_submissions_are_hidden_from_public_leaderboard(client, app): + with app.app_context(): + conn = get_db_connection() + with conn.cursor() as cur: + submissions = [ + ( + 900011, + "hidden_submission_status_hacked.py", + "123456789012345", + "hacked", + ), + (900012, "hidden_job_status_hacked.py", "234567890123456", "active"), + (900013, "visible_clean_status.py", "345678901234567", "active"), + ] + for submission_id, file_name, user_id, status in submissions: + cur.execute( + """ + INSERT INTO leaderboard.submission + ( + id, + leaderboard_id, + file_name, + user_id, + code_id, + submission_time, + done, + status + ) + VALUES + (%s, 339, %s, %s, 13, NOW(), TRUE, %s) + """, + (submission_id, file_name, user_id, status), + ) + cur.execute( + """ + INSERT INTO leaderboard.runs + ( + id, + submission_id, + start_time, + end_time, + mode, + secret, + runner, + score, + passed, + compilation, + meta, + result, + system_info + ) + VALUES + ( + %s, + %s, + NOW(), + NOW(), + 'leaderboard', + FALSE, + 'H100', + %s, + TRUE, + '{}', + '{}', + '{}', + '{}' + ), + ( + %s, + %s, + NOW(), + NOW(), + 'leaderboard', + TRUE, + 'H100', + %s, + TRUE, + '{}', + '{}', + '{}', + '{}' + ) + """, + ( + submission_id * 10, + submission_id, + -submission_id, + submission_id * 10 + 1, + submission_id, + -submission_id, + ), + ) + + cur.execute( + """ + INSERT INTO leaderboard.submission_job_status + (submission_id, status, created_at, last_heartbeat) + VALUES + (900012, 'hacked', NOW(), NOW()) + """ + ) + conn.commit() + + response = client.get("/api/leaderboard/339") + assert response.status_code == 200 + + payload = response.get_json() + ranked_files = { + row["file_name"] + for row in payload["data"]["rankings"]["H100"] + } + + assert "hidden_submission_status_hacked.py" not in ranked_files + assert "hidden_job_status_hacked.py" not in ranked_files + assert "visible_clean_status.py" in ranked_files diff --git a/tests/data.sql b/tests/data.sql index 811fc10..c99155e 100644 --- a/tests/data.sql +++ b/tests/data.sql @@ -105,6 +105,18 @@ CREATE SEQUENCE leaderboard.leaderboard_id_seq ALTER SEQUENCE leaderboard.leaderboard_id_seq OWNED BY leaderboard.leaderboard.id; +-- +-- Name: templates; Type: TABLE; Schema: leaderboard; Owner: - +-- + +CREATE TABLE leaderboard.templates ( + id SERIAL PRIMARY KEY, + leaderboard_id integer NOT NULL, + lang text NOT NULL, + code text NOT NULL +); + + -- -- Name: runs; Type: TABLE; Schema: leaderboard; Owner: - -- @@ -157,7 +169,8 @@ CREATE TABLE leaderboard.submission ( user_id text NOT NULL, code_id integer NOT NULL, submission_time timestamp with time zone NOT NULL, - done boolean DEFAULT false + done boolean DEFAULT false, + status text DEFAULT 'active'::text NOT NULL ); diff --git a/tests/test_hacked_submission_filters.py b/tests/test_hacked_submission_filters.py new file mode 100644 index 0000000..a03d73f --- /dev/null +++ b/tests/test_hacked_submission_filters.py @@ -0,0 +1,20 @@ +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +SUBMISSION_STATUS_FILTER = "s.status <> 'hacked'" +JOB_STATUS_FILTER = "sjs.status = 'hacked'" + + +def test_leaderboard_queries_filter_hacked_submissions(): + expected_counts = { + "kernelboard/api/leaderboard.py": 7, + "kernelboard/api/leaderboard_summaries.py": 2, + "kernelboard/index.py": 1, + "kernelboard/leaderboard.py": 1, + "ranking_worker.py": 1, + } + + for relative_path, expected_count in expected_counts.items(): + source = (ROOT / relative_path).read_text(encoding="utf-8") + assert source.count(SUBMISSION_STATUS_FILTER) == expected_count + assert source.count(JOB_STATUS_FILTER) == expected_count