Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Avoid random failures in system tests #5319

Closed
wants to merge 24 commits into from
Closed

Avoid random failures in system tests #5319

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
24 commits
Select commit Hold shift + click to select a range
c9ff1aa
Avoid race conditions in tests
jorg-vr Jan 22, 2024
2c46fde
Use different objects for each test
jorg-vr Jan 22, 2024
69f0f3a
Fix code tab link
jorg-vr Jan 23, 2024
d081b9c
Merge branch 'main' into fix/randomly-failing-tests
jorg-vr Jan 26, 2024
0a0930a
Use other renderer
jorg-vr Jan 26, 2024
2d8a08a
Use different submission json
jorg-vr Jan 26, 2024
cdc755a
Increase chance of failure
jorg-vr Jan 26, 2024
8d0e10b
Revert "Increase chance of failure"
jorg-vr Jan 26, 2024
9612e88
Increase wait time
jorg-vr Jan 26, 2024
7b4f5f2
Also run on dispatch
jorg-vr Jan 26, 2024
43888d8
Increase wait time
jorg-vr Jan 26, 2024
a96258c
Try if page loads at all
jorg-vr Jan 26, 2024
af92d3d
Make submission counts unique
jorg-vr Jan 29, 2024
8a81403
Test a selector that actually changes
jorg-vr Jan 29, 2024
13b0d3b
Increase maximum scratchpad wait time
jorg-vr Jan 30, 2024
6609925
Serialise tests
jorg-vr Jan 30, 2024
6c339c3
Revert "Serialise tests"
jorg-vr Jan 30, 2024
5022443
Increase default wait time
jorg-vr Jan 30, 2024
95df238
Remove non useull extra waits
jorg-vr Jan 30, 2024
7b2f472
Undo link rename
jorg-vr Jan 30, 2024
aba9424
Undo change
jorg-vr Jan 30, 2024
b463e1c
Undo changes
jorg-vr Jan 30, 2024
21e1c4f
Increase wait time
jorg-vr Jan 30, 2024
4bc0829
Remve extra wait for papyros
jorg-vr Jan 30, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion test/application_system_test_case.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ def dump_js_coverage
end

Capybara.configure do |config|
config.default_max_wait_time = 10
config.default_max_wait_time = 60 # seconds
end

class ApplicationSystemTestCase < ActionDispatch::SystemTestCase
Expand Down
39 changes: 24 additions & 15 deletions test/system/annotations_test.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,9 +12,14 @@ class AnnotationsTest < ApplicationSystemTestCase
@zeus = create :zeus
sign_in @zeus
@code_lines = Faker::Lorem.sentences(number: 5)
@instance = create :correct_submission, result: Rails.root.join('db/results/python-result.json').read, code: @code_lines.join("\n"), course: create(:course)
@instance.exercise.judge.renderer = PythiaRenderer
@instance.exercise.judge.save
@instance = new_submission
end

def new_submission
sub = create :correct_submission, result: Rails.root.join('db/results/python-result.json').read, code: @code_lines.join("\n"), course: create(:course)
sub.exercise.judge.renderer = PythiaRenderer
sub.exercise.judge.save
sub
end

test 'Can view submission page' do
Expand Down Expand Up @@ -373,8 +378,9 @@ class AnnotationsTest < ApplicationSystemTestCase
end

test 'Can reply to an annotation' do
create :annotation, submission: @instance, user: @zeus
visit(submission_path(id: @instance.id))
instance = new_submission
create :annotation, submission: instance, user: @zeus
visit(submission_path(id: instance.id))
click_on 'Code'

thread = find('d-thread')
Expand All @@ -395,9 +401,10 @@ class AnnotationsTest < ApplicationSystemTestCase
end

test 'Cannot delete an annotation with replies' do
annot = create :annotation, submission: @instance, user: @zeus
create :annotation, submission: @instance, user: @zeus, thread_root: annot
visit(submission_path(id: @instance.id))
instance = new_submission
annot = create :annotation, submission: instance, user: @zeus
create :annotation, submission: instance, user: @zeus, thread_root: annot
visit(submission_path(id: instance.id))
click_on 'Code'

within 'd-thread' do
Expand All @@ -412,9 +419,10 @@ class AnnotationsTest < ApplicationSystemTestCase
end

test 'Can delete an annotation after deleting all replies' do
annot = create :annotation, submission: @instance, user: @zeus
create :annotation, submission: @instance, user: @zeus, thread_root: annot
visit(submission_path(id: @instance.id))
instance = new_submission
annot = create :annotation, submission: instance, user: @zeus
create :annotation, submission: instance, user: @zeus, thread_root: annot
visit(submission_path(id: instance.id))
click_on 'Code'

within 'd-thread' do
Expand All @@ -436,10 +444,11 @@ class AnnotationsTest < ApplicationSystemTestCase
end

test 'can delete the middle annotation in a thread' do
annot = create :annotation, submission: @instance, user: @zeus
create :annotation, submission: @instance, user: @zeus, thread_root: annot
create :annotation, submission: @instance, user: @zeus, thread_root: annot
visit(submission_path(id: @instance.id))
instance = new_submission
annot = create :annotation, submission: instance, user: @zeus
create :annotation, submission: instance, user: @zeus, thread_root: annot
create :annotation, submission: instance, user: @zeus, thread_root: annot
visit(submission_path(id: instance.id))
click_on 'Code'

within 'd-thread' do
Expand Down
59 changes: 35 additions & 24 deletions test/system/questions_test.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,12 +8,17 @@ class QuestionsTest < ApplicationSystemTestCase
# Make `assert_*` methods behave like Minitest assertions
include Capybara::Minitest::Assertions

def new_submission
sub = create :correct_submission, result: Rails.root.join('db/results/python-result.json').read, code: @code_lines.join("\n"), course: create(:course)
sub.exercise.judge.renderer = PythiaRenderer
sub.exercise.judge.save
sub
end

setup do
@code_lines = Faker::Lorem.sentences(number: 5)
@course = create :course, enabled_questions: true
@submission = create :correct_submission, result: Rails.root.join('db/results/python-result.json').read, code: @code_lines.join("\n"), course: @course
@submission.exercise.judge.renderer = PythiaRenderer
@submission.exercise.judge.save
@submission = new_submission
@student = @submission.user
sign_in @student

Expand Down Expand Up @@ -61,7 +66,8 @@ class QuestionsTest < ApplicationSystemTestCase
end

test 'Can submit a question' do
visit(submission_path(id: @submission.id))
submission = new_submission
visit(submission_path(id: submission.id))
click_on 'Code'

question = Faker::Lorem.question
Expand All @@ -82,20 +88,21 @@ class QuestionsTest < ApplicationSystemTestCase
# Also acts as sleep until full ajax call is completed
end

assert_equal 1, Question.count, 'Too little or too many questions were created'
assert_equal 1, Question.where(submission: submission).count, 'Too little or too many questions were created'
q = Question.first

assert_equal q.question_text, question, 'Something went wrong in saving the question'
assert_predicate q, :unanswered?, 'Should be an unanswered question'
end

test 'student can mark a question as resolved' do
q = create :question, submission: @submission, user: @student
submission = new_submission
q = create :question, submission: submission, user: submission.user

assert_equal 1, Question.count, 'Test is invalid if magically no or more questions appear here'
assert_equal 1, Question.where(submission: submission).count, 'Test is invalid if magically no or more questions appear here'
assert_predicate q, :unanswered?, 'Question should start as unanswered'

visit(submission_path(id: @submission.id))
visit(submission_path(id: submission.id))
click_on 'Code'

thread = find('d-thread')
Expand All @@ -106,25 +113,26 @@ class QuestionsTest < ApplicationSystemTestCase
assert_no_css '.mdi-comment-question-outline'
end

assert_equal 1, Question.count, 'There should still only be one question'
assert_equal 1, Question.where(submission: submission).count, 'There should still only be one question'
q = Question.first

assert_not q.unanswered?, 'Question should have moved onto answered status'
assert_predicate q, :answered?, 'Question should have moved onto answered status'
end

test 'Responding to a question should mark the question as answered' do
q = create :question, submission: @submission, user: @student
submission = new_submission
q = create :question, submission: submission, user: submission.user

assert_equal 1, Question.count, 'Test is invalid if magically no or more questions appear here'
assert_equal 1, Question.where(submission: submission).count, 'Test is invalid if magically no or more questions appear here'
assert_predicate q, :unanswered?, 'Question should start as unanswered'

visit(submission_path(id: @submission.id))
visit(submission_path(id: submission.id))
click_on 'Code'

thread = find('d-thread')
within thread do
assert_selector '.annotation', count: 1
assert_selector '.annotation-text', count: 1

fake_answer_input = find('input')
fake_answer_input.click
Expand All @@ -135,21 +143,22 @@ class QuestionsTest < ApplicationSystemTestCase

click_on 'Reply'

assert_selector '.annotation', count: 2
assert_selector '.annotation-text', count: 2
end

assert_equal 2, Question.count, 'There should be two questions now'
assert_equal 2, Question.where(submission: submission).count, 'There should be two questions now'
assert_not q.reload.unanswered?, 'Question should have moved onto answered status'
assert_predicate q.reload, :answered?, 'Question should have moved onto answered status'
end

test 'An unanswered question should contain an icon to visualize its status' do
q = create :question, submission: @submission, user: @student
submission = new_submission
q = create :question, submission: submission, user: submission.user

assert_equal 1, Question.count, 'Test is invalid if magically no or more questions appear here'
assert_equal 1, Question.where(submission: submission).count, 'Test is invalid if magically no or more questions appear here'
assert_predicate q, :unanswered?, 'Question should start as unanswered'

visit(submission_path(id: @submission.id))
visit(submission_path(id: submission.id))
click_on 'Code'

thread = find('d-thread')
Expand All @@ -160,12 +169,13 @@ class QuestionsTest < ApplicationSystemTestCase
end

test 'The status icon should change to in progress when someone clicks reply' do
q = create :question, submission: @submission, user: @student
submission = new_submission
q = create :question, submission: submission, user: submission.user

assert_equal 1, Question.count, 'Test is invalid if magically no or more questions appear here'
assert_equal 1, Question.where(submission: submission).count, 'Test is invalid if magically no or more questions appear here'
assert_predicate q, :unanswered?, 'Question should start as unanswered'

visit(submission_path(id: @submission.id))
visit(submission_path(id: submission.id))
click_on 'Code'

thread = find('d-thread')
Expand All @@ -180,12 +190,13 @@ class QuestionsTest < ApplicationSystemTestCase
end

test 'The question becomes unanswered again when a teacher cancels the reply' do
q = create :question, submission: @submission, user: @student
submission = new_submission
q = create :question, submission: submission, user: submission.user

assert_equal 1, Question.count, 'Test is invalid if magically no or more questions appear here'
assert_equal 1, Question.where(submission: submission).count, 'Test is invalid if magically no or more questions appear here'
assert_predicate q, :unanswered?, 'Question should start as unanswered'

visit(submission_path(id: @submission.id))
visit(submission_path(id: submission.id))
click_on 'Code'

thread = find('d-thread')
Expand Down
3 changes: 1 addition & 2 deletions test/system/scratchpad_test.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,8 +43,7 @@ def run_code(code)
code = "print(\"Hello World!\")\n"
run_code code
output_area = find_by_id('scratchpad-output-wrapper')
# First run, so wait longer for output to appear
output_area.find('span', text: 'Hello World!', wait: 20)
output_area.find('span', text: 'Hello World!')

# Scratchpad can process user input in interactive mode
scratchpad_input = 'Echo'
Expand Down
Loading