-
-
Notifications
You must be signed in to change notification settings - Fork 767
Expand file tree
/
Copy pathrun_tests.sh
More file actions
executable file
·383 lines (328 loc) · 14.1 KB
/
run_tests.sh
File metadata and controls
executable file
·383 lines (328 loc) · 14.1 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
#!/bin/bash
# Test runner - runs Vader test suite
set -euo pipefail
# Cleanup function to remove temporary files on exit
cleanup() {
# Remove any leftover temporary test scripts
rm -f .tmp_run_test_*.sh
# Cleanup root-owned files created by Docker container
cleanup_root_files "$(pwd)"
}
# Cleanup function to remove root-owned files created by Docker container
# This function ensures cleanup only happens within the git repository root
cleanup_root_files() {
local provided_path="${1:-$(pwd)}"
# Find git root directory - this ensures we only operate within the project
local git_root
if ! git_root=$(cd "$provided_path" && git rev-parse --show-toplevel 2>/dev/null); then
log_warn "Not in a git repository, skipping cleanup"
return 0
fi
# Normalize paths for comparison
git_root=$(cd "$git_root" && pwd)
local normalized_path=$(cd "$provided_path" && pwd)
# Safety check: ensure the provided path is within git root
if [[ "$normalized_path" != "$git_root"* ]]; then
log_error "Path '$normalized_path' is outside git root '$git_root', aborting cleanup"
return 1
fi
# Use git root as the base for cleanup operations
local project_root="$git_root"
log_info "Cleaning up files created by Docker container in: $project_root"
# Find and remove root-owned files/directories that shouldn't persist
# Use sudo if available, otherwise try without (may fail silently)
if command -v sudo &> /dev/null; then
# Remove Python cache files (only within git root)
sudo find "$project_root" -type d -name "__pycache__" -user root -exec rm -rf {} + 2>/dev/null || true
sudo find "$project_root" -type f \( -name "*.pyc" -o -name "*.pyo" \) -user root -delete 2>/dev/null || true
# Remove temporary test scripts (only within git root)
sudo find "$project_root" -type f -name ".tmp_run_test_*.sh" -user root -delete 2>/dev/null || true
# Remove test artifacts (only within git root)
sudo rm -rf "$project_root/test-logs" "$project_root/results" 2>/dev/null || true
sudo rm -f "$project_root/test-results.json" "$project_root/coverage.xml" 2>/dev/null || true
# Remove Vim swap files (only within git root)
sudo find "$project_root" -type f \( -name "*.swp" -o -name "*.swo" -o -name ".*.swp" -o -name ".*.swo" \) -user root -delete 2>/dev/null || true
else
# Without sudo, try to remove files we can access (only within git root)
find "$project_root" -type d -name "__pycache__" -exec rm -rf {} + 2>/dev/null || true
find "$project_root" -type f \( -name "*.pyc" -o -name "*.pyo" -o -name ".tmp_run_test_*.sh" -o -name "*.swp" -o -name "*.swo" \) -delete 2>/dev/null || true
rm -rf "$project_root/test-logs" "$project_root/results" 2>/dev/null || true
rm -f "$project_root/test-results.json" "$project_root/coverage.xml" 2>/dev/null || true
fi
}
trap cleanup EXIT INT TERM
echo "⚡ Running Vader Test Suite (Final)..."
# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
BLUE='\033[0;34m'
YELLOW='\033[1;33m'
NC='\033[0m' # No Color
log_info() {
echo -e "${BLUE}[INFO]${NC} $*"
}
log_success() {
echo -e "${GREEN}[SUCCESS]${NC} $*"
}
log_error() {
echo -e "${RED}[ERROR]${NC} $*"
}
log_warn() {
echo -e "${YELLOW}[WARN]${NC} $*"
}
# Find test files
TEST_FILES=()
if [[ -d "tests/vader" ]]; then
mapfile -t TEST_FILES < <(find tests/vader -name "*.vader" -type f | sort)
fi
if [[ ${#TEST_FILES[@]} -eq 0 ]]; then
log_error "No Vader test files found"
exit 1
fi
log_info "Found ${#TEST_FILES[@]} test file(s)"
# Log environment information for debugging
log_info "Environment:"
log_info " Docker: $(docker --version 2>&1 || echo 'not available')"
log_info " Docker Compose: $(docker compose version 2>&1 || echo 'not available')"
log_info " Working directory: $(pwd)"
log_info " CI environment: ${CI:-false}"
log_info " GITHUB_ACTIONS: ${GITHUB_ACTIONS:-false}"
log_info " PYTHON_VERSION: ${PYTHON_VERSION:-not set}"
# Check if docker compose is available
if ! command -v docker &> /dev/null; then
log_error "Docker is not available"
exit 1
fi
if ! docker compose version &> /dev/null; then
log_error "Docker Compose is not available"
exit 1
fi
# Ensure docker compose file exists
if [ ! -f "docker-compose.yml" ]; then
log_error "docker-compose.yml not found in current directory"
exit 1
fi
# Verify docker compose can see the service
if ! docker compose config --services | grep -q "python-mode-tests"; then
log_error "python-mode-tests service not found in docker-compose.yml"
log_info "Available services: $(docker compose config --services 2>&1 || echo 'failed to get services')"
exit 1
fi
# Run tests using docker compose
FAILED_TESTS=()
PASSED_TESTS=()
for test_file in "${TEST_FILES[@]}"; do
test_name=$(basename "$test_file" .vader)
log_info "Running test: $test_name"
# Create a test script that closely follows the legacy test approach
TEST_SCRIPT=$(cat <<'EOFSCRIPT'
#!/bin/bash
set -euo pipefail
cd /workspace/python-mode
# Ensure vader.vim is available (should be installed in Dockerfile, but check anyway)
if [ ! -d /root/.vim/pack/vader/start/vader.vim ]; then
mkdir -p /root/.vim/pack/vader/start
git clone --depth 1 https://github.com/junegunn/vader.vim.git /root/.vim/pack/vader/start/vader.vim 2>&1 || {
echo "ERROR: Failed to install Vader.vim"
exit 1
}
fi
# Set up environment variables similar to legacy tests
export VIM_BINARY=${VIM_BINARY:-vim}
export VIM_TEST_VIMRC="tests/utils/vimrc"
export VIM_OUTPUT_FILE="/tmp/vader_output.txt"
export VIM_DISPOSABLE_PYFILE="/tmp/test_sample.py"
# Create a sample Python file for testing
cat > "$VIM_DISPOSABLE_PYFILE" << 'EOFPY'
def hello():
print("Hello, World!")
return True
EOFPY
# Run the Vader test with minimal setup and verbose output
# Use absolute path for test file
TEST_FILE_PATH="/workspace/python-mode/PLACEHOLDER_TEST_FILE"
if [ ! -f "$TEST_FILE_PATH" ]; then
echo "ERROR: Test file not found: $TEST_FILE_PATH"
exit 1
fi
echo "=== Starting Vader test: $TEST_FILE_PATH ==="
echo "=== Vim binary: $VIM_BINARY ==="
echo "=== Vimrc: $VIM_TEST_VIMRC ==="
# Verify vim is available
if ! command -v "$VIM_BINARY" &> /dev/null; then
echo "ERROR: Vim binary not found: $VIM_BINARY"
exit 1
fi
# Use -es (ex mode, silent) for better output handling as Vader recommends
# Add explicit error handling and ensure vim exits
timeout 60 $VIM_BINARY \
--not-a-term \
-es \
-i NONE \
-u /root/.vimrc \
-c "Vader! $TEST_FILE_PATH" \
-c "qa!" \
< /dev/null > "$VIM_OUTPUT_FILE" 2>&1
EXIT_CODE=$?
echo "=== Vim exit code: $EXIT_CODE ==="
# Show all output for debugging
echo "=== Full Vader output ==="
cat "$VIM_OUTPUT_FILE" 2>/dev/null || echo "No output file generated"
echo "=== End output ==="
# Check the output for success - Vader outputs various success patterns
# Look for patterns like "Success/Total: X/Y" or "X/Y tests passed" or just check for no failures
if grep -qiE "(Success/Total|tests? passed|all tests? passed)" "$VIM_OUTPUT_FILE" 2>/dev/null; then
# Check if there are any failures mentioned
if grep -qiE "(FAILED|failed|error)" "$VIM_OUTPUT_FILE" 2>/dev/null && ! grep -qiE "(Success/Total.*[1-9]|tests? passed)" "$VIM_OUTPUT_FILE" 2>/dev/null; then
echo "ERROR: Test failed - failures detected in output"
exit 1
else
echo "SUCCESS: Test passed"
exit 0
fi
elif [ "$EXIT_CODE" -eq 0 ] && ! grep -qiE "(FAILED|failed|error|E[0-9]+)" "$VIM_OUTPUT_FILE" 2>/dev/null; then
# If exit code is 0 and no errors found, consider it a pass
echo "SUCCESS: Test passed (exit code 0, no errors)"
exit 0
else
echo "ERROR: Test failed"
echo "=== Debug info ==="
echo "Exit code: $EXIT_CODE"
echo "Output file size: $(wc -l < "$VIM_OUTPUT_FILE" 2>/dev/null || echo 0) lines"
echo "Last 20 lines of output:"
tail -20 "$VIM_OUTPUT_FILE" 2>/dev/null || echo "No output available"
exit 1
fi
EOFSCRIPT
)
# Replace placeholder with actual test file
# The template already has /workspace/python-mode/ prefix, so just use the relative path
TEST_SCRIPT="${TEST_SCRIPT//PLACEHOLDER_TEST_FILE/$test_file}"
# Run test in container and capture full output
# Use a temporary file to capture output reliably
TEMP_OUTPUT=$(mktemp)
TEMP_SCRIPT=$(mktemp)
echo "$TEST_SCRIPT" > "$TEMP_SCRIPT"
chmod +x "$TEMP_SCRIPT"
# Use a more reliable method: write script to workspace (which is mounted as volume)
# This avoids stdin redirection issues that can cause hanging
SCRIPT_PATH_IN_CONTAINER="/workspace/python-mode/.tmp_run_test_${test_name}.sh"
cp "$TEMP_SCRIPT" ".tmp_run_test_${test_name}.sh"
chmod +x ".tmp_run_test_${test_name}.sh"
# Execute script in container with proper timeout and error handling
# Use --no-TTY to prevent hanging on TTY allocation
# Capture both stdout and stderr, and check exit code properly
# Note: timeout returns 124 if timeout occurred, otherwise returns the command's exit code
set +e # Temporarily disable exit on error to capture exit code
# Build docker compose command with environment variables
# Environment variables are passed via -e flags before the service name
DOCKER_ENV_ARGS=()
if [ -n "${PYTHON_VERSION:-}" ]; then
DOCKER_ENV_ARGS+=(-e "PYTHON_VERSION=${PYTHON_VERSION}")
fi
if [ -n "${GITHUB_ACTIONS:-}" ]; then
DOCKER_ENV_ARGS+=(-e "GITHUB_ACTIONS=${GITHUB_ACTIONS}")
fi
log_info "Running docker compose with env: PYTHON_VERSION=${PYTHON_VERSION:-not set}, GITHUB_ACTIONS=${GITHUB_ACTIONS:-not set}"
timeout 120 docker compose run --rm --no-TTY "${DOCKER_ENV_ARGS[@]}" python-mode-tests bash "$SCRIPT_PATH_IN_CONTAINER" > "$TEMP_OUTPUT" 2>&1
DOCKER_EXIT_CODE=$?
set -e # Re-enable exit on error
log_info "Docker command completed with exit code: $DOCKER_EXIT_CODE"
OUTPUT=$(cat "$TEMP_OUTPUT" 2>/dev/null || echo "")
# Cleanup temporary files
rm -f "$TEMP_SCRIPT" ".tmp_run_test_${test_name}.sh"
# Cleanup root-owned files after each Docker execution
cleanup_root_files "$(pwd)"
# Check if docker command timed out or failed
if [ "$DOCKER_EXIT_CODE" -eq 124 ]; then
log_error "Test timed out: $test_name (exceeded 120s timeout)"
echo "--- Timeout Details for $test_name ---"
echo "$OUTPUT" | tail -50
echo "--- End Timeout Details ---"
FAILED_TESTS+=("$test_name")
rm -f "$TEMP_OUTPUT"
continue
fi
# Check if docker compose command itself failed (e.g., image not found, service not available)
if [ "$DOCKER_EXIT_CODE" -ne 0 ] && [ -z "$OUTPUT" ]; then
log_error "Docker compose command failed for test: $test_name (exit code: $DOCKER_EXIT_CODE, no output)"
log_info "Attempting to verify docker compose setup..."
docker compose ps 2>&1 || true
docker compose images 2>&1 || true
FAILED_TESTS+=("$test_name")
rm -f "$TEMP_OUTPUT"
continue
fi
# Check if output is empty (potential issue)
if [ -z "$OUTPUT" ]; then
log_error "Test produced no output: $test_name"
echo "--- Error: No output from test execution ---"
echo "Docker exit code: $DOCKER_EXIT_CODE"
FAILED_TESTS+=("$test_name")
rm -f "$TEMP_OUTPUT"
continue
fi
# Check for success message in output
if echo "$OUTPUT" | grep -q "SUCCESS: Test passed"; then
log_success "Test passed: $test_name"
PASSED_TESTS+=("$test_name")
else
# Check if Vader reported success (even with some failures, if most pass we might want to continue)
# Extract Success/Total ratio from output
SUCCESS_LINE=$(echo "$OUTPUT" | grep -iE "Success/Total:" | tail -1)
if [ -n "$SUCCESS_LINE" ]; then
# Extract numbers like "Success/Total: 6/7" or "Success/Total: 1/8"
TOTAL_TESTS=$(echo "$SUCCESS_LINE" | sed -nE 's/.*Success\/Total:[^0-9]*([0-9]+)\/([0-9]+).*/\2/p')
PASSED_COUNT=$(echo "$SUCCESS_LINE" | sed -nE 's/.*Success\/Total:[^0-9]*([0-9]+)\/([0-9]+).*/\1/p')
if [ -n "$TOTAL_TESTS" ] && [ -n "$PASSED_COUNT" ]; then
if [ "$PASSED_COUNT" -eq "$TOTAL_TESTS" ]; then
log_success "Test passed: $test_name ($PASSED_COUNT/$TOTAL_TESTS)"
PASSED_TESTS+=("$test_name")
else
log_error "Test partially failed: $test_name ($PASSED_COUNT/$TOTAL_TESTS passed)"
echo "--- Test Results for $test_name ---"
echo "$SUCCESS_LINE"
echo "$OUTPUT" | grep -E "\(X\)|FAILED|failed|error" | head -10
echo "--- End Test Results ---"
FAILED_TESTS+=("$test_name")
fi
else
log_error "Test failed: $test_name (could not parse results)"
echo "--- Error Details for $test_name ---"
echo "Docker exit code: $DOCKER_EXIT_CODE"
echo "$OUTPUT" | tail -50
echo "--- End Error Details ---"
FAILED_TESTS+=("$test_name")
fi
else
log_error "Test failed: $test_name (no success message found)"
echo "--- Error Details for $test_name ---"
echo "Docker exit code: $DOCKER_EXIT_CODE"
echo "$OUTPUT" | tail -50
echo "--- End Error Details ---"
FAILED_TESTS+=("$test_name")
fi
fi
rm -f "$TEMP_OUTPUT"
done
# Summary
echo
log_info "Test Summary"
log_info "============"
log_info "Total tests: ${#TEST_FILES[@]}"
log_info "Passed: ${#PASSED_TESTS[@]}"
log_info "Failed: ${#FAILED_TESTS[@]}"
# Final cleanup before exit
cleanup_root_files "$(pwd)"
if [[ ${#FAILED_TESTS[@]} -gt 0 ]]; then
echo
log_error "Failed tests:"
for test in "${FAILED_TESTS[@]}"; do
echo " ✗ $test"
done
exit 1
else
echo
log_success "All tests passed!"
exit 0
fi