Fix: Ensure all skills are tracked as files, not submodules
This commit is contained in:
sck_0
78
skills/loki-mode/tests/run-all-tests.sh
Executable file
78
skills/loki-mode/tests/run-all-tests.sh
Executable file
@@ -0,0 +1,78 @@
|
||||
#!/bin/bash
|
||||
# Loki Mode Test Suite Runner
|
||||
# Runs all test cases for the Loki Mode skill
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
TOTAL_PASSED=0
|
||||
TOTAL_FAILED=0
|
||||
TESTS_RUN=0
|
||||
|
||||
# Colors
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
BLUE='\033[0;34m'
|
||||
NC='\033[0m'
|
||||
|
||||
echo ""
|
||||
echo -e "${BLUE}╔════════════════════════════════════════════════════════════════╗${NC}"
|
||||
echo -e "${BLUE}║ LOKI MODE - COMPREHENSIVE TEST SUITE ║${NC}"
|
||||
echo -e "${BLUE}╚════════════════════════════════════════════════════════════════╝${NC}"
|
||||
echo ""
|
||||
|
||||
run_test() {
|
||||
local test_name="$1"
|
||||
local test_file="$2"
|
||||
|
||||
echo -e "${YELLOW}┌────────────────────────────────────────────────────────────────┐${NC}"
|
||||
echo -e "${YELLOW}│ Running: ${test_name}${NC}"
|
||||
echo -e "${YELLOW}└────────────────────────────────────────────────────────────────┘${NC}"
|
||||
echo ""
|
||||
|
||||
TESTS_RUN=$((TESTS_RUN + 1))
|
||||
|
||||
if bash "$test_file"; then
|
||||
echo ""
|
||||
echo -e "${GREEN}✓ ${test_name} PASSED${NC}"
|
||||
TOTAL_PASSED=$((TOTAL_PASSED + 1))
|
||||
else
|
||||
echo ""
|
||||
echo -e "${RED}✗ ${test_name} FAILED${NC}"
|
||||
TOTAL_FAILED=$((TOTAL_FAILED + 1))
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo ""
|
||||
}
|
||||
|
||||
# Run all tests
|
||||
run_test "Bootstrap Tests" "$SCRIPT_DIR/test-bootstrap.sh"
|
||||
run_test "Task Queue Tests" "$SCRIPT_DIR/test-task-queue.sh"
|
||||
run_test "Circuit Breaker Tests" "$SCRIPT_DIR/test-circuit-breaker.sh"
|
||||
run_test "Timeout & Stuck Process Tests" "$SCRIPT_DIR/test-agent-timeout.sh"
|
||||
run_test "State Recovery Tests" "$SCRIPT_DIR/test-state-recovery.sh"
|
||||
run_test "Wrapper Script Tests" "$SCRIPT_DIR/test-wrapper.sh"
|
||||
|
||||
# Summary
|
||||
echo -e "${BLUE}╔════════════════════════════════════════════════════════════════╗${NC}"
|
||||
echo -e "${BLUE}║ TEST SUITE SUMMARY ║${NC}"
|
||||
echo -e "${BLUE}╚════════════════════════════════════════════════════════════════╝${NC}"
|
||||
echo ""
|
||||
echo -e "Tests Run: ${TESTS_RUN}"
|
||||
echo -e "${GREEN}Passed: ${TOTAL_PASSED}${NC}"
|
||||
echo -e "${RED}Failed: ${TOTAL_FAILED}${NC}"
|
||||
echo ""
|
||||
|
||||
if [ $TOTAL_FAILED -eq 0 ]; then
|
||||
echo -e "${GREEN}╔════════════════════════════════════════════════════════════════╗${NC}"
|
||||
echo -e "${GREEN}║ ALL TESTS PASSED SUCCESSFULLY! ║${NC}"
|
||||
echo -e "${GREEN}╚════════════════════════════════════════════════════════════════╝${NC}"
|
||||
exit 0
|
||||
else
|
||||
echo -e "${RED}╔════════════════════════════════════════════════════════════════╗${NC}"
|
||||
echo -e "${RED}║ SOME TESTS FAILED - PLEASE REVIEW ║${NC}"
|
||||
echo -e "${RED}╚════════════════════════════════════════════════════════════════╝${NC}"
|
||||
exit 1
|
||||
fi
|
||||
348
skills/loki-mode/tests/test-agent-timeout.sh
Executable file
348
skills/loki-mode/tests/test-agent-timeout.sh
Executable file
@@ -0,0 +1,348 @@
|
||||
#!/bin/bash
|
||||
# Test: Agent Timeout and Stuck Process Handling
|
||||
# Tests timeout mechanisms for long-running commands like npm build
|
||||
|
||||
set -uo pipefail
|
||||
# Note: Not using -e to allow collecting all test results
|
||||
|
||||
TEST_DIR=$(mktemp -d)
|
||||
PASSED=0
|
||||
FAILED=0
|
||||
|
||||
# Colors
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
NC='\033[0m'
|
||||
|
||||
log_pass() { echo -e "${GREEN}[PASS]${NC} $1"; ((PASSED++)); }
|
||||
log_fail() { echo -e "${RED}[FAIL]${NC} $1"; ((FAILED++)); }
|
||||
log_test() { echo -e "${YELLOW}[TEST]${NC} $1"; }
|
||||
|
||||
cleanup() {
|
||||
rm -rf "$TEST_DIR"
|
||||
# Kill any test processes
|
||||
pkill -f "test-long-running" 2>/dev/null || true
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
cd "$TEST_DIR"
|
||||
|
||||
echo "========================================"
|
||||
echo "Loki Mode Timeout & Stuck Process Tests"
|
||||
echo "========================================"
|
||||
echo ""
|
||||
|
||||
# macOS-compatible timeout function
|
||||
run_with_timeout() {
|
||||
local timeout_seconds="$1"
|
||||
shift
|
||||
local cmd="$@"
|
||||
|
||||
# Use gtimeout if available (from coreutils), otherwise use Perl
|
||||
if command -v gtimeout &> /dev/null; then
|
||||
gtimeout "$timeout_seconds" bash -c "$cmd"
|
||||
return $?
|
||||
elif command -v timeout &> /dev/null; then
|
||||
timeout "$timeout_seconds" bash -c "$cmd"
|
||||
return $?
|
||||
else
|
||||
# Perl-based timeout (works on macOS)
|
||||
perl -e '
|
||||
alarm shift @ARGV;
|
||||
$SIG{ALRM} = sub { exit 124 };
|
||||
exec @ARGV;
|
||||
' "$timeout_seconds" bash -c "$cmd"
|
||||
return $?
|
||||
fi
|
||||
}
|
||||
|
||||
# Test 1: Command timeout with short process
|
||||
log_test "Command completes within timeout"
|
||||
START=$(date +%s)
|
||||
run_with_timeout 5 "sleep 1" && RESULT="success" || RESULT="timeout"
|
||||
END=$(date +%s)
|
||||
DURATION=$((END - START))
|
||||
|
||||
if [ "$RESULT" = "success" ] && [ $DURATION -lt 3 ]; then
|
||||
log_pass "Short command completed in ${DURATION}s"
|
||||
else
|
||||
log_fail "Short command handling failed (result: $RESULT, duration: ${DURATION}s)"
|
||||
fi
|
||||
|
||||
# Test 2: Command timeout with long process
|
||||
log_test "Command times out correctly"
|
||||
START=$(date +%s)
|
||||
run_with_timeout 2 "sleep 10" && RESULT="success" || RESULT="timeout"
|
||||
END=$(date +%s)
|
||||
DURATION=$((END - START))
|
||||
|
||||
if [ "$RESULT" = "timeout" ] && [ $DURATION -lt 5 ]; then
|
||||
log_pass "Long command timed out correctly in ${DURATION}s"
|
||||
else
|
||||
log_fail "Timeout mechanism failed (duration: ${DURATION}s, result: $RESULT)"
|
||||
fi
|
||||
|
||||
# Test 3: Task timeout configuration
|
||||
log_test "Task timeout configuration"
|
||||
python3 << 'EOF'
|
||||
import json
|
||||
|
||||
# Task with custom timeout
|
||||
task = {
|
||||
"id": "task-build-001",
|
||||
"type": "eng-frontend",
|
||||
"payload": {
|
||||
"action": "build",
|
||||
"command": "npm run build"
|
||||
},
|
||||
"timeout": 600, # 10 minutes for builds
|
||||
"createdAt": "2025-01-15T10:00:00Z"
|
||||
}
|
||||
|
||||
# Different timeouts for different task types
|
||||
TIMEOUT_CONFIG = {
|
||||
'default': 300, # 5 minutes
|
||||
'build': 600, # 10 minutes
|
||||
'test': 900, # 15 minutes
|
||||
'deploy': 1800, # 30 minutes
|
||||
'quick': 60 # 1 minute
|
||||
}
|
||||
|
||||
def get_timeout(task):
|
||||
action = task.get('payload', {}).get('action', 'default')
|
||||
return task.get('timeout', TIMEOUT_CONFIG.get(action, TIMEOUT_CONFIG['default']))
|
||||
|
||||
timeout = get_timeout(task)
|
||||
print(f"TIMEOUT:{timeout}")
|
||||
assert timeout == 600, f"Expected 600, got {timeout}"
|
||||
print("VERIFIED")
|
||||
EOF
|
||||
|
||||
if [ $? -eq 0 ]; then
|
||||
log_pass "Task timeout configuration works"
|
||||
else
|
||||
log_fail "Task timeout configuration failed"
|
||||
fi
|
||||
|
||||
# Test 4: Stuck process detection
|
||||
log_test "Stuck process detection (heartbeat)"
|
||||
python3 << 'EOF'
|
||||
import json
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
# Simulate agent state with heartbeat
|
||||
agent_state = {
|
||||
"id": "eng-backend-01",
|
||||
"status": "active",
|
||||
"currentTask": "task-001",
|
||||
"lastHeartbeat": (datetime.utcnow() - timedelta(minutes=10)).isoformat() + 'Z'
|
||||
}
|
||||
|
||||
HEARTBEAT_TIMEOUT = 300 # 5 minutes
|
||||
|
||||
def is_agent_stuck(agent):
|
||||
if not agent.get('lastHeartbeat'):
|
||||
return False
|
||||
|
||||
last_heartbeat = datetime.fromisoformat(agent['lastHeartbeat'].replace('Z', '+00:00'))
|
||||
age = (datetime.now(last_heartbeat.tzinfo) - last_heartbeat).total_seconds()
|
||||
|
||||
return age > HEARTBEAT_TIMEOUT
|
||||
|
||||
is_stuck = is_agent_stuck(agent_state)
|
||||
print(f"STUCK:{is_stuck}")
|
||||
assert is_stuck == True, "Agent should be detected as stuck"
|
||||
print("VERIFIED")
|
||||
EOF
|
||||
|
||||
if [ $? -eq 0 ]; then
|
||||
log_pass "Stuck process detection works"
|
||||
else
|
||||
log_fail "Stuck process detection failed"
|
||||
fi
|
||||
|
||||
# Test 5: Process group killing
|
||||
log_test "Process group killing (cleanup)"
|
||||
# Create a process that spawns children
|
||||
(
|
||||
echo "parent-$$" > "$TEST_DIR/parent.pid"
|
||||
(sleep 100 & echo $! > "$TEST_DIR/child.pid") &
|
||||
wait
|
||||
) &
|
||||
PARENT_PID=$!
|
||||
sleep 0.5
|
||||
|
||||
# Kill the process group
|
||||
if kill -0 $PARENT_PID 2>/dev/null; then
|
||||
kill -TERM -$PARENT_PID 2>/dev/null || kill -TERM $PARENT_PID 2>/dev/null || true
|
||||
sleep 0.5
|
||||
if ! kill -0 $PARENT_PID 2>/dev/null; then
|
||||
log_pass "Process group killed successfully"
|
||||
else
|
||||
kill -9 $PARENT_PID 2>/dev/null || true
|
||||
log_pass "Process killed with SIGKILL"
|
||||
fi
|
||||
else
|
||||
log_pass "Process already terminated"
|
||||
fi
|
||||
|
||||
# Test 6: npm/node process timeout simulation
|
||||
log_test "npm/node process timeout handling"
|
||||
cat > "$TEST_DIR/slow-script.js" << 'EOF'
|
||||
// Simulate a slow npm build
|
||||
console.log('Starting slow process...');
|
||||
setTimeout(() => {
|
||||
console.log('Still running...');
|
||||
}, 1000);
|
||||
setTimeout(() => {
|
||||
console.log('Completed!');
|
||||
process.exit(0);
|
||||
}, 5000);
|
||||
EOF
|
||||
|
||||
if command -v node &> /dev/null; then
|
||||
START=$(date +%s)
|
||||
run_with_timeout 2 "node '$TEST_DIR/slow-script.js'" > /dev/null 2>&1 && RESULT="success" || RESULT="timeout"
|
||||
END=$(date +%s)
|
||||
DURATION=$((END - START))
|
||||
|
||||
if [ "$RESULT" = "timeout" ]; then
|
||||
log_pass "Node process timed out correctly in ${DURATION}s"
|
||||
else
|
||||
log_fail "Node process should have timed out"
|
||||
fi
|
||||
else
|
||||
log_pass "Node not available - skipping (acceptable)"
|
||||
fi
|
||||
|
||||
# Test 7: Task retry after timeout
|
||||
log_test "Task retry after timeout"
|
||||
python3 << 'EOF'
|
||||
import json
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
# Task that timed out
|
||||
task = {
|
||||
"id": "task-timeout-001",
|
||||
"type": "eng-frontend",
|
||||
"payload": {"action": "build"},
|
||||
"timeout": 300,
|
||||
"retries": 0,
|
||||
"maxRetries": 3,
|
||||
"lastError": "Timeout after 300 seconds",
|
||||
"claimedBy": "agent-001",
|
||||
"claimedAt": (datetime.utcnow() - timedelta(seconds=310)).isoformat() + 'Z'
|
||||
}
|
||||
|
||||
def handle_timeout(task):
|
||||
task['retries'] += 1
|
||||
task['lastError'] = f"Timeout after {task['timeout']} seconds"
|
||||
task['claimedBy'] = None
|
||||
task['claimedAt'] = None
|
||||
|
||||
# Increase timeout for retry (25% increase)
|
||||
task['timeout'] = int(task['timeout'] * 1.25)
|
||||
|
||||
return task
|
||||
|
||||
task = handle_timeout(task)
|
||||
print(f"RETRIES:{task['retries']}")
|
||||
print(f"NEW_TIMEOUT:{task['timeout']}")
|
||||
assert task['retries'] == 1
|
||||
assert task['timeout'] == 375 # 300 * 1.25
|
||||
print("VERIFIED")
|
||||
EOF
|
||||
|
||||
if [ $? -eq 0 ]; then
|
||||
log_pass "Task retry after timeout works"
|
||||
else
|
||||
log_fail "Task retry after timeout failed"
|
||||
fi
|
||||
|
||||
# Test 8: Watchdog timer pattern
|
||||
log_test "Watchdog timer pattern"
|
||||
python3 << 'EOF'
|
||||
import time
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
class Watchdog:
|
||||
def __init__(self, timeout_seconds):
|
||||
self.timeout = timeout_seconds
|
||||
self.last_pet = datetime.utcnow()
|
||||
|
||||
def pet(self):
|
||||
"""Reset the watchdog timer"""
|
||||
self.last_pet = datetime.utcnow()
|
||||
|
||||
def is_expired(self):
|
||||
"""Check if watchdog has expired"""
|
||||
age = (datetime.utcnow() - self.last_pet).total_seconds()
|
||||
return age > self.timeout
|
||||
|
||||
def remaining(self):
|
||||
"""Get remaining time before expiry"""
|
||||
age = (datetime.utcnow() - self.last_pet).total_seconds()
|
||||
return max(0, self.timeout - age)
|
||||
|
||||
# Create watchdog with 2 second timeout
|
||||
wd = Watchdog(2)
|
||||
print(f"Initial remaining: {wd.remaining():.1f}s")
|
||||
assert not wd.is_expired(), "Should not be expired initially"
|
||||
|
||||
# Simulate work with petting
|
||||
time.sleep(0.5)
|
||||
wd.pet()
|
||||
print(f"After pet: {wd.remaining():.1f}s")
|
||||
assert not wd.is_expired(), "Should not be expired after pet"
|
||||
|
||||
# Let it expire
|
||||
time.sleep(0.1)
|
||||
# Simulate expiry by setting last_pet in past
|
||||
wd.last_pet = datetime.utcnow() - timedelta(seconds=3)
|
||||
assert wd.is_expired(), "Should be expired"
|
||||
print("Watchdog expired correctly")
|
||||
print("VERIFIED")
|
||||
EOF
|
||||
|
||||
if [ $? -eq 0 ]; then
|
||||
log_pass "Watchdog timer pattern works"
|
||||
else
|
||||
log_fail "Watchdog timer pattern failed"
|
||||
fi
|
||||
|
||||
# Test 9: Graceful shutdown with timeout
|
||||
log_test "Graceful shutdown with timeout"
|
||||
(
|
||||
trap 'echo "Received SIGTERM"; exit 0' TERM
|
||||
sleep 100
|
||||
) &
|
||||
PID=$!
|
||||
sleep 0.2
|
||||
|
||||
# Send SIGTERM
|
||||
kill -TERM $PID 2>/dev/null || true
|
||||
sleep 0.5
|
||||
|
||||
if ! kill -0 $PID 2>/dev/null; then
|
||||
log_pass "Process handled SIGTERM gracefully"
|
||||
else
|
||||
kill -9 $PID 2>/dev/null || true
|
||||
log_pass "Process required SIGKILL (acceptable)"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "========================================"
|
||||
echo "Test Summary"
|
||||
echo "========================================"
|
||||
echo -e "${GREEN}Passed: $PASSED${NC}"
|
||||
echo -e "${RED}Failed: $FAILED${NC}"
|
||||
echo ""
|
||||
|
||||
if [ $FAILED -eq 0 ]; then
|
||||
echo -e "${GREEN}All tests passed!${NC}"
|
||||
exit 0
|
||||
else
|
||||
echo -e "${RED}Some tests failed!${NC}"
|
||||
exit 1
|
||||
fi
|
||||
196
skills/loki-mode/tests/test-bootstrap.sh
Executable file
196
skills/loki-mode/tests/test-bootstrap.sh
Executable file
@@ -0,0 +1,196 @@
|
||||
#!/bin/bash
|
||||
# Test: Bootstrap Script Functionality
|
||||
# Tests the .loki directory initialization and state management
|
||||
|
||||
set -uo pipefail
|
||||
# Note: Not using -e to allow collecting all test results
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
TEST_DIR=$(mktemp -d)
|
||||
PASSED=0
|
||||
FAILED=0
|
||||
|
||||
# Colors
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
NC='\033[0m'
|
||||
|
||||
log_pass() { echo -e "${GREEN}[PASS]${NC} $1"; ((PASSED++)); }
|
||||
log_fail() { echo -e "${RED}[FAIL]${NC} $1"; ((FAILED++)); }
|
||||
log_test() { echo -e "${YELLOW}[TEST]${NC} $1"; }
|
||||
|
||||
cleanup() {
|
||||
rm -rf "$TEST_DIR"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
cd "$TEST_DIR"
|
||||
|
||||
echo "========================================"
|
||||
echo "Loki Mode Bootstrap Tests"
|
||||
echo "========================================"
|
||||
echo "Test directory: $TEST_DIR"
|
||||
echo ""
|
||||
|
||||
# Test 1: Directory structure creation
|
||||
log_test "Directory structure creation"
|
||||
mkdir -p .loki/{state/{agents,checkpoints,locks},queue,messages/{inbox,outbox,broadcast},logs/{agents,decisions,archive},config,prompts,artifacts/{releases,reports,backups},scripts,memory/{episodic,semantic,skills},metrics/{efficiency,rewards}}
|
||||
|
||||
if [ -d ".loki/state/agents" ] && [ -d ".loki/queue" ] && [ -d ".loki/logs" ]; then
|
||||
log_pass "All directories created"
|
||||
else
|
||||
log_fail "Missing directories"
|
||||
fi
|
||||
|
||||
# Test 2: Queue files initialization
|
||||
log_test "Queue files initialization"
|
||||
for f in pending in-progress completed failed dead-letter; do
|
||||
echo '{"tasks":[]}' > ".loki/queue/$f.json"
|
||||
done
|
||||
|
||||
all_queues_exist=true
|
||||
for f in pending in-progress completed failed dead-letter; do
|
||||
if [ ! -f ".loki/queue/$f.json" ]; then
|
||||
all_queues_exist=false
|
||||
fi
|
||||
done
|
||||
|
||||
if $all_queues_exist; then
|
||||
log_pass "All queue files created"
|
||||
else
|
||||
log_fail "Missing queue files"
|
||||
fi
|
||||
|
||||
# Test 3: Orchestrator state initialization
|
||||
log_test "Orchestrator state initialization"
|
||||
cat > .loki/state/orchestrator.json << 'EOF'
|
||||
{
|
||||
"version": "2.1.0",
|
||||
"startupId": "",
|
||||
"phase": "bootstrap",
|
||||
"prdPath": "",
|
||||
"prdHash": "",
|
||||
"agents": {"active":[],"idle":[],"failed":[],"totalSpawned":0},
|
||||
"metrics": {"tasksCompleted":0,"tasksFailed":0,"deployments":0},
|
||||
"circuitBreakers": {},
|
||||
"lastCheckpoint": "",
|
||||
"lastBackup": "",
|
||||
"currentRelease": "0.0.0"
|
||||
}
|
||||
EOF
|
||||
|
||||
if [ -f ".loki/state/orchestrator.json" ]; then
|
||||
version=$(cat .loki/state/orchestrator.json | grep -o '"version": "[^"]*"' | cut -d'"' -f4)
|
||||
if [ "$version" = "2.1.0" ]; then
|
||||
log_pass "Orchestrator state created with correct version"
|
||||
else
|
||||
log_fail "Orchestrator state has wrong version: $version"
|
||||
fi
|
||||
else
|
||||
log_fail "Orchestrator state file not created"
|
||||
fi
|
||||
|
||||
# Test 4: UUID generation (macOS compatible)
|
||||
log_test "UUID generation (macOS compatible)"
|
||||
if command -v uuidgen &> /dev/null; then
|
||||
STARTUP_ID=$(uuidgen)
|
||||
if [ -n "$STARTUP_ID" ]; then
|
||||
log_pass "UUID generated via uuidgen: $STARTUP_ID"
|
||||
else
|
||||
log_fail "uuidgen failed to generate UUID"
|
||||
fi
|
||||
elif [ -f /proc/sys/kernel/random/uuid ]; then
|
||||
STARTUP_ID=$(cat /proc/sys/kernel/random/uuid)
|
||||
if [ -n "$STARTUP_ID" ]; then
|
||||
log_pass "UUID generated via /proc: $STARTUP_ID"
|
||||
else
|
||||
log_fail "Failed to generate UUID from /proc"
|
||||
fi
|
||||
else
|
||||
STARTUP_ID="$(date +%s)-$$"
|
||||
log_pass "Fallback UUID generated: $STARTUP_ID"
|
||||
fi
|
||||
|
||||
# Test 5: sed macOS compatibility
|
||||
log_test "sed macOS compatibility"
|
||||
echo '{"startupId": ""}' > test_sed.json
|
||||
if [[ "$OSTYPE" == "darwin"* ]]; then
|
||||
sed -i '' 's/"startupId": ""/"startupId": "test-uuid"/' test_sed.json
|
||||
else
|
||||
sed -i 's/"startupId": ""/"startupId": "test-uuid"/' test_sed.json
|
||||
fi
|
||||
|
||||
if grep -q '"startupId": "test-uuid"' test_sed.json; then
|
||||
log_pass "sed works correctly on $OSTYPE"
|
||||
else
|
||||
log_fail "sed failed on $OSTYPE"
|
||||
fi
|
||||
|
||||
# Test 6: JSON validation
|
||||
log_test "JSON validation of queue files"
|
||||
json_valid=true
|
||||
for f in .loki/queue/*.json; do
|
||||
if ! python3 -c "import json; json.load(open('$f'))" 2>/dev/null; then
|
||||
if ! node -e "require('$f')" 2>/dev/null; then
|
||||
json_valid=false
|
||||
log_fail "Invalid JSON: $f"
|
||||
fi
|
||||
fi
|
||||
done
|
||||
if $json_valid; then
|
||||
log_pass "All queue JSON files are valid"
|
||||
fi
|
||||
|
||||
# Test 7: File locking mechanism
|
||||
log_test "File locking mechanism"
|
||||
mkdir -p .loki/state/locks
|
||||
LOCK_FILE=".loki/state/locks/test.lock"
|
||||
|
||||
# Test acquiring lock
|
||||
(
|
||||
exec 200>"$LOCK_FILE"
|
||||
if flock -x -w 1 200; then
|
||||
echo "locked" > "$LOCK_FILE.status"
|
||||
sleep 0.1
|
||||
fi
|
||||
) &
|
||||
LOCK_PID=$!
|
||||
sleep 0.2
|
||||
wait $LOCK_PID 2>/dev/null || true
|
||||
|
||||
if [ -f "$LOCK_FILE.status" ] && grep -q "locked" "$LOCK_FILE.status"; then
|
||||
log_pass "File locking works"
|
||||
else
|
||||
log_pass "File locking works (or flock not available - acceptable)"
|
||||
fi
|
||||
|
||||
# Test 8: Backup directory structure
|
||||
log_test "Backup directory structure"
|
||||
mkdir -p .loki/artifacts/backups
|
||||
TIMESTAMP=$(date +%Y%m%d-%H%M%S)
|
||||
BACKUP_PATH=".loki/artifacts/backups/state-$TIMESTAMP"
|
||||
mkdir -p "$BACKUP_PATH"
|
||||
cp .loki/state/orchestrator.json "$BACKUP_PATH/"
|
||||
|
||||
if [ -f "$BACKUP_PATH/orchestrator.json" ]; then
|
||||
log_pass "Backup structure works"
|
||||
else
|
||||
log_fail "Backup structure failed"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "========================================"
|
||||
echo "Test Summary"
|
||||
echo "========================================"
|
||||
echo -e "${GREEN}Passed: $PASSED${NC}"
|
||||
echo -e "${RED}Failed: $FAILED${NC}"
|
||||
echo ""
|
||||
|
||||
if [ $FAILED -eq 0 ]; then
|
||||
echo -e "${GREEN}All tests passed!${NC}"
|
||||
exit 0
|
||||
else
|
||||
echo -e "${RED}Some tests failed!${NC}"
|
||||
exit 1
|
||||
fi
|
||||
389
skills/loki-mode/tests/test-circuit-breaker.sh
Executable file
389
skills/loki-mode/tests/test-circuit-breaker.sh
Executable file
@@ -0,0 +1,389 @@
|
||||
#!/bin/bash
|
||||
# Test: Circuit Breaker Functionality
|
||||
# Tests circuit breaker states, transitions, and recovery
|
||||
|
||||
set -uo pipefail
|
||||
# Note: Not using -e to allow collecting all test results
|
||||
|
||||
TEST_DIR=$(mktemp -d)
|
||||
PASSED=0
|
||||
FAILED=0
|
||||
|
||||
# Colors
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
NC='\033[0m'
|
||||
|
||||
log_pass() { echo -e "${GREEN}[PASS]${NC} $1"; ((PASSED++)); }
|
||||
log_fail() { echo -e "${RED}[FAIL]${NC} $1"; ((FAILED++)); }
|
||||
log_test() { echo -e "${YELLOW}[TEST]${NC} $1"; }
|
||||
|
||||
cleanup() {
|
||||
rm -rf "$TEST_DIR"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
cd "$TEST_DIR"
|
||||
|
||||
echo "========================================"
|
||||
echo "Loki Mode Circuit Breaker Tests"
|
||||
echo "========================================"
|
||||
echo ""
|
||||
|
||||
# Initialize structure
|
||||
mkdir -p .loki/{state,config}
|
||||
|
||||
# Create circuit breaker config
|
||||
cat > .loki/config/circuit-breakers.yaml << 'EOF'
|
||||
defaults:
|
||||
failureThreshold: 5
|
||||
cooldownSeconds: 300
|
||||
halfOpenRequests: 3
|
||||
|
||||
overrides:
|
||||
external-api:
|
||||
failureThreshold: 3
|
||||
cooldownSeconds: 600
|
||||
eng-frontend:
|
||||
failureThreshold: 10
|
||||
cooldownSeconds: 180
|
||||
EOF
|
||||
|
||||
# Initialize orchestrator state
|
||||
cat > .loki/state/orchestrator.json << 'EOF'
|
||||
{
|
||||
"circuitBreakers": {}
|
||||
}
|
||||
EOF
|
||||
|
||||
# Test 1: Initialize circuit breaker (CLOSED state)
|
||||
log_test "Initialize circuit breaker in CLOSED state"
|
||||
python3 << 'EOF'
|
||||
import json
|
||||
from datetime import datetime
|
||||
|
||||
with open('.loki/state/orchestrator.json', 'r') as f:
|
||||
state = json.load(f)
|
||||
|
||||
# Initialize circuit breaker for eng-backend
|
||||
state['circuitBreakers']['eng-backend'] = {
|
||||
'state': 'closed',
|
||||
'failures': 0,
|
||||
'lastFailure': None,
|
||||
'cooldownUntil': None,
|
||||
'halfOpenAttempts': 0
|
||||
}
|
||||
|
||||
with open('.loki/state/orchestrator.json', 'w') as f:
|
||||
json.dump(state, f, indent=2)
|
||||
|
||||
print("INITIALIZED")
|
||||
EOF
|
||||
|
||||
cb_state=$(python3 -c "
|
||||
import json
|
||||
data = json.load(open('.loki/state/orchestrator.json'))
|
||||
print(data['circuitBreakers']['eng-backend']['state'])
|
||||
")
|
||||
|
||||
if [ "$cb_state" = "closed" ]; then
|
||||
log_pass "Circuit breaker initialized in CLOSED state"
|
||||
else
|
||||
log_fail "Expected CLOSED, got $cb_state"
|
||||
fi
|
||||
|
||||
# Test 2: Record failures
|
||||
log_test "Record failures incrementally"
|
||||
python3 << 'EOF'
|
||||
import json
|
||||
from datetime import datetime
|
||||
|
||||
with open('.loki/state/orchestrator.json', 'r') as f:
|
||||
state = json.load(f)
|
||||
|
||||
cb = state['circuitBreakers']['eng-backend']
|
||||
|
||||
# Record 3 failures
|
||||
for i in range(3):
|
||||
cb['failures'] += 1
|
||||
cb['lastFailure'] = datetime.utcnow().isoformat() + 'Z'
|
||||
|
||||
with open('.loki/state/orchestrator.json', 'w') as f:
|
||||
json.dump(state, f, indent=2)
|
||||
|
||||
print(f"FAILURES:{cb['failures']}")
|
||||
EOF
|
||||
|
||||
failures=$(python3 -c "
|
||||
import json
|
||||
data = json.load(open('.loki/state/orchestrator.json'))
|
||||
print(data['circuitBreakers']['eng-backend']['failures'])
|
||||
")
|
||||
|
||||
if [ "$failures" -eq 3 ]; then
|
||||
log_pass "Recorded 3 failures"
|
||||
else
|
||||
log_fail "Expected 3 failures, got $failures"
|
||||
fi
|
||||
|
||||
# Test 3: Trip circuit breaker (CLOSED -> OPEN)
|
||||
log_test "Trip circuit breaker after threshold"
|
||||
python3 << 'EOF'
|
||||
import json
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
FAILURE_THRESHOLD = 5
|
||||
COOLDOWN_SECONDS = 300
|
||||
|
||||
with open('.loki/state/orchestrator.json', 'r') as f:
|
||||
state = json.load(f)
|
||||
|
||||
cb = state['circuitBreakers']['eng-backend']
|
||||
|
||||
# Add 2 more failures to reach threshold
|
||||
cb['failures'] += 2
|
||||
cb['lastFailure'] = datetime.utcnow().isoformat() + 'Z'
|
||||
|
||||
# Check if threshold reached
|
||||
if cb['failures'] >= FAILURE_THRESHOLD:
|
||||
cb['state'] = 'open'
|
||||
cb['cooldownUntil'] = (datetime.utcnow() + timedelta(seconds=COOLDOWN_SECONDS)).isoformat() + 'Z'
|
||||
print(f"TRIPPED:open")
|
||||
else:
|
||||
print(f"NOT_TRIPPED:{cb['failures']}")
|
||||
|
||||
with open('.loki/state/orchestrator.json', 'w') as f:
|
||||
json.dump(state, f, indent=2)
|
||||
EOF
|
||||
|
||||
cb_state=$(python3 -c "
|
||||
import json
|
||||
data = json.load(open('.loki/state/orchestrator.json'))
|
||||
print(data['circuitBreakers']['eng-backend']['state'])
|
||||
")
|
||||
|
||||
if [ "$cb_state" = "open" ]; then
|
||||
log_pass "Circuit breaker tripped to OPEN"
|
||||
else
|
||||
log_fail "Expected OPEN, got $cb_state"
|
||||
fi
|
||||
|
||||
# Test 4: Block requests when OPEN
|
||||
log_test "Block requests when circuit is OPEN"
|
||||
python3 << 'EOF'
|
||||
import json
|
||||
from datetime import datetime
|
||||
|
||||
with open('.loki/state/orchestrator.json', 'r') as f:
|
||||
state = json.load(f)
|
||||
|
||||
cb = state['circuitBreakers']['eng-backend']
|
||||
|
||||
def can_proceed(circuit_breaker):
|
||||
if circuit_breaker['state'] == 'closed':
|
||||
return True
|
||||
if circuit_breaker['state'] == 'open':
|
||||
cooldown = circuit_breaker.get('cooldownUntil')
|
||||
if cooldown:
|
||||
# Check if cooldown expired
|
||||
cooldown_time = datetime.fromisoformat(cooldown.replace('Z', '+00:00'))
|
||||
if datetime.now(cooldown_time.tzinfo) > cooldown_time:
|
||||
return True # Can transition to half-open
|
||||
return False
|
||||
if circuit_breaker['state'] == 'half-open':
|
||||
return True
|
||||
return False
|
||||
|
||||
result = can_proceed(cb)
|
||||
print("BLOCKED" if not result else "ALLOWED")
|
||||
EOF
|
||||
|
||||
log_pass "Requests blocked when circuit is OPEN"
|
||||
|
||||
# Test 5: Transition to HALF-OPEN after cooldown
|
||||
log_test "Transition to HALF-OPEN after cooldown"
|
||||
python3 << 'EOF'
|
||||
import json
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
with open('.loki/state/orchestrator.json', 'r') as f:
|
||||
state = json.load(f)
|
||||
|
||||
cb = state['circuitBreakers']['eng-backend']
|
||||
|
||||
# Simulate cooldown expired
|
||||
cb['cooldownUntil'] = (datetime.utcnow() - timedelta(seconds=10)).isoformat() + 'Z'
|
||||
|
||||
# Check and transition
|
||||
cooldown_time = datetime.fromisoformat(cb['cooldownUntil'].replace('Z', '+00:00'))
|
||||
if datetime.now(cooldown_time.tzinfo) > cooldown_time and cb['state'] == 'open':
|
||||
cb['state'] = 'half-open'
|
||||
cb['halfOpenAttempts'] = 0
|
||||
print("TRANSITIONED:half-open")
|
||||
|
||||
with open('.loki/state/orchestrator.json', 'w') as f:
|
||||
json.dump(state, f, indent=2)
|
||||
EOF
|
||||
|
||||
cb_state=$(python3 -c "
|
||||
import json
|
||||
data = json.load(open('.loki/state/orchestrator.json'))
|
||||
print(data['circuitBreakers']['eng-backend']['state'])
|
||||
")
|
||||
|
||||
if [ "$cb_state" = "half-open" ]; then
|
||||
log_pass "Circuit breaker transitioned to HALF-OPEN"
|
||||
else
|
||||
log_fail "Expected HALF-OPEN, got $cb_state"
|
||||
fi
|
||||
|
||||
# Test 6: Success in HALF-OPEN -> CLOSED
|
||||
log_test "Success in HALF-OPEN transitions to CLOSED"
|
||||
python3 << 'EOF'
|
||||
import json
|
||||
|
||||
HALF_OPEN_REQUESTS = 3
|
||||
|
||||
with open('.loki/state/orchestrator.json', 'r') as f:
|
||||
state = json.load(f)
|
||||
|
||||
cb = state['circuitBreakers']['eng-backend']
|
||||
|
||||
# Simulate successful requests in half-open
|
||||
for i in range(HALF_OPEN_REQUESTS):
|
||||
cb['halfOpenAttempts'] += 1
|
||||
|
||||
# After enough successes, transition to closed
|
||||
if cb['halfOpenAttempts'] >= HALF_OPEN_REQUESTS:
|
||||
cb['state'] = 'closed'
|
||||
cb['failures'] = 0
|
||||
cb['lastFailure'] = None
|
||||
cb['cooldownUntil'] = None
|
||||
cb['halfOpenAttempts'] = 0
|
||||
print("RECOVERED:closed")
|
||||
|
||||
with open('.loki/state/orchestrator.json', 'w') as f:
|
||||
json.dump(state, f, indent=2)
|
||||
EOF
|
||||
|
||||
cb_state=$(python3 -c "
|
||||
import json
|
||||
data = json.load(open('.loki/state/orchestrator.json'))
|
||||
print(data['circuitBreakers']['eng-backend']['state'])
|
||||
")
|
||||
|
||||
if [ "$cb_state" = "closed" ]; then
|
||||
log_pass "Circuit breaker recovered to CLOSED"
|
||||
else
|
||||
log_fail "Expected CLOSED, got $cb_state"
|
||||
fi
|
||||
|
||||
# Test 7: Failure in HALF-OPEN -> OPEN
|
||||
log_test "Failure in HALF-OPEN transitions back to OPEN"
|
||||
python3 << 'EOF'
|
||||
import json
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
COOLDOWN_SECONDS = 300
|
||||
|
||||
with open('.loki/state/orchestrator.json', 'r') as f:
|
||||
state = json.load(f)
|
||||
|
||||
cb = state['circuitBreakers']['eng-backend']
|
||||
|
||||
# Set to half-open
|
||||
cb['state'] = 'half-open'
|
||||
cb['halfOpenAttempts'] = 1
|
||||
|
||||
# Simulate failure
|
||||
cb['state'] = 'open'
|
||||
cb['failures'] += 1
|
||||
cb['lastFailure'] = datetime.utcnow().isoformat() + 'Z'
|
||||
cb['cooldownUntil'] = (datetime.utcnow() + timedelta(seconds=COOLDOWN_SECONDS)).isoformat() + 'Z'
|
||||
cb['halfOpenAttempts'] = 0
|
||||
|
||||
print("REOPENED")
|
||||
|
||||
with open('.loki/state/orchestrator.json', 'w') as f:
|
||||
json.dump(state, f, indent=2)
|
||||
EOF
|
||||
|
||||
cb_state=$(python3 -c "
|
||||
import json
|
||||
data = json.load(open('.loki/state/orchestrator.json'))
|
||||
print(data['circuitBreakers']['eng-backend']['state'])
|
||||
")
|
||||
|
||||
if [ "$cb_state" = "open" ]; then
|
||||
log_pass "Circuit breaker reopened after HALF-OPEN failure"
|
||||
else
|
||||
log_fail "Expected OPEN, got $cb_state"
|
||||
fi
|
||||
|
||||
# Test 8: Per-agent-type thresholds
|
||||
log_test "Per-agent-type thresholds from config"
|
||||
python3 << 'EOF'
|
||||
import json
|
||||
|
||||
# Simulate reading config (in real usage, would parse YAML)
|
||||
config = {
|
||||
'defaults': {
|
||||
'failureThreshold': 5,
|
||||
'cooldownSeconds': 300
|
||||
},
|
||||
'overrides': {
|
||||
'external-api': {
|
||||
'failureThreshold': 3,
|
||||
'cooldownSeconds': 600
|
||||
},
|
||||
'eng-frontend': {
|
||||
'failureThreshold': 10,
|
||||
'cooldownSeconds': 180
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def get_threshold(agent_type):
|
||||
if agent_type in config['overrides']:
|
||||
return config['overrides'][agent_type].get('failureThreshold', config['defaults']['failureThreshold'])
|
||||
return config['defaults']['failureThreshold']
|
||||
|
||||
# Test different agent types
|
||||
backend_threshold = get_threshold('eng-backend') # Should use default
|
||||
frontend_threshold = get_threshold('eng-frontend') # Should use override
|
||||
api_threshold = get_threshold('external-api') # Should use override
|
||||
|
||||
results = {
|
||||
'eng-backend': backend_threshold,
|
||||
'eng-frontend': frontend_threshold,
|
||||
'external-api': api_threshold
|
||||
}
|
||||
|
||||
print(f"THRESHOLDS:backend={backend_threshold},frontend={frontend_threshold},api={api_threshold}")
|
||||
|
||||
# Verify
|
||||
assert backend_threshold == 5, f"Expected 5, got {backend_threshold}"
|
||||
assert frontend_threshold == 10, f"Expected 10, got {frontend_threshold}"
|
||||
assert api_threshold == 3, f"Expected 3, got {api_threshold}"
|
||||
|
||||
print("VERIFIED")
|
||||
EOF
|
||||
|
||||
log_pass "Per-agent-type thresholds work correctly"
|
||||
|
||||
echo ""
|
||||
echo "========================================"
|
||||
echo "Test Summary"
|
||||
echo "========================================"
|
||||
echo -e "${GREEN}Passed: $PASSED${NC}"
|
||||
echo -e "${RED}Failed: $FAILED${NC}"
|
||||
echo ""
|
||||
|
||||
if [ $FAILED -eq 0 ]; then
|
||||
echo -e "${GREEN}All tests passed!${NC}"
|
||||
exit 0
|
||||
else
|
||||
echo -e "${RED}Some tests failed!${NC}"
|
||||
exit 1
|
||||
fi
|
||||
393
skills/loki-mode/tests/test-state-recovery.sh
Executable file
393
skills/loki-mode/tests/test-state-recovery.sh
Executable file
@@ -0,0 +1,393 @@
|
||||
#!/bin/bash
|
||||
# Test: State Recovery and Checkpoint Functionality
|
||||
# Tests checkpoint creation, recovery, and rate limit handling
|
||||
|
||||
set -uo pipefail
|
||||
# Note: Not using -e to allow collecting all test results
|
||||
|
||||
TEST_DIR=$(mktemp -d)
|
||||
PASSED=0
|
||||
FAILED=0
|
||||
|
||||
# Colors
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
NC='\033[0m'
|
||||
|
||||
log_pass() { echo -e "${GREEN}[PASS]${NC} $1"; ((PASSED++)); }
|
||||
log_fail() { echo -e "${RED}[FAIL]${NC} $1"; ((FAILED++)); }
|
||||
log_test() { echo -e "${YELLOW}[TEST]${NC} $1"; }
|
||||
|
||||
cleanup() {
|
||||
rm -rf "$TEST_DIR"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
cd "$TEST_DIR"
|
||||
|
||||
echo "========================================"
|
||||
echo "Loki Mode State Recovery Tests"
|
||||
echo "========================================"
|
||||
echo ""
|
||||
|
||||
# Initialize structure
|
||||
mkdir -p .loki/{state/{agents,checkpoints},queue,artifacts/backups}
|
||||
|
||||
# Create initial state
|
||||
cat > .loki/state/orchestrator.json << 'EOF'
|
||||
{
|
||||
"version": "2.1.0",
|
||||
"startupId": "test-session-001",
|
||||
"phase": "development",
|
||||
"agents": {"active":["eng-backend-01"],"idle":[],"failed":[],"totalSpawned":5},
|
||||
"metrics": {"tasksCompleted":10,"tasksFailed":2,"deployments":0},
|
||||
"circuitBreakers": {},
|
||||
"lastCheckpoint": "",
|
||||
"currentRelease": "0.1.0"
|
||||
}
|
||||
EOF
|
||||
|
||||
# Create agent state
|
||||
cat > .loki/state/agents/eng-backend-01.json << 'EOF'
|
||||
{
|
||||
"id": "eng-backend-01",
|
||||
"status": "active",
|
||||
"currentTask": "task-042",
|
||||
"tasksCompleted": 8,
|
||||
"lastHeartbeat": "2025-01-15T10:30:00Z"
|
||||
}
|
||||
EOF
|
||||
|
||||
# Create queue state
|
||||
cat > .loki/queue/pending.json << 'EOF'
|
||||
{"tasks":[{"id":"task-043","type":"eng-frontend","priority":5}]}
|
||||
EOF
|
||||
cat > .loki/queue/in-progress.json << 'EOF'
|
||||
{"tasks":[{"id":"task-042","type":"eng-backend","claimedBy":"eng-backend-01"}]}
|
||||
EOF
|
||||
|
||||
# Test 1: Create checkpoint
|
||||
log_test "Create checkpoint"
|
||||
CHECKPOINT_DIR=".loki/state/checkpoints/$(date +%Y%m%d-%H%M%S)"
|
||||
mkdir -p "$CHECKPOINT_DIR"
|
||||
cp .loki/state/orchestrator.json "$CHECKPOINT_DIR/"
|
||||
cp -r .loki/state/agents "$CHECKPOINT_DIR/"
|
||||
cp -r .loki/queue "$CHECKPOINT_DIR/"
|
||||
|
||||
if [ -f "$CHECKPOINT_DIR/orchestrator.json" ] && [ -d "$CHECKPOINT_DIR/agents" ]; then
|
||||
log_pass "Checkpoint created at $CHECKPOINT_DIR"
|
||||
else
|
||||
log_fail "Checkpoint creation failed"
|
||||
fi
|
||||
|
||||
# Test 2: Update lastCheckpoint in state
|
||||
log_test "Update lastCheckpoint timestamp"
|
||||
python3 << EOF
|
||||
import json
|
||||
from datetime import datetime
|
||||
|
||||
with open('.loki/state/orchestrator.json', 'r') as f:
|
||||
state = json.load(f)
|
||||
|
||||
state['lastCheckpoint'] = datetime.utcnow().isoformat() + 'Z'
|
||||
|
||||
with open('.loki/state/orchestrator.json', 'w') as f:
|
||||
json.dump(state, f, indent=2)
|
||||
|
||||
print("UPDATED")
|
||||
EOF
|
||||
|
||||
has_checkpoint=$(python3 -c "
|
||||
import json
|
||||
data = json.load(open('.loki/state/orchestrator.json'))
|
||||
print('yes' if data.get('lastCheckpoint') else 'no')
|
||||
")
|
||||
|
||||
if [ "$has_checkpoint" = "yes" ]; then
|
||||
log_pass "lastCheckpoint timestamp updated"
|
||||
else
|
||||
log_fail "lastCheckpoint not set"
|
||||
fi
|
||||
|
||||
# Test 3: Simulate crash and corrupt state
|
||||
log_test "Detect corrupted state"
|
||||
echo "corrupted{json" > .loki/state/orchestrator.json.corrupted
|
||||
|
||||
python3 << 'EOF'
|
||||
import json
|
||||
|
||||
def is_valid_state(filepath):
|
||||
try:
|
||||
with open(filepath, 'r') as f:
|
||||
data = json.load(f)
|
||||
return isinstance(data, dict) and 'version' in data
|
||||
except (json.JSONDecodeError, KeyError):
|
||||
return False
|
||||
|
||||
is_valid = is_valid_state('.loki/state/orchestrator.json.corrupted')
|
||||
print("CORRUPTED" if not is_valid else "VALID")
|
||||
assert not is_valid, "Should detect corrupted state"
|
||||
EOF
|
||||
|
||||
log_pass "Corrupted state detected"
|
||||
|
||||
# Test 4: Restore from checkpoint
|
||||
log_test "Restore from checkpoint"
|
||||
python3 << EOF
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
|
||||
# Find latest checkpoint
|
||||
checkpoints_dir = Path('.loki/state/checkpoints')
|
||||
checkpoints = sorted(checkpoints_dir.iterdir(), reverse=True)
|
||||
|
||||
if checkpoints:
|
||||
latest = checkpoints[0]
|
||||
|
||||
# Restore orchestrator state
|
||||
if (latest / 'orchestrator.json').exists():
|
||||
shutil.copy(latest / 'orchestrator.json', '.loki/state/orchestrator.json')
|
||||
|
||||
# Restore agent states
|
||||
if (latest / 'agents').exists():
|
||||
for agent_file in (latest / 'agents').iterdir():
|
||||
shutil.copy(agent_file, f'.loki/state/agents/{agent_file.name}')
|
||||
|
||||
# Restore queue
|
||||
if (latest / 'queue').exists():
|
||||
for queue_file in (latest / 'queue').iterdir():
|
||||
shutil.copy(queue_file, f'.loki/queue/{queue_file.name}')
|
||||
|
||||
print(f"RESTORED:{latest.name}")
|
||||
else:
|
||||
print("NO_CHECKPOINT")
|
||||
EOF
|
||||
|
||||
# Verify restoration
|
||||
restored_version=$(python3 -c "
|
||||
import json
|
||||
data = json.load(open('.loki/state/orchestrator.json'))
|
||||
print(data.get('version', 'unknown'))
|
||||
")
|
||||
|
||||
if [ "$restored_version" = "2.1.0" ]; then
|
||||
log_pass "State restored from checkpoint"
|
||||
else
|
||||
log_fail "State restoration failed (version: $restored_version)"
|
||||
fi
|
||||
|
||||
# Test 5: Orphaned task detection
|
||||
log_test "Detect orphaned tasks"
|
||||
python3 << 'EOF'
|
||||
import json
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
CLAIM_TIMEOUT = 3600 # 1 hour
|
||||
|
||||
# Create an old claimed task
|
||||
old_task = {
|
||||
"id": "task-old-001",
|
||||
"type": "eng-backend",
|
||||
"claimedBy": "dead-agent-99",
|
||||
"claimedAt": (datetime.utcnow() - timedelta(hours=2)).isoformat() + 'Z'
|
||||
}
|
||||
|
||||
with open('.loki/queue/in-progress.json', 'r') as f:
|
||||
in_progress = json.load(f)
|
||||
|
||||
in_progress['tasks'].append(old_task)
|
||||
|
||||
with open('.loki/queue/in-progress.json', 'w') as f:
|
||||
json.dump(in_progress, f)
|
||||
|
||||
def find_orphaned_tasks(in_progress_tasks):
|
||||
orphaned = []
|
||||
now = datetime.utcnow()
|
||||
|
||||
for task in in_progress_tasks:
|
||||
if task.get('claimedAt'):
|
||||
claimed_at = datetime.fromisoformat(task['claimedAt'].replace('Z', '+00:00'))
|
||||
age = (now.replace(tzinfo=claimed_at.tzinfo) - claimed_at).total_seconds()
|
||||
if age > CLAIM_TIMEOUT:
|
||||
orphaned.append(task['id'])
|
||||
|
||||
return orphaned
|
||||
|
||||
orphaned = find_orphaned_tasks(in_progress['tasks'])
|
||||
print(f"ORPHANED:{len(orphaned)}")
|
||||
assert len(orphaned) >= 1, "Should find orphaned task"
|
||||
print("VERIFIED")
|
||||
EOF
|
||||
|
||||
log_pass "Orphaned task detection works"
|
||||
|
||||
# Test 6: Re-queue orphaned tasks
|
||||
log_test "Re-queue orphaned tasks"
|
||||
python3 << 'EOF'
|
||||
import json
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
CLAIM_TIMEOUT = 3600
|
||||
|
||||
with open('.loki/queue/in-progress.json', 'r') as f:
|
||||
in_progress = json.load(f)
|
||||
|
||||
with open('.loki/queue/pending.json', 'r') as f:
|
||||
pending = json.load(f)
|
||||
|
||||
now = datetime.utcnow()
|
||||
requeued = []
|
||||
|
||||
for task in in_progress['tasks'][:]:
|
||||
if task.get('claimedAt'):
|
||||
claimed_at = datetime.fromisoformat(task['claimedAt'].replace('Z', '+00:00'))
|
||||
age = (now.replace(tzinfo=claimed_at.tzinfo) - claimed_at).total_seconds()
|
||||
|
||||
if age > CLAIM_TIMEOUT:
|
||||
# Re-queue: clear claim and move to pending
|
||||
task['claimedBy'] = None
|
||||
task['claimedAt'] = None
|
||||
task['requeuedAt'] = now.isoformat() + 'Z'
|
||||
task['requeueReason'] = 'claim_timeout'
|
||||
|
||||
pending['tasks'].append(task)
|
||||
in_progress['tasks'].remove(task)
|
||||
requeued.append(task['id'])
|
||||
|
||||
with open('.loki/queue/in-progress.json', 'w') as f:
|
||||
json.dump(in_progress, f)
|
||||
|
||||
with open('.loki/queue/pending.json', 'w') as f:
|
||||
json.dump(pending, f)
|
||||
|
||||
print(f"REQUEUED:{len(requeued)}")
|
||||
EOF
|
||||
|
||||
log_pass "Orphaned tasks re-queued"
|
||||
|
||||
# Test 7: Rate limit backoff simulation
|
||||
log_test "Rate limit exponential backoff"
|
||||
python3 << 'EOF'
|
||||
import time
|
||||
import random
|
||||
|
||||
def calculate_backoff(attempt, base_delay=60, max_delay=3600):
|
||||
"""Calculate exponential backoff with jitter"""
|
||||
delay = min(base_delay * (2 ** attempt), max_delay)
|
||||
jitter = random.uniform(0, delay * 0.1)
|
||||
return delay + jitter
|
||||
|
||||
# Test backoff progression
|
||||
delays = []
|
||||
for attempt in range(5):
|
||||
delay = calculate_backoff(attempt)
|
||||
delays.append(int(delay))
|
||||
print(f"Attempt {attempt}: {delay:.0f}s")
|
||||
|
||||
# Verify exponential growth
|
||||
assert delays[0] >= 60, "Initial delay should be ~60s"
|
||||
assert delays[1] >= 120, "Second delay should be ~120s"
|
||||
assert delays[2] >= 240, "Third delay should be ~240s"
|
||||
assert delays[4] <= 4000, "Should cap at max_delay"
|
||||
|
||||
print("VERIFIED")
|
||||
EOF
|
||||
|
||||
log_pass "Exponential backoff works"
|
||||
|
||||
# Test 8: Full system recovery
|
||||
log_test "Full system recovery simulation"
|
||||
python3 << 'EOF'
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
def recover_system():
|
||||
"""Full system recovery procedure"""
|
||||
recovery_log = []
|
||||
|
||||
# 1. Check orchestrator state
|
||||
try:
|
||||
with open('.loki/state/orchestrator.json', 'r') as f:
|
||||
state = json.load(f)
|
||||
recovery_log.append("Orchestrator state: OK")
|
||||
except:
|
||||
recovery_log.append("Orchestrator state: RESTORE FROM CHECKPOINT")
|
||||
# Would restore here
|
||||
|
||||
# 2. Check agent states
|
||||
agents_dir = Path('.loki/state/agents')
|
||||
active_agents = []
|
||||
dead_agents = []
|
||||
|
||||
for agent_file in agents_dir.glob('*.json'):
|
||||
with open(agent_file, 'r') as f:
|
||||
agent = json.load(f)
|
||||
|
||||
# Check heartbeat
|
||||
if agent.get('lastHeartbeat'):
|
||||
hb = datetime.fromisoformat(agent['lastHeartbeat'].replace('Z', '+00:00'))
|
||||
age = (datetime.now(hb.tzinfo) - hb).total_seconds()
|
||||
if age > 600: # 10 min heartbeat timeout
|
||||
dead_agents.append(agent['id'])
|
||||
else:
|
||||
active_agents.append(agent['id'])
|
||||
|
||||
recovery_log.append(f"Active agents: {len(active_agents)}")
|
||||
recovery_log.append(f"Dead agents: {len(dead_agents)}")
|
||||
|
||||
# 3. Re-queue tasks from dead agents
|
||||
with open('.loki/queue/in-progress.json', 'r') as f:
|
||||
in_progress = json.load(f)
|
||||
|
||||
requeued = 0
|
||||
for task in in_progress['tasks'][:]:
|
||||
if task.get('claimedBy') in dead_agents:
|
||||
task['claimedBy'] = None
|
||||
task['claimedAt'] = None
|
||||
requeued += 1
|
||||
|
||||
with open('.loki/queue/in-progress.json', 'w') as f:
|
||||
json.dump(in_progress, f)
|
||||
|
||||
recovery_log.append(f"Re-queued tasks: {requeued}")
|
||||
|
||||
# 4. Reset circuit breakers if cooldown expired
|
||||
if 'circuitBreakers' in state:
|
||||
for cb_name, cb in state['circuitBreakers'].items():
|
||||
if cb.get('state') == 'open' and cb.get('cooldownUntil'):
|
||||
cooldown = datetime.fromisoformat(cb['cooldownUntil'].replace('Z', '+00:00'))
|
||||
if datetime.now(cooldown.tzinfo) > cooldown:
|
||||
cb['state'] = 'half-open'
|
||||
recovery_log.append(f"Circuit breaker {cb_name}: OPEN -> HALF-OPEN")
|
||||
|
||||
return recovery_log
|
||||
|
||||
log = recover_system()
|
||||
for entry in log:
|
||||
print(entry)
|
||||
|
||||
print("RECOVERY_COMPLETE")
|
||||
EOF
|
||||
|
||||
log_pass "Full system recovery works"
|
||||
|
||||
echo ""
|
||||
echo "========================================"
|
||||
echo "Test Summary"
|
||||
echo "========================================"
|
||||
echo -e "${GREEN}Passed: $PASSED${NC}"
|
||||
echo -e "${RED}Failed: $FAILED${NC}"
|
||||
echo ""
|
||||
|
||||
if [ $FAILED -eq 0 ]; then
|
||||
echo -e "${GREEN}All tests passed!${NC}"
|
||||
exit 0
|
||||
else
|
||||
echo -e "${RED}Some tests failed!${NC}"
|
||||
exit 1
|
||||
fi
|
||||
396
skills/loki-mode/tests/test-task-queue.sh
Executable file
396
skills/loki-mode/tests/test-task-queue.sh
Executable file
@@ -0,0 +1,396 @@
|
||||
#!/bin/bash
|
||||
# Test: Distributed Task Queue Functionality
|
||||
# Tests task creation, claiming, completion, and failure handling
|
||||
|
||||
set -uo pipefail
|
||||
# Note: Not using -e to allow collecting all test results
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
TEST_DIR=$(mktemp -d)
|
||||
PASSED=0
|
||||
FAILED=0
|
||||
|
||||
# Colors
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
NC='\033[0m'
|
||||
|
||||
log_pass() { echo -e "${GREEN}[PASS]${NC} $1"; ((PASSED++)); }
|
||||
log_fail() { echo -e "${RED}[FAIL]${NC} $1"; ((FAILED++)); }
|
||||
log_test() { echo -e "${YELLOW}[TEST]${NC} $1"; }
|
||||
|
||||
cleanup() {
|
||||
rm -rf "$TEST_DIR"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
cd "$TEST_DIR"
|
||||
|
||||
echo "========================================"
|
||||
echo "Loki Mode Task Queue Tests"
|
||||
echo "========================================"
|
||||
echo ""
|
||||
|
||||
# Initialize structure
|
||||
mkdir -p .loki/{state/locks,queue}
|
||||
for f in pending in-progress completed failed dead-letter; do
|
||||
echo '{"tasks":[]}' > ".loki/queue/$f.json"
|
||||
done
|
||||
|
||||
# Helper function to add task
|
||||
add_task() {
|
||||
local id="$1"
|
||||
local type="$2"
|
||||
local priority="${3:-5}"
|
||||
|
||||
local task=$(cat <<EOF
|
||||
{
|
||||
"id": "$id",
|
||||
"type": "$type",
|
||||
"priority": $priority,
|
||||
"payload": {"action": "test"},
|
||||
"createdAt": "$(date -u +%Y-%m-%dT%H:%M:%SZ)",
|
||||
"claimedBy": null,
|
||||
"claimedAt": null,
|
||||
"timeout": 3600,
|
||||
"retries": 0,
|
||||
"maxRetries": 3
|
||||
}
|
||||
EOF
|
||||
)
|
||||
|
||||
# Add to pending queue
|
||||
if command -v jq &> /dev/null; then
|
||||
jq --argjson task "$task" '.tasks += [$task]' .loki/queue/pending.json > tmp.json && mv tmp.json .loki/queue/pending.json
|
||||
else
|
||||
# Fallback without jq
|
||||
python3 -c "
|
||||
import json
|
||||
with open('.loki/queue/pending.json', 'r') as f:
|
||||
data = json.load(f)
|
||||
task = json.loads('''$task''')
|
||||
data['tasks'].append(task)
|
||||
with open('.loki/queue/pending.json', 'w') as f:
|
||||
json.dump(data, f)
|
||||
"
|
||||
fi
|
||||
}
|
||||
|
||||
# Test 1: Add task to pending queue
|
||||
log_test "Add task to pending queue"
|
||||
add_task "task-001" "eng-backend" 5
|
||||
|
||||
task_count=$(python3 -c "import json; print(len(json.load(open('.loki/queue/pending.json'))['tasks']))")
|
||||
if [ "$task_count" -eq 1 ]; then
|
||||
log_pass "Task added to pending queue"
|
||||
else
|
||||
log_fail "Failed to add task (count: $task_count)"
|
||||
fi
|
||||
|
||||
# Test 2: Add multiple tasks with priorities
|
||||
log_test "Add multiple tasks with priorities"
|
||||
add_task "task-002" "eng-frontend" 3
|
||||
add_task "task-003" "eng-backend" 10
|
||||
add_task "task-004" "ops-devops" 1
|
||||
|
||||
task_count=$(python3 -c "import json; print(len(json.load(open('.loki/queue/pending.json'))['tasks']))")
|
||||
if [ "$task_count" -eq 4 ]; then
|
||||
log_pass "Multiple tasks added"
|
||||
else
|
||||
log_fail "Failed to add multiple tasks (count: $task_count)"
|
||||
fi
|
||||
|
||||
# Test 3: Priority ordering
|
||||
log_test "Priority ordering"
|
||||
highest_priority=$(python3 -c "
|
||||
import json
|
||||
data = json.load(open('.loki/queue/pending.json'))
|
||||
sorted_tasks = sorted(data['tasks'], key=lambda t: -t['priority'])
|
||||
print(sorted_tasks[0]['id'])
|
||||
")
|
||||
|
||||
if [ "$highest_priority" = "task-003" ]; then
|
||||
log_pass "Highest priority task is task-003 (priority 10)"
|
||||
else
|
||||
log_fail "Priority ordering wrong: got $highest_priority, expected task-003"
|
||||
fi
|
||||
|
||||
# Test 4: Claim task (atomic operation simulation)
|
||||
log_test "Claim task atomically"
|
||||
python3 << 'EOF'
|
||||
import json
|
||||
import os
|
||||
from datetime import datetime
|
||||
|
||||
# Simulate atomic claim with file locking
|
||||
queue_file = '.loki/queue/pending.json'
|
||||
progress_file = '.loki/queue/in-progress.json'
|
||||
lock_file = '.loki/state/locks/queue.lock'
|
||||
|
||||
# Read pending
|
||||
with open(queue_file, 'r') as f:
|
||||
pending = json.load(f)
|
||||
|
||||
# Find highest priority unclaimed task
|
||||
tasks = sorted(pending['tasks'], key=lambda t: -t['priority'])
|
||||
claimed_task = None
|
||||
for task in tasks:
|
||||
if task.get('claimedBy') is None:
|
||||
task['claimedBy'] = 'agent-001'
|
||||
task['claimedAt'] = datetime.utcnow().isoformat() + 'Z'
|
||||
claimed_task = task
|
||||
break
|
||||
|
||||
if claimed_task:
|
||||
# Remove from pending
|
||||
pending['tasks'] = [t for t in pending['tasks'] if t['id'] != claimed_task['id']]
|
||||
|
||||
# Add to in-progress
|
||||
with open(progress_file, 'r') as f:
|
||||
progress = json.load(f)
|
||||
progress['tasks'].append(claimed_task)
|
||||
|
||||
# Write both files
|
||||
with open(queue_file, 'w') as f:
|
||||
json.dump(pending, f)
|
||||
with open(progress_file, 'w') as f:
|
||||
json.dump(progress, f)
|
||||
|
||||
print(f"CLAIMED:{claimed_task['id']}")
|
||||
else:
|
||||
print("NONE")
|
||||
EOF
|
||||
|
||||
claimed=$(python3 -c "
|
||||
import json
|
||||
data = json.load(open('.loki/queue/in-progress.json'))
|
||||
if data['tasks']:
|
||||
print(data['tasks'][0]['id'])
|
||||
else:
|
||||
print('NONE')
|
||||
")
|
||||
|
||||
if [ "$claimed" = "task-003" ]; then
|
||||
log_pass "Claimed highest priority task (task-003)"
|
||||
else
|
||||
log_fail "Claim failed: got $claimed"
|
||||
fi
|
||||
|
||||
# Test 5: Complete task
|
||||
log_test "Complete task"
|
||||
python3 << 'EOF'
|
||||
import json
|
||||
from datetime import datetime
|
||||
|
||||
progress_file = '.loki/queue/in-progress.json'
|
||||
completed_file = '.loki/queue/completed.json'
|
||||
|
||||
with open(progress_file, 'r') as f:
|
||||
progress = json.load(f)
|
||||
|
||||
with open(completed_file, 'r') as f:
|
||||
completed = json.load(f)
|
||||
|
||||
# Complete first task
|
||||
if progress['tasks']:
|
||||
task = progress['tasks'][0]
|
||||
task['completedAt'] = datetime.utcnow().isoformat() + 'Z'
|
||||
task['result'] = {'status': 'success'}
|
||||
|
||||
completed['tasks'].append(task)
|
||||
progress['tasks'] = progress['tasks'][1:]
|
||||
|
||||
with open(progress_file, 'w') as f:
|
||||
json.dump(progress, f)
|
||||
with open(completed_file, 'w') as f:
|
||||
json.dump(completed, f)
|
||||
|
||||
print("COMPLETED")
|
||||
EOF
|
||||
|
||||
completed_count=$(python3 -c "import json; print(len(json.load(open('.loki/queue/completed.json'))['tasks']))")
|
||||
if [ "$completed_count" -eq 1 ]; then
|
||||
log_pass "Task completed successfully"
|
||||
else
|
||||
log_fail "Task completion failed"
|
||||
fi
|
||||
|
||||
# Test 6: Fail task with retry
|
||||
log_test "Fail task with retry"
|
||||
# First claim a task
|
||||
python3 << 'EOF'
|
||||
import json
|
||||
from datetime import datetime
|
||||
|
||||
queue_file = '.loki/queue/pending.json'
|
||||
progress_file = '.loki/queue/in-progress.json'
|
||||
|
||||
with open(queue_file, 'r') as f:
|
||||
pending = json.load(f)
|
||||
|
||||
if pending['tasks']:
|
||||
task = pending['tasks'][0]
|
||||
task['claimedBy'] = 'agent-002'
|
||||
task['claimedAt'] = datetime.utcnow().isoformat() + 'Z'
|
||||
|
||||
with open(progress_file, 'r') as f:
|
||||
progress = json.load(f)
|
||||
|
||||
progress['tasks'].append(task)
|
||||
pending['tasks'] = pending['tasks'][1:]
|
||||
|
||||
with open(queue_file, 'w') as f:
|
||||
json.dump(pending, f)
|
||||
with open(progress_file, 'w') as f:
|
||||
json.dump(progress, f)
|
||||
EOF
|
||||
|
||||
# Now fail it
|
||||
python3 << 'EOF'
|
||||
import json
|
||||
from datetime import datetime
|
||||
|
||||
progress_file = '.loki/queue/in-progress.json'
|
||||
pending_file = '.loki/queue/pending.json'
|
||||
|
||||
with open(progress_file, 'r') as f:
|
||||
progress = json.load(f)
|
||||
|
||||
if progress['tasks']:
|
||||
task = progress['tasks'][0]
|
||||
task['retries'] = task.get('retries', 0) + 1
|
||||
task['lastError'] = 'Test failure'
|
||||
task['claimedBy'] = None
|
||||
task['claimedAt'] = None
|
||||
task['backoffSeconds'] = 60 * (2 ** (task['retries'] - 1))
|
||||
|
||||
# Move back to pending for retry
|
||||
with open(pending_file, 'r') as f:
|
||||
pending = json.load(f)
|
||||
|
||||
pending['tasks'].append(task)
|
||||
progress['tasks'] = progress['tasks'][1:]
|
||||
|
||||
with open(progress_file, 'w') as f:
|
||||
json.dump(progress, f)
|
||||
with open(pending_file, 'w') as f:
|
||||
json.dump(pending, f)
|
||||
|
||||
print(f"RETRY:{task['retries']}")
|
||||
EOF
|
||||
|
||||
retry_count=$(python3 -c "
|
||||
import json
|
||||
data = json.load(open('.loki/queue/pending.json'))
|
||||
for t in data['tasks']:
|
||||
if t.get('retries', 0) > 0:
|
||||
print(t['retries'])
|
||||
break
|
||||
else:
|
||||
print(0)
|
||||
")
|
||||
|
||||
if [ "$retry_count" -eq 1 ]; then
|
||||
log_pass "Task moved back to pending with retry count"
|
||||
else
|
||||
log_fail "Retry handling failed"
|
||||
fi
|
||||
|
||||
# Test 7: Dead letter queue
|
||||
log_test "Move to dead letter queue after max retries"
|
||||
python3 << 'EOF'
|
||||
import json
|
||||
from datetime import datetime
|
||||
|
||||
pending_file = '.loki/queue/pending.json'
|
||||
dlq_file = '.loki/queue/dead-letter.json'
|
||||
|
||||
with open(pending_file, 'r') as f:
|
||||
pending = json.load(f)
|
||||
|
||||
with open(dlq_file, 'r') as f:
|
||||
dlq = json.load(f)
|
||||
|
||||
# Find task with retries and simulate max retries exceeded
|
||||
for task in pending['tasks']:
|
||||
if task.get('retries', 0) > 0:
|
||||
task['retries'] = task.get('maxRetries', 3)
|
||||
task['lastError'] = 'Max retries exceeded'
|
||||
task['movedToDLQ'] = datetime.utcnow().isoformat() + 'Z'
|
||||
|
||||
dlq['tasks'].append(task)
|
||||
pending['tasks'] = [t for t in pending['tasks'] if t['id'] != task['id']]
|
||||
break
|
||||
|
||||
with open(pending_file, 'w') as f:
|
||||
json.dump(pending, f)
|
||||
with open(dlq_file, 'w') as f:
|
||||
json.dump(dlq, f)
|
||||
|
||||
print("MOVED_TO_DLQ")
|
||||
EOF
|
||||
|
||||
dlq_count=$(python3 -c "import json; print(len(json.load(open('.loki/queue/dead-letter.json'))['tasks']))")
|
||||
if [ "$dlq_count" -eq 1 ]; then
|
||||
log_pass "Task moved to dead letter queue"
|
||||
else
|
||||
log_fail "Dead letter queue handling failed"
|
||||
fi
|
||||
|
||||
# Test 8: Idempotency check
|
||||
log_test "Idempotency check (duplicate prevention)"
|
||||
python3 << 'EOF'
|
||||
import json
|
||||
import hashlib
|
||||
|
||||
pending_file = '.loki/queue/pending.json'
|
||||
|
||||
with open(pending_file, 'r') as f:
|
||||
pending = json.load(f)
|
||||
|
||||
# Try to add duplicate task
|
||||
new_task = {
|
||||
"id": "task-duplicate",
|
||||
"type": "eng-backend",
|
||||
"payload": {"action": "test"}
|
||||
}
|
||||
|
||||
# Generate idempotency key
|
||||
idempotency_key = hashlib.md5(json.dumps(new_task['payload'], sort_keys=True).encode()).hexdigest()
|
||||
new_task['idempotencyKey'] = idempotency_key
|
||||
|
||||
# Check if already exists
|
||||
existing = [t for t in pending['tasks'] if t.get('idempotencyKey') == idempotency_key]
|
||||
if not existing:
|
||||
pending['tasks'].append(new_task)
|
||||
print("ADDED")
|
||||
else:
|
||||
print("DUPLICATE")
|
||||
|
||||
# Try again with same payload
|
||||
existing = [t for t in pending['tasks'] if t.get('idempotencyKey') == idempotency_key]
|
||||
if existing:
|
||||
print("DUPLICATE_DETECTED")
|
||||
|
||||
with open(pending_file, 'w') as f:
|
||||
json.dump(pending, f)
|
||||
EOF
|
||||
|
||||
log_pass "Idempotency check works"
|
||||
|
||||
echo ""
|
||||
echo "========================================"
|
||||
echo "Test Summary"
|
||||
echo "========================================"
|
||||
echo -e "${GREEN}Passed: $PASSED${NC}"
|
||||
echo -e "${RED}Failed: $FAILED${NC}"
|
||||
echo ""
|
||||
|
||||
if [ $FAILED -eq 0 ]; then
|
||||
echo -e "${GREEN}All tests passed!${NC}"
|
||||
exit 0
|
||||
else
|
||||
echo -e "${RED}Some tests failed!${NC}"
|
||||
exit 1
|
||||
fi
|
||||
314
skills/loki-mode/tests/test-wrapper.sh
Executable file
314
skills/loki-mode/tests/test-wrapper.sh
Executable file
@@ -0,0 +1,314 @@
|
||||
#!/bin/bash
|
||||
# Test: Loki Mode Wrapper Script
|
||||
# Tests the autonomous wrapper functionality
|
||||
|
||||
set -uo pipefail
|
||||
|
||||
TEST_DIR=$(mktemp -d)
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
WRAPPER_SCRIPT="$SCRIPT_DIR/../scripts/loki-wrapper.sh"
|
||||
PASSED=0
|
||||
FAILED=0
|
||||
|
||||
# Colors
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
NC='\033[0m'
|
||||
|
||||
log_pass() { echo -e "${GREEN}[PASS]${NC} $1"; ((PASSED++)); }
|
||||
log_fail() { echo -e "${RED}[FAIL]${NC} $1"; ((FAILED++)); }
|
||||
log_test() { echo -e "${YELLOW}[TEST]${NC} $1"; }
|
||||
|
||||
cleanup() {
|
||||
rm -rf "$TEST_DIR"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
cd "$TEST_DIR"
|
||||
|
||||
echo "=========================================="
|
||||
echo "Loki Mode Wrapper Script Tests"
|
||||
echo "=========================================="
|
||||
echo ""
|
||||
|
||||
# Test 1: Wrapper script exists and is executable
|
||||
log_test "Wrapper script exists and is executable"
|
||||
if [ -x "$WRAPPER_SCRIPT" ]; then
|
||||
log_pass "Wrapper script is executable"
|
||||
else
|
||||
log_fail "Wrapper script not found or not executable"
|
||||
fi
|
||||
|
||||
# Test 2: Wrapper script has correct shebang
|
||||
log_test "Wrapper script has correct shebang"
|
||||
SHEBANG=$(head -1 "$WRAPPER_SCRIPT")
|
||||
if [ "$SHEBANG" = "#!/bin/bash" ]; then
|
||||
log_pass "Correct shebang"
|
||||
else
|
||||
log_fail "Incorrect shebang: $SHEBANG"
|
||||
fi
|
||||
|
||||
# Test 3: Exponential backoff calculation
|
||||
log_test "Exponential backoff calculation"
|
||||
python3 << 'EOF'
|
||||
import os
|
||||
|
||||
BASE_WAIT = 60
|
||||
MAX_WAIT = 3600
|
||||
|
||||
def calculate_wait(retry):
|
||||
wait_time = BASE_WAIT * (2 ** retry)
|
||||
# Add jitter would be random, just test base calculation
|
||||
if wait_time > MAX_WAIT:
|
||||
wait_time = MAX_WAIT
|
||||
return wait_time
|
||||
|
||||
# Test exponential growth
|
||||
assert calculate_wait(0) == 60, f"Retry 0: expected 60, got {calculate_wait(0)}"
|
||||
assert calculate_wait(1) == 120, f"Retry 1: expected 120, got {calculate_wait(1)}"
|
||||
assert calculate_wait(2) == 240, f"Retry 2: expected 240, got {calculate_wait(2)}"
|
||||
assert calculate_wait(3) == 480, f"Retry 3: expected 480, got {calculate_wait(3)}"
|
||||
assert calculate_wait(4) == 960, f"Retry 4: expected 960, got {calculate_wait(4)}"
|
||||
assert calculate_wait(5) == 1920, f"Retry 5: expected 1920, got {calculate_wait(5)}"
|
||||
|
||||
# Test max cap
|
||||
assert calculate_wait(6) == 3600, f"Retry 6: expected 3600 (capped), got {calculate_wait(6)}"
|
||||
assert calculate_wait(10) == 3600, f"Retry 10: expected 3600 (capped), got {calculate_wait(10)}"
|
||||
|
||||
print("VERIFIED")
|
||||
EOF
|
||||
|
||||
if [ $? -eq 0 ]; then
|
||||
log_pass "Exponential backoff calculation works"
|
||||
else
|
||||
log_fail "Exponential backoff calculation failed"
|
||||
fi
|
||||
|
||||
# Test 4: State file JSON structure
|
||||
log_test "State file JSON structure"
|
||||
python3 << 'EOF'
|
||||
import json
|
||||
from datetime import datetime
|
||||
|
||||
# Simulate wrapper state
|
||||
state = {
|
||||
"retryCount": 3,
|
||||
"status": "running",
|
||||
"lastExitCode": 0,
|
||||
"lastRun": datetime.utcnow().isoformat() + 'Z',
|
||||
"prdPath": "./docs/requirements.md",
|
||||
"pid": 12345
|
||||
}
|
||||
|
||||
# Verify JSON serialization
|
||||
json_str = json.dumps(state)
|
||||
parsed = json.loads(json_str)
|
||||
|
||||
assert parsed["retryCount"] == 3
|
||||
assert parsed["status"] == "running"
|
||||
assert parsed["pid"] == 12345
|
||||
print("VERIFIED")
|
||||
EOF
|
||||
|
||||
if [ $? -eq 0 ]; then
|
||||
log_pass "State file JSON structure is valid"
|
||||
else
|
||||
log_fail "State file JSON structure failed"
|
||||
fi
|
||||
|
||||
# Test 5: Completion detection logic
|
||||
log_test "Completion detection logic"
|
||||
mkdir -p "$TEST_DIR/.loki/state"
|
||||
cat > "$TEST_DIR/.loki/state/orchestrator.json" << 'EOF'
|
||||
{
|
||||
"currentPhase": "COMPLETED",
|
||||
"startedAt": "2025-01-15T10:00:00Z",
|
||||
"completedAt": "2025-01-15T12:00:00Z"
|
||||
}
|
||||
EOF
|
||||
|
||||
python3 << EOF
|
||||
import json
|
||||
|
||||
with open("$TEST_DIR/.loki/state/orchestrator.json") as f:
|
||||
state = json.load(f)
|
||||
|
||||
phase = state.get("currentPhase", "")
|
||||
is_completed = phase == "COMPLETED"
|
||||
assert is_completed, f"Expected COMPLETED, got {phase}"
|
||||
print("VERIFIED")
|
||||
EOF
|
||||
|
||||
if [ $? -eq 0 ]; then
|
||||
log_pass "Completion detection works"
|
||||
else
|
||||
log_fail "Completion detection failed"
|
||||
fi
|
||||
|
||||
# Test 6: PRD path validation
|
||||
log_test "PRD path validation"
|
||||
touch "$TEST_DIR/test-prd.md"
|
||||
if [ -f "$TEST_DIR/test-prd.md" ]; then
|
||||
log_pass "PRD path validation works"
|
||||
else
|
||||
log_fail "PRD path validation failed"
|
||||
fi
|
||||
|
||||
# Test 7: Resume prompt generation
|
||||
log_test "Resume prompt generation"
|
||||
python3 << 'EOF'
|
||||
def build_resume_prompt(retry, prd_path=None, initial_prompt="Loki Mode"):
|
||||
if retry == 0:
|
||||
return initial_prompt
|
||||
else:
|
||||
if prd_path:
|
||||
return f"Loki Mode - Resume from checkpoint. PRD at {prd_path}. This is retry #{retry} after rate limit. Check .loki/state/ for current progress and continue from where we left off."
|
||||
else:
|
||||
return f"Loki Mode - Resume from checkpoint. This is retry #{retry} after rate limit. Check .loki/state/ for current progress and continue from where we left off."
|
||||
|
||||
# Test initial prompt
|
||||
assert build_resume_prompt(0) == "Loki Mode"
|
||||
|
||||
# Test resume prompt without PRD
|
||||
resume = build_resume_prompt(3)
|
||||
assert "Resume from checkpoint" in resume
|
||||
assert "retry #3" in resume
|
||||
assert ".loki/state/" in resume
|
||||
|
||||
# Test resume prompt with PRD
|
||||
resume = build_resume_prompt(5, "./docs/req.md")
|
||||
assert "PRD at ./docs/req.md" in resume
|
||||
assert "retry #5" in resume
|
||||
|
||||
print("VERIFIED")
|
||||
EOF
|
||||
|
||||
if [ $? -eq 0 ]; then
|
||||
log_pass "Resume prompt generation works"
|
||||
else
|
||||
log_fail "Resume prompt generation failed"
|
||||
fi
|
||||
|
||||
# Test 8: Rate limit detection logic
|
||||
log_test "Rate limit detection logic"
|
||||
python3 << 'EOF'
|
||||
def is_rate_limit(exit_code, log_content=""):
|
||||
# Any non-zero exit is treated as potential rate limit
|
||||
if exit_code != 0:
|
||||
# Could check logs for specific indicators
|
||||
rate_limit_indicators = ["rate limit", "429", "too many requests", "quota exceeded"]
|
||||
for indicator in rate_limit_indicators:
|
||||
if indicator.lower() in log_content.lower():
|
||||
return True
|
||||
# Conservative: treat any non-zero as rate limit
|
||||
return True
|
||||
return False
|
||||
|
||||
# Test cases
|
||||
assert is_rate_limit(0) == False, "Exit 0 should not be rate limit"
|
||||
assert is_rate_limit(1) == True, "Exit 1 should be treated as rate limit"
|
||||
assert is_rate_limit(1, "Error: Rate limit exceeded") == True
|
||||
assert is_rate_limit(1, "HTTP 429 Too Many Requests") == True
|
||||
assert is_rate_limit(0, "Rate limit in logs but exit 0") == False
|
||||
|
||||
print("VERIFIED")
|
||||
EOF
|
||||
|
||||
if [ $? -eq 0 ]; then
|
||||
log_pass "Rate limit detection logic works"
|
||||
else
|
||||
log_fail "Rate limit detection logic failed"
|
||||
fi
|
||||
|
||||
# Test 9: Log file creation
|
||||
log_test "Log file and directory creation"
|
||||
mkdir -p "$TEST_DIR/.loki"
|
||||
LOG_FILE="$TEST_DIR/.loki/wrapper.log"
|
||||
echo "[2025-01-15 10:00:00] [INFO] Test log entry" >> "$LOG_FILE"
|
||||
|
||||
if [ -f "$LOG_FILE" ] && grep -q "Test log entry" "$LOG_FILE"; then
|
||||
log_pass "Log file creation works"
|
||||
else
|
||||
log_fail "Log file creation failed"
|
||||
fi
|
||||
|
||||
# Test 10: COMPLETED file marker detection
|
||||
log_test "COMPLETED file marker detection"
|
||||
touch "$TEST_DIR/.loki/COMPLETED"
|
||||
if [ -f "$TEST_DIR/.loki/COMPLETED" ]; then
|
||||
log_pass "COMPLETED file marker detection works"
|
||||
else
|
||||
log_fail "COMPLETED file marker detection failed"
|
||||
fi
|
||||
|
||||
# Test 11: Environment variable defaults
|
||||
log_test "Environment variable defaults"
|
||||
python3 << 'EOF'
|
||||
import os
|
||||
|
||||
# Simulate reading with defaults
|
||||
MAX_RETRIES = int(os.environ.get('LOKI_MAX_RETRIES', '50'))
|
||||
BASE_WAIT = int(os.environ.get('LOKI_BASE_WAIT', '60'))
|
||||
MAX_WAIT = int(os.environ.get('LOKI_MAX_WAIT', '3600'))
|
||||
|
||||
assert MAX_RETRIES == 50, f"Expected 50, got {MAX_RETRIES}"
|
||||
assert BASE_WAIT == 60, f"Expected 60, got {BASE_WAIT}"
|
||||
assert MAX_WAIT == 3600, f"Expected 3600, got {MAX_WAIT}"
|
||||
|
||||
print("VERIFIED")
|
||||
EOF
|
||||
|
||||
if [ $? -eq 0 ]; then
|
||||
log_pass "Environment variable defaults work"
|
||||
else
|
||||
log_fail "Environment variable defaults failed"
|
||||
fi
|
||||
|
||||
# Test 12: Wrapper state loading
|
||||
log_test "Wrapper state loading and saving"
|
||||
STATE_FILE="$TEST_DIR/.loki/wrapper-state.json"
|
||||
cat > "$STATE_FILE" << 'EOF'
|
||||
{
|
||||
"retryCount": 7,
|
||||
"status": "running",
|
||||
"lastExitCode": 1,
|
||||
"lastRun": "2025-01-15T10:30:00Z",
|
||||
"prdPath": "./test.md",
|
||||
"pid": 99999
|
||||
}
|
||||
EOF
|
||||
|
||||
python3 << EOF
|
||||
import json
|
||||
|
||||
with open("$STATE_FILE") as f:
|
||||
state = json.load(f)
|
||||
|
||||
assert state["retryCount"] == 7
|
||||
assert state["status"] == "running"
|
||||
assert state["lastExitCode"] == 1
|
||||
print("VERIFIED")
|
||||
EOF
|
||||
|
||||
if [ $? -eq 0 ]; then
|
||||
log_pass "Wrapper state loading works"
|
||||
else
|
||||
log_fail "Wrapper state loading failed"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "=========================================="
|
||||
echo "Test Summary"
|
||||
echo "=========================================="
|
||||
echo -e "${GREEN}Passed: $PASSED${NC}"
|
||||
echo -e "${RED}Failed: $FAILED${NC}"
|
||||
echo ""
|
||||
|
||||
if [ $FAILED -eq 0 ]; then
|
||||
echo -e "${GREEN}All tests passed!${NC}"
|
||||
exit 0
|
||||
else
|
||||
echo -e "${RED}Some tests failed!${NC}"
|
||||
exit 1
|
||||
fi
|
||||
Reference in New Issue
Block a user