Files
Gov_Travel_App/scripts/testDatabase.js
mblanke 15094ac94b Add Python web scraper for NJC travel rates with currency extraction
- Implemented Python scraper using BeautifulSoup and pandas to automatically collect travel rates from official NJC website
- Added currency extraction from table titles (supports EUR, USD, AUD, CAD, ARS, etc.)
- Added country extraction from table titles for international rates
- Flatten pandas MultiIndex columns for cleaner data structure
- Default to CAD for domestic Canadian sources (accommodations and domestic tables)
- Created SQLite database schema (raw_tables, rate_entries, exchange_rates, accommodations)
- Successfully scraped 92 tables with 17,205 rate entries covering 25 international cities
- Added migration script to convert scraped data to Node.js database format
- Updated .gitignore for Python files (.venv/, __pycache__, *.pyc, *.sqlite3)
- Fixed city validation and currency conversion in main app
- Added comprehensive debug and verification scripts

This replaces manual JSON maintenance with automated data collection from official government source.
2026-01-13 09:21:43 -05:00

66 lines
2.1 KiB
JavaScript

const sqlite3 = require('sqlite3').verbose();
const path = require('path');
const dbPath = path.join(__dirname, '..', 'database', 'travel_rates.db');
console.log('🔍 Testing Database...\n');
console.log(`📁 Database path: ${dbPath}\n`);
const db = new sqlite3.Database(dbPath, (err) => {
if (err) {
console.error('❌ Failed to open database:', err);
process.exit(1);
}
});
// Test 1: Check if Canberra exists
db.get('SELECT * FROM accommodation_rates WHERE city_key = ?', ['canberra'], (err, row) => {
if (err) {
console.error('❌ Query failed:', err);
} else if (row) {
console.log('✅ CANBERRA FOUND!');
console.log(' City:', row.city_name);
console.log(' Country:', row.country);
console.log(' Region:', row.region);
console.log(' Jan Rate:', `$${row.jan_rate} ${row.currency}`);
console.log(' Standard Rate:', `$${row.standard_rate} ${row.currency}`);
console.log(' International:', row.is_international ? 'Yes' : 'No');
} else {
console.log('❌ CANBERRA NOT FOUND IN DATABASE!');
}
});
// Test 2: Count total cities
db.get('SELECT COUNT(*) as count FROM accommodation_rates', [], (err, row) => {
if (err) {
console.error('❌ Count query failed:', err);
} else {
console.log(`\n📊 Total cities in database: ${row.count}`);
}
});
// Test 3: List all Australian cities
db.all('SELECT city_key, city_name, standard_rate FROM accommodation_rates WHERE country = ?', ['Australia'], (err, rows) => {
if (err) {
console.error('❌ Australia query failed:', err);
} else {
console.log('\n🇦🇺 Australian cities:');
if (rows.length === 0) {
console.log(' ❌ No Australian cities found!');
} else {
rows.forEach(row => {
console.log(` - ${row.city_name}: $${row.standard_rate} USD`);
});
}
}
// Close database
db.close((err) => {
if (err) {
console.error('Error closing database:', err);
} else {
console.log('\n✅ Test complete!');
}
});
});