mirror of
https://github.com/mblanke/Gov_Travel_App.git
synced 2026-03-01 14:10:22 -05:00
- Implemented Python scraper using BeautifulSoup and pandas to automatically collect travel rates from official NJC website - Added currency extraction from table titles (supports EUR, USD, AUD, CAD, ARS, etc.) - Added country extraction from table titles for international rates - Flatten pandas MultiIndex columns for cleaner data structure - Default to CAD for domestic Canadian sources (accommodations and domestic tables) - Created SQLite database schema (raw_tables, rate_entries, exchange_rates, accommodations) - Successfully scraped 92 tables with 17,205 rate entries covering 25 international cities - Added migration script to convert scraped data to Node.js database format - Updated .gitignore for Python files (.venv/, __pycache__, *.pyc, *.sqlite3) - Fixed city validation and currency conversion in main app - Added comprehensive debug and verification scripts This replaces manual JSON maintenance with automated data collection from official government source.
66 lines
2.1 KiB
JavaScript
66 lines
2.1 KiB
JavaScript
const sqlite3 = require('sqlite3').verbose();
|
|
const path = require('path');
|
|
|
|
const dbPath = path.join(__dirname, '..', 'database', 'travel_rates.db');
|
|
|
|
console.log('🔍 Testing Database...\n');
|
|
console.log(`📁 Database path: ${dbPath}\n`);
|
|
|
|
const db = new sqlite3.Database(dbPath, (err) => {
|
|
if (err) {
|
|
console.error('❌ Failed to open database:', err);
|
|
process.exit(1);
|
|
}
|
|
});
|
|
|
|
// Test 1: Check if Canberra exists
|
|
db.get('SELECT * FROM accommodation_rates WHERE city_key = ?', ['canberra'], (err, row) => {
|
|
if (err) {
|
|
console.error('❌ Query failed:', err);
|
|
} else if (row) {
|
|
console.log('✅ CANBERRA FOUND!');
|
|
console.log(' City:', row.city_name);
|
|
console.log(' Country:', row.country);
|
|
console.log(' Region:', row.region);
|
|
console.log(' Jan Rate:', `$${row.jan_rate} ${row.currency}`);
|
|
console.log(' Standard Rate:', `$${row.standard_rate} ${row.currency}`);
|
|
console.log(' International:', row.is_international ? 'Yes' : 'No');
|
|
} else {
|
|
console.log('❌ CANBERRA NOT FOUND IN DATABASE!');
|
|
}
|
|
});
|
|
|
|
// Test 2: Count total cities
|
|
db.get('SELECT COUNT(*) as count FROM accommodation_rates', [], (err, row) => {
|
|
if (err) {
|
|
console.error('❌ Count query failed:', err);
|
|
} else {
|
|
console.log(`\n📊 Total cities in database: ${row.count}`);
|
|
}
|
|
});
|
|
|
|
// Test 3: List all Australian cities
|
|
db.all('SELECT city_key, city_name, standard_rate FROM accommodation_rates WHERE country = ?', ['Australia'], (err, rows) => {
|
|
if (err) {
|
|
console.error('❌ Australia query failed:', err);
|
|
} else {
|
|
console.log('\n🇦🇺 Australian cities:');
|
|
if (rows.length === 0) {
|
|
console.log(' ❌ No Australian cities found!');
|
|
} else {
|
|
rows.forEach(row => {
|
|
console.log(` - ${row.city_name}: $${row.standard_rate} USD`);
|
|
});
|
|
}
|
|
}
|
|
|
|
// Close database
|
|
db.close((err) => {
|
|
if (err) {
|
|
console.error('Error closing database:', err);
|
|
} else {
|
|
console.log('\n✅ Test complete!');
|
|
}
|
|
});
|
|
});
|