mirror of
https://github.com/mblanke/Gov_Travel_App.git
synced 2026-03-01 14:10:22 -05:00
Add Python web scraper for NJC travel rates with currency extraction
- Implemented Python scraper using BeautifulSoup and pandas to automatically collect travel rates from official NJC website - Added currency extraction from table titles (supports EUR, USD, AUD, CAD, ARS, etc.) - Added country extraction from table titles for international rates - Flatten pandas MultiIndex columns for cleaner data structure - Default to CAD for domestic Canadian sources (accommodations and domestic tables) - Created SQLite database schema (raw_tables, rate_entries, exchange_rates, accommodations) - Successfully scraped 92 tables with 17,205 rate entries covering 25 international cities - Added migration script to convert scraped data to Node.js database format - Updated .gitignore for Python files (.venv/, __pycache__, *.pyc, *.sqlite3) - Fixed city validation and currency conversion in main app - Added comprehensive debug and verification scripts This replaces manual JSON maintenance with automated data collection from official government source.
This commit is contained in:
65
scripts/testDatabase.js
Normal file
65
scripts/testDatabase.js
Normal file
@@ -0,0 +1,65 @@
|
||||
const sqlite3 = require('sqlite3').verbose();
|
||||
const path = require('path');
|
||||
|
||||
const dbPath = path.join(__dirname, '..', 'database', 'travel_rates.db');
|
||||
|
||||
console.log('🔍 Testing Database...\n');
|
||||
console.log(`📁 Database path: ${dbPath}\n`);
|
||||
|
||||
const db = new sqlite3.Database(dbPath, (err) => {
|
||||
if (err) {
|
||||
console.error('❌ Failed to open database:', err);
|
||||
process.exit(1);
|
||||
}
|
||||
});
|
||||
|
||||
// Test 1: Check if Canberra exists
|
||||
db.get('SELECT * FROM accommodation_rates WHERE city_key = ?', ['canberra'], (err, row) => {
|
||||
if (err) {
|
||||
console.error('❌ Query failed:', err);
|
||||
} else if (row) {
|
||||
console.log('✅ CANBERRA FOUND!');
|
||||
console.log(' City:', row.city_name);
|
||||
console.log(' Country:', row.country);
|
||||
console.log(' Region:', row.region);
|
||||
console.log(' Jan Rate:', `$${row.jan_rate} ${row.currency}`);
|
||||
console.log(' Standard Rate:', `$${row.standard_rate} ${row.currency}`);
|
||||
console.log(' International:', row.is_international ? 'Yes' : 'No');
|
||||
} else {
|
||||
console.log('❌ CANBERRA NOT FOUND IN DATABASE!');
|
||||
}
|
||||
});
|
||||
|
||||
// Test 2: Count total cities
|
||||
db.get('SELECT COUNT(*) as count FROM accommodation_rates', [], (err, row) => {
|
||||
if (err) {
|
||||
console.error('❌ Count query failed:', err);
|
||||
} else {
|
||||
console.log(`\n📊 Total cities in database: ${row.count}`);
|
||||
}
|
||||
});
|
||||
|
||||
// Test 3: List all Australian cities
|
||||
db.all('SELECT city_key, city_name, standard_rate FROM accommodation_rates WHERE country = ?', ['Australia'], (err, rows) => {
|
||||
if (err) {
|
||||
console.error('❌ Australia query failed:', err);
|
||||
} else {
|
||||
console.log('\n🇦🇺 Australian cities:');
|
||||
if (rows.length === 0) {
|
||||
console.log(' ❌ No Australian cities found!');
|
||||
} else {
|
||||
rows.forEach(row => {
|
||||
console.log(` - ${row.city_name}: $${row.standard_rate} USD`);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Close database
|
||||
db.close((err) => {
|
||||
if (err) {
|
||||
console.error('Error closing database:', err);
|
||||
} else {
|
||||
console.log('\n✅ Test complete!');
|
||||
}
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user