mirror of
https://github.com/mblanke/Gov_Travel_App.git
synced 2026-03-01 14:10:22 -05:00
Add Python web scraper for NJC travel rates with currency extraction
- Implemented Python scraper using BeautifulSoup and pandas to automatically collect travel rates from official NJC website - Added currency extraction from table titles (supports EUR, USD, AUD, CAD, ARS, etc.) - Added country extraction from table titles for international rates - Flatten pandas MultiIndex columns for cleaner data structure - Default to CAD for domestic Canadian sources (accommodations and domestic tables) - Created SQLite database schema (raw_tables, rate_entries, exchange_rates, accommodations) - Successfully scraped 92 tables with 17,205 rate entries covering 25 international cities - Added migration script to convert scraped data to Node.js database format - Updated .gitignore for Python files (.venv/, __pycache__, *.pyc, *.sqlite3) - Fixed city validation and currency conversion in main app - Added comprehensive debug and verification scripts This replaces manual JSON maintenance with automated data collection from official government source.
This commit is contained in:
59
scripts/testCanberraAPI.js
Normal file
59
scripts/testCanberraAPI.js
Normal file
@@ -0,0 +1,59 @@
|
||||
const http = require('http');
|
||||
|
||||
async function testAPI() {
|
||||
console.log('\n🧪 Testing Canberra Search API...\n');
|
||||
|
||||
const options = {
|
||||
hostname: 'localhost',
|
||||
port: 5001,
|
||||
path: '/api/accommodation/search?city=canberra',
|
||||
method: 'GET'
|
||||
};
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
const req = http.request(options, (res) => {
|
||||
let data = '';
|
||||
|
||||
res.on('data', (chunk) => {
|
||||
data += chunk;
|
||||
});
|
||||
|
||||
res.on('end', () => {
|
||||
try {
|
||||
const json = JSON.parse(data);
|
||||
|
||||
if (json.city) {
|
||||
console.log('✅ SUCCESS! Canberra Found:\n');
|
||||
console.log(` City: ${json.city}`);
|
||||
console.log(` Country: ${json.country}`);
|
||||
console.log(` Region: ${json.region}`);
|
||||
console.log(` Currency: ${json.currency}`);
|
||||
console.log(` Standard Rate: $${json.rates.standard || json.rates[0]} ${json.currency}`);
|
||||
console.log(` January Rate: $${json.rates[0]} ${json.currency}`);
|
||||
console.log('\n🎉 CANBERRA IS 100% SEARCHABLE!\n');
|
||||
} else if (json.error) {
|
||||
console.log(`❌ API Error: ${json.error}`);
|
||||
} else {
|
||||
console.log('❓ Unexpected response:', json);
|
||||
}
|
||||
|
||||
resolve();
|
||||
} catch (err) {
|
||||
console.error('❌ Failed to parse response:', err.message);
|
||||
console.log('Raw response:', data);
|
||||
reject(err);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
req.on('error', (err) => {
|
||||
console.error(`❌ Connection failed: ${err.message}`);
|
||||
console.error('Make sure the server is running: node server.js');
|
||||
reject(err);
|
||||
});
|
||||
|
||||
req.end();
|
||||
});
|
||||
}
|
||||
|
||||
testAPI();
|
||||
Reference in New Issue
Block a user