Add Python web scraper for NJC travel rates with currency extraction

- Implemented Python scraper using BeautifulSoup and pandas to automatically collect travel rates from official NJC website
- Added currency extraction from table titles (supports EUR, USD, AUD, CAD, ARS, etc.)
- Added country extraction from table titles for international rates
- Flatten pandas MultiIndex columns for cleaner data structure
- Default to CAD for domestic Canadian sources (accommodations and domestic tables)
- Created SQLite database schema (raw_tables, rate_entries, exchange_rates, accommodations)
- Successfully scraped 92 tables with 17,205 rate entries covering 25 international cities
- Added migration script to convert scraped data to Node.js database format
- Updated .gitignore for Python files (.venv/, __pycache__, *.pyc, *.sqlite3)
- Fixed city validation and currency conversion in main app
- Added comprehensive debug and verification scripts

This replaces manual JSON maintenance with automated data collection from official government source.
This commit is contained in:
2026-01-13 09:21:43 -05:00
commit 15094ac94b
84 changed files with 19859 additions and 0 deletions

View File

@@ -0,0 +1,59 @@
const http = require('http');
async function testAPI() {
console.log('\n🧪 Testing Canberra Search API...\n');
const options = {
hostname: 'localhost',
port: 5001,
path: '/api/accommodation/search?city=canberra',
method: 'GET'
};
return new Promise((resolve, reject) => {
const req = http.request(options, (res) => {
let data = '';
res.on('data', (chunk) => {
data += chunk;
});
res.on('end', () => {
try {
const json = JSON.parse(data);
if (json.city) {
console.log('✅ SUCCESS! Canberra Found:\n');
console.log(` City: ${json.city}`);
console.log(` Country: ${json.country}`);
console.log(` Region: ${json.region}`);
console.log(` Currency: ${json.currency}`);
console.log(` Standard Rate: $${json.rates.standard || json.rates[0]} ${json.currency}`);
console.log(` January Rate: $${json.rates[0]} ${json.currency}`);
console.log('\n🎉 CANBERRA IS 100% SEARCHABLE!\n');
} else if (json.error) {
console.log(`❌ API Error: ${json.error}`);
} else {
console.log('❓ Unexpected response:', json);
}
resolve();
} catch (err) {
console.error('❌ Failed to parse response:', err.message);
console.log('Raw response:', data);
reject(err);
}
});
});
req.on('error', (err) => {
console.error(`❌ Connection failed: ${err.message}`);
console.error('Make sure the server is running: node server.js');
reject(err);
});
req.end();
});
}
testAPI();