mirror of
https://github.com/mblanke/Gov_Travel_App.git
synced 2026-03-01 14:10:22 -05:00
Add Python web scraper for NJC travel rates with currency extraction
- Implemented Python scraper using BeautifulSoup and pandas to automatically collect travel rates from official NJC website - Added currency extraction from table titles (supports EUR, USD, AUD, CAD, ARS, etc.) - Added country extraction from table titles for international rates - Flatten pandas MultiIndex columns for cleaner data structure - Default to CAD for domestic Canadian sources (accommodations and domestic tables) - Created SQLite database schema (raw_tables, rate_entries, exchange_rates, accommodations) - Successfully scraped 92 tables with 17,205 rate entries covering 25 international cities - Added migration script to convert scraped data to Node.js database format - Updated .gitignore for Python files (.venv/, __pycache__, *.pyc, *.sqlite3) - Fixed city validation and currency conversion in main app - Added comprehensive debug and verification scripts This replaces manual JSON maintenance with automated data collection from official government source.
This commit is contained in:
43
package.json
Normal file
43
package.json
Normal file
@@ -0,0 +1,43 @@
|
||||
{
|
||||
"name": "govt-travel-estimator",
|
||||
"version": "1.2.0",
|
||||
"description": "Government Travel Cost Estimator Web Application",
|
||||
"main": "server.js",
|
||||
"scripts": {
|
||||
"start": "node server.js",
|
||||
"dev": "nodemon server.js",
|
||||
"test": "jest",
|
||||
"test:watch": "jest --watch",
|
||||
"test:coverage": "jest --coverage"
|
||||
},
|
||||
"keywords": [
|
||||
"government",
|
||||
"travel",
|
||||
"cost-estimator",
|
||||
"njc"
|
||||
],
|
||||
"author": "",
|
||||
"license": "ISC",
|
||||
"dependencies": {
|
||||
"amadeus": "^11.0.0",
|
||||
"axios": "^1.13.1",
|
||||
"better-sqlite3": "^11.0.0",
|
||||
"compression": "^1.7.4",
|
||||
"cors": "^2.8.5",
|
||||
"dotenv": "^17.2.3",
|
||||
"express": "^4.18.2",
|
||||
"express-rate-limit": "^7.1.5",
|
||||
"helmet": "^7.1.0",
|
||||
"joi": "^17.11.0",
|
||||
"node-cache": "^5.1.2",
|
||||
"sqlite3": "^5.1.7",
|
||||
"winston": "^3.11.0",
|
||||
"winston-daily-rotate-file": "^4.7.1",
|
||||
"xlsx": "^0.18.5"
|
||||
},
|
||||
"devDependencies": {
|
||||
"jest": "^29.7.0",
|
||||
"nodemon": "^3.0.2",
|
||||
"supertest": "^6.3.3"
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user