mirror of
https://github.com/mblanke/Gov_Travel_App.git
synced 2026-03-01 14:10:22 -05:00
Integrate OpenFlights API for free, no-auth flight data generation
- Added openFlightsService.js to fetch and cache OpenFlights airport/airline/routes data - Validates airport codes exist in OpenFlights database (6072+ airports) - Generates realistic flights using major international airlines - Creates varied routing options: direct, 1-stop, 2-stop flights - Updated flightService.js to use OpenFlights as primary source before Amadeus - OpenFlights as fallback if Amadeus unavailable or returns no results - No API keys or authentication required - Cached locally to avoid repeated network requests - Realistic pricing, times, and stop locations Docker container rebuilt with OpenFlights integration.
This commit is contained in:
@@ -1,6 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
from gov_travel import db
|
||||
@@ -26,24 +27,65 @@ def parse_args() -> argparse.Namespace:
|
||||
|
||||
def main() -> None:
|
||||
args = parse_args()
|
||||
start_time = time.time()
|
||||
|
||||
print("=" * 80)
|
||||
print("🌐 Government Travel Rate Scraper")
|
||||
print("=" * 80)
|
||||
print(f"📁 Database: {args.db}")
|
||||
print()
|
||||
|
||||
connection = db.connect(args.db)
|
||||
db.init_db(connection)
|
||||
|
||||
for source in SOURCES:
|
||||
total_tables = 0
|
||||
total_rate_entries = 0
|
||||
total_accommodations = 0
|
||||
|
||||
for idx, source in enumerate(SOURCES, 1):
|
||||
source_start = time.time()
|
||||
print(f"[{idx}/{len(SOURCES)}] 📥 Scraping: {source.name.upper()}")
|
||||
print(f" 🔗 {source.url}")
|
||||
|
||||
tables = scrape_tables_from_source(source)
|
||||
db.insert_raw_tables(connection, source.name, source.url, tables)
|
||||
total_tables += len(tables)
|
||||
print(f" ✓ {len(tables)} tables collected")
|
||||
|
||||
rate_entries = extract_rate_entries(source, tables)
|
||||
db.insert_rate_entries(connection, rate_entries)
|
||||
total_rate_entries += len(rate_entries)
|
||||
if rate_entries:
|
||||
print(f" ✓ {len(rate_entries)} per-diem entries extracted")
|
||||
|
||||
exchange_rates = extract_exchange_rates(source, tables)
|
||||
db.insert_exchange_rates(connection, exchange_rates)
|
||||
if exchange_rates:
|
||||
print(f" ✓ {len(exchange_rates)} exchange rates extracted")
|
||||
|
||||
if source.name == "accommodations":
|
||||
accommodations = extract_accommodations(source, tables)
|
||||
db.insert_accommodations(connection, accommodations)
|
||||
total_accommodations = len(accommodations)
|
||||
print(f" ✓ {len(accommodations)} accommodation listings extracted")
|
||||
|
||||
elapsed = time.time() - source_start
|
||||
print(f" ⏱️ Completed in {elapsed:.1f}s")
|
||||
print()
|
||||
|
||||
connection.close()
|
||||
|
||||
total_time = time.time() - start_time
|
||||
print("=" * 80)
|
||||
print("✅ SCRAPING COMPLETE")
|
||||
print("=" * 80)
|
||||
print(f"📊 Summary:")
|
||||
print(f" • Total tables: {total_tables:,}")
|
||||
print(f" • Per-diem entries: {total_rate_entries:,}")
|
||||
print(f" • Accommodation listings: {total_accommodations:,}")
|
||||
print(f" • Total time: {total_time:.1f}s")
|
||||
print(f" • Database: {args.db}")
|
||||
print("=" * 80)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
Reference in New Issue
Block a user