Gov_Travel_App/scripts/migrateScrapedToNodeJS.js

/**
 * Migration script to convert scraped SQLite database to Node.js travel_rates schema
 *
 * Source DB: travel_rates_scraped.sqlite3 (from Python scraper)
 * Target DB: travel_rates.db (Node.js app schema)
 *
 * This script:
 * 1. Reads rate_entries from scraped DB
 * 2. Aggregates meal rates (breakfast, lunch, dinner) and incidentals by city
 * 3. Inserts into travel_rates table in Node.js format
 */

const sqlite3 = require("sqlite3").verbose();
const path = require("path");

// Database paths
const SOURCE_DB = path.join(
  __dirname,
  "..",
  "data",
  "travel_rates_scraped.sqlite3"
);
const TARGET_DB = path.join(__dirname, "..", "travel_rates.db");

// Exchange rates for display (not used for conversion, just for reference)
const EXCHANGE_RATES = {
  EUR: 1.54, // EUR to CAD
  USD: 1.39, // USD to CAD
  AUD: 0.92, // AUD to CAD
  CAD: 1.0,
  ARS: 0.0014, // ARS to CAD (approximate)
};

async function openDatabase(dbPath) {
  return new Promise((resolve, reject) => {
    const db = new sqlite3.Database(dbPath, (err) => {
      if (err) reject(err);
      else resolve(db);
    });
  });
}

async function queryAll(db, sql, params = []) {
  return new Promise((resolve, reject) => {
    db.all(sql, params, (err, rows) => {
      if (err) reject(err);
      else resolve(rows);
    });
  });
}

async function runQuery(db, sql, params = []) {
  return new Promise((resolve, reject) => {
    db.run(sql, params, function (err) {
      if (err) reject(err);
      else resolve(this);
    });
  });
}

async function aggregateCityRates(sourceDb) {
  // Get all international cities with their meal rates
  const query = `
        SELECT 
            country,
            city,
            currency,
            MAX(CASE WHEN rate_type LIKE '%breakfast%' THEN rate_amount END) as breakfast,
            MAX(CASE WHEN rate_type LIKE '%lunch%' THEN rate_amount END) as lunch,
            MAX(CASE WHEN rate_type LIKE '%dinner%' THEN rate_amount END) as dinner,
            MAX(CASE WHEN rate_type LIKE '%incidental%' THEN rate_amount END) as incidentals
        FROM rate_entries
        WHERE city IS NOT NULL
            AND country IS NOT NULL
            AND source = 'international'
        GROUP BY country, city, currency
        HAVING breakfast IS NOT NULL OR lunch IS NOT NULL OR dinner IS NOT NULL
    `;

  return await queryAll(sourceDb, query);
}

async function clearTargetDatabase(targetDb) {
  await runQuery(targetDb, "DELETE FROM travel_rates");
  console.log("Cleared existing travel_rates data");
}

async function insertCityRates(targetDb, cities) {
  const insertStmt = `
        INSERT INTO travel_rates (
            city_key, city_name, country, breakfast, lunch, dinner, 
            incidentals, currency, standardRate, standard_rate
        ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
    `;

  let inserted = 0;
  let skipped = 0;

  for (const city of cities) {
    try {
      // Create city key (lowercase, spaces to dashes)
      const cityKey = `${city.city
        .toLowerCase()
        .replace(/\s+/g, "-")}-${city.country
        .toLowerCase()
        .replace(/\s+/g, "-")}`;

      // Standard rate is typically breakfast + lunch + dinner
      const standardRate =
        (city.breakfast || 0) + (city.lunch || 0) + (city.dinner || 0);

      await runQuery(targetDb, insertStmt, [
        cityKey,
        city.city,
        city.country,
        city.breakfast,
        city.lunch,
        city.dinner,
        city.incidentals,
        city.currency,
        standardRate,
        standardRate, // Both standardRate and standard_rate for compatibility
      ]);

      inserted++;
    } catch (err) {
      console.error(
        `Error inserting ${city.city}, ${city.country}: ${err.message}`
      );
      skipped++;
    }
  }

  return { inserted, skipped };
}

async function migrate() {
  console.log(
    "Starting migration from scraped database to Node.js schema...\n"
  );

  let sourceDb, targetDb;

  try {
    // Open databases
    console.log(`Opening source database: ${SOURCE_DB}`);
    sourceDb = await openDatabase(SOURCE_DB);

    console.log(`Opening target database: ${TARGET_DB}`);
    targetDb = await openDatabase(TARGET_DB);

    // Aggregate city rates from scraped data
    console.log("\nAggregating city rates from scraped data...");
    const cities = await aggregateCityRates(sourceDb);
    console.log(`Found ${cities.length} cities with meal rates`);

    // Show currency distribution
    const currencyCounts = cities.reduce((acc, city) => {
      acc[city.currency] = (acc[city.currency] || 0) + 1;
      return acc;
    }, {});
    console.log("\nCurrency distribution:");
    for (const [currency, count] of Object.entries(currencyCounts)) {
      console.log(`  ${currency}: ${count} cities`);
    }

    // Clear target database
    console.log("\nClearing target database...");
    await clearTargetDatabase(targetDb);

    // Insert city rates
    console.log("\nInserting city rates into target database...");
    const result = await insertCityRates(targetDb, cities);

    console.log(`\nMigration complete!`);
    console.log(`  Inserted: ${result.inserted} cities`);
    console.log(`  Skipped: ${result.skipped} cities`);

    // Show sample entries
    console.log("\nSample migrated entries:");
    const samples = await queryAll(
      targetDb,
      `
            SELECT city_name, country, breakfast, lunch, dinner, incidentals, currency
            FROM travel_rates
            WHERE country IN ('Argentina', 'Albania', 'Australia')
            LIMIT 5
        `
    );
    for (const sample of samples) {
      console.log(
        `  ${sample.city_name}, ${sample.country}: B:${sample.breakfast} L:${sample.lunch} D:${sample.dinner} I:${sample.incidentals} (${sample.currency})`
      );
    }
  } catch (err) {
    console.error("\nMigration failed:", err);
    process.exit(1);
  } finally {
    if (sourceDb) sourceDb.close();
    if (targetDb) targetDb.close();
  }
}

// Run migration
if (require.main === module) {
  migrate().catch((err) => {
    console.error("Fatal error:", err);
    process.exit(1);
  });
}

module.exports = { migrate };