feat: add scripts for database inspection and migration

- Implemented multiple scripts to check and inspect meal plans, scraped data, and accommodations for Munich and Riga.
- Added a migration script to convert scraped data into the application's database format.
- Introduced new database service methods for querying and updating travel rates.
- Enhanced server configuration for serving static files in production.
- Updated PostCSS configuration for consistency.

scripts/checkMealPlan.js

@@ -0,0 +1,12 @@
+const Database = require("better-sqlite3");
+const path = require("path");
+const db = new Database(
+  path.join(__dirname, "..", "database", "travel_rates.db")
+);
+console.log(
+  db
+    .prepare(
+      "select city_name, meal_plan_type from travel_rates where lower(city_name)='munich'"
+    )
+    .get()
+);

scripts/checkScrapedData.js

@@ -0,0 +1,39 @@
+const Database = require("better-sqlite3");
+const db = new Database("travel_rates.db");
+
+// Check tables
+const tables = db
+  .prepare("SELECT name FROM sqlite_master WHERE type='table'")
+  .all();
+console.log(
+  "Tables in database:",
+  tables.map((t) => t.name)
+);
+
+// Check rate entries
+const rateCount = db
+  .prepare("SELECT COUNT(*) as count FROM rate_entries")
+  .get();
+console.log("\nRate entries:", rateCount.count);
+
+// Check accommodations
+const accommCount = db
+  .prepare("SELECT COUNT(*) as count FROM accommodations")
+  .get();
+console.log("Accommodations:", accommCount.count);
+
+// Check Latvia/Riga
+const latvia = db
+  .prepare(
+    `
+  SELECT city, jan, feb, mar, apr, may, jun, jul, aug, sep, oct, nov, dec 
+  FROM accommodations 
+  WHERE country = 'Latvia'
+`
+  )
+  .all();
+
+console.log("\nLatvia accommodations:");
+console.log(JSON.stringify(latvia, null, 2));
+
+db.close();

scripts/debugMunichPlan.js

@@ -0,0 +1,61 @@
+const Database = require("better-sqlite3");
+const path = require("path");
+const sourceDb = new Database(
+  path.join(__dirname, "..", "data", "travel_rates_scraped.sqlite3")
+);
+const typePriority = [
+  "C-Day 1-30",
+  "C-Day 31-120",
+  "C-Day 121 +",
+  "P-Day 1-30",
+  "P-Day 31-120",
+  "P-Day 121 +",
+];
+const rateEntries = sourceDb
+  .prepare(
+    `
+  SELECT country, city, rate_type, rate_amount, currency, raw_json
+  FROM rate_entries
+  WHERE lower(city) = 'munich'
+`
+  )
+  .all();
+const cityRates = {};
+for (const row of rateEntries) {
+  const data = JSON.parse(row.raw_json);
+  const type = (data["Type of Accommodation"] || "").trim();
+  const priority = typePriority.indexOf(type);
+  const key = `${row.country}_${row.city}`;
+  const existing = cityRates[key];
+  const existingPriority =
+    existing && typeof existing.priority === "number"
+      ? existing.priority
+      : Infinity;
+  const isKnownPriority = priority !== -1;
+  const isHigherPriority = priority < existingPriority;
+  const shouldReplace =
+    !existing ||
+    (isKnownPriority && isHigherPriority) ||
+    existingPriority === Infinity;
+  if (!shouldReplace) continue;
+  cityRates[key] = {
+    country: row.country,
+    city: row.city,
+    currency: row.currency,
+    meal_plan_type:
+      type ||
+      existing?.meal_plan_type ||
+      (isKnownPriority ? typePriority[priority] : null),
+    priority: isKnownPriority ? priority : existingPriority,
+    breakfast: parseFloat(data.Breakfast) || existing?.breakfast || null,
+    lunch: parseFloat(data.Lunch || data["Lunch"]) || existing?.lunch || null,
+    dinner: parseFloat(data.Dinner) || existing?.dinner || null,
+    incidentals:
+      parseFloat(data["Incidental Amount"]) || existing?.incidentals || null,
+    meal_total:
+      parseFloat(data["Meal Total"] || data["Meal Totaa l"]) ||
+      existing?.meal_total ||
+      null,
+  };
+}
+console.log(cityRates);

scripts/inspectFreshData.js

@@ -0,0 +1,38 @@
+const Database = require("better-sqlite3");
+const sourceDb = new Database("data/travel_rates_scraped.sqlite3");
+
+// Check schema
+console.log("\n=== ACCOMMODATIONS SCHEMA ===");
+const accomSchema = sourceDb
+  .prepare(
+    "SELECT sql FROM sqlite_master WHERE type='table' AND name='accommodations'"
+  )
+  .get();
+console.log(accomSchema.sql);
+
+// Get Latvia data
+console.log("\n=== LATVIA ACCOMMODATIONS ===");
+const latvia = sourceDb
+  .prepare(
+    `
+  SELECT * FROM accommodations 
+  WHERE source_url LIKE '%Latvia%' OR raw_json LIKE '%Latvia%'
+  LIMIT 5
+`
+  )
+  .all();
+console.log(JSON.stringify(latvia, null, 2));
+
+// Search for Riga specifically
+console.log("\n=== RIGA SEARCH ===");
+const riga = sourceDb
+  .prepare(
+    `
+  SELECT * FROM accommodations 
+  WHERE city LIKE '%Riga%' OR raw_json LIKE '%Riga%'
+`
+  )
+  .all();
+console.log(JSON.stringify(riga, null, 2));
+
+sourceDb.close();

scripts/inspectMunichEntry.js

@@ -0,0 +1,16 @@
+const Database = require("better-sqlite3");
+const path = require("path");
+const db = new Database(
+  path.join(__dirname, "..", "data", "travel_rates_scraped.sqlite3")
+);
+const row = db
+  .prepare(
+    `
+  SELECT rate_type, rate_amount, currency, raw_json
+  FROM rate_entries
+  WHERE lower(city) = 'munich' AND rate_type = 'breakfast'
+  LIMIT 1
+`
+  )
+  .get();
+console.log(row);

scripts/inspectMunichKeyCodes.js

@@ -0,0 +1,16 @@
+const Database = require("better-sqlite3");
+const path = require("path");
+const db = new Database(
+  path.join(__dirname, "..", "data", "travel_rates_scraped.sqlite3")
+);
+const row = db
+  .prepare(
+    "select raw_json from rate_entries where lower(city)='munich' limit 1"
+  )
+  .get();
+const data = JSON.parse(row.raw_json);
+const keys = Object.keys(data);
+for (const k of keys) {
+  const codes = Array.from(k).map((c) => c.charCodeAt(0));
+  console.log(k, codes);
+}

scripts/inspectMunichRaw.js

@@ -0,0 +1,16 @@
+const Database = require("better-sqlite3");
+const path = require("path");
+const db = new Database(
+  path.join(__dirname, "..", "data", "travel_rates_scraped.sqlite3")
+);
+const row = db
+  .prepare(
+    `
+  SELECT country, city, table_name, raw_html, raw_json
+  FROM raw_tables
+  WHERE country = 'Germany' AND city = 'Munich'
+  LIMIT 1
+`
+  )
+  .get();
+console.log(row);

scripts/inspectMunichRawTables.js

@@ -0,0 +1,14 @@
+const Database = require("better-sqlite3");
+const path = require("path");
+const db = new Database(
+  path.join(__dirname, "..", "data", "travel_rates_scraped.sqlite3")
+);
+const row = db
+  .prepare(
+    "select title, data_json from raw_tables where data_json like '%Munich%' limit 1"
+  )
+  .get();
+console.log(row?.title);
+if (row?.data_json) {
+  console.log(row.data_json.slice(0, 2000));
+}

scripts/listMunichTypes.js

@@ -0,0 +1,14 @@
+const Database = require("better-sqlite3");
+const path = require("path");
+const db = new Database(
+  path.join(__dirname, "..", "data", "travel_rates_scraped.sqlite3")
+);
+const rows = db
+  .prepare("select raw_json from rate_entries where lower(city)='munich'")
+  .all();
+const types = new Set();
+for (const r of rows) {
+  const data = JSON.parse(r.raw_json);
+  types.add(data["Type of Accommodation"]);
+}
+console.log(types);

scripts/migrateFreshData.js

@@ -0,0 +1,391 @@
+/**
+ * Migration script to properly convert freshly scraped data to Node.js app format
+ * Source: data/travel_rates_scraped.sqlite3 (Python scraper with raw_json)
+ * Target: database/travel_rates.db (Node.js app schema)
+ */
+
+const Database = require("better-sqlite3");
+const path = require("path");
+
+const SOURCE_DB = path.join(
+  __dirname,
+  "..",
+  "data",
+  "travel_rates_scraped.sqlite3"
+);
+const TARGET_DB = path.join(__dirname, "..", "database", "travel_rates.db");
+
+console.log("🚀 Starting fresh data migration...\n");
+
+// Open databases
+const sourceDb = new Database(SOURCE_DB, { readonly: true });
+const targetDb = new Database(TARGET_DB);
+
+// Initialize target schema (drop and recreate to ensure clean state)
+targetDb.exec(`
+DROP TABLE IF EXISTS travel_rates;
+
+CREATE TABLE travel_rates (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    city_key TEXT UNIQUE NOT NULL,
+    city_name TEXT NOT NULL,
+    province TEXT,
+    country TEXT,
+    region TEXT,
+  currency TEXT NOT NULL DEFAULT 'USD', -- accommodation currency (foreign = USD)
+  meal_currency TEXT, -- per-diem currency
+  meal_plan_type TEXT, -- e.g., C-Day 1-30
+  meal_total REAL,
+    jan_accommodation REAL,
+    feb_accommodation REAL,
+    mar_accommodation REAL,
+    apr_accommodation REAL,
+    may_accommodation REAL,
+    jun_accommodation REAL,
+    jul_accommodation REAL,
+    aug_accommodation REAL,
+    sep_accommodation REAL,
+    oct_accommodation REAL,
+    nov_accommodation REAL,
+    dec_accommodation REAL,
+    standard_accommodation REAL,
+    breakfast REAL,
+    lunch REAL,
+    dinner REAL,
+    incidentals REAL,
+    created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+    updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+);
+`);
+
+// Clear existing data (already done by DROP TABLE above)
+console.log("📋 Schema created with nullable meal fields");
+
+let inserted = 0;
+let errors = 0;
+
+// Migrate accommodations (international rates)
+console.log("📥 Migrating international accommodations...");
+const accommodations = sourceDb
+  .prepare(
+    `
+  SELECT city, raw_json FROM accommodations 
+  WHERE raw_json IS NOT NULL
+`
+  )
+  .all();
+
+const insertStmt = targetDb.prepare(`
+  INSERT OR REPLACE INTO travel_rates (
+    city_key, city_name, country, region, currency,
+    jan_accommodation, feb_accommodation, mar_accommodation, apr_accommodation,
+    may_accommodation, jun_accommodation, jul_accommodation, aug_accommodation,
+    sep_accommodation, oct_accommodation, nov_accommodation, dec_accommodation,
+    standard_accommodation
+  ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+`);
+
+for (const row of accommodations) {
+  try {
+    const data = JSON.parse(row.raw_json);
+    const country = data.Country || "";
+    const city = data.City || row.city || "";
+
+    if (!city || !country) continue;
+
+    const cityKey = `${country.toLowerCase().replace(/\s+/g, "_")}_${city
+      .toLowerCase()
+      .replace(/\s+/g, "_")}`;
+
+    // Parse monthly rates
+    const jan = parseFloat(data["Jan."]) || null;
+    const feb = parseFloat(data["Feb."]) || null;
+    const mar = parseFloat(data["Mar."]) || null;
+    const apr = parseFloat(data["Apr."]) || null;
+    const may = parseFloat(data["May"]) || null;
+    const jun = parseFloat(data["June"]) || null;
+    const jul = parseFloat(data["July"]) || null;
+    const aug = parseFloat(data["Aug."]) || null;
+    const sep = parseFloat(data["Sept."]) || null;
+    const oct = parseFloat(data["Oct."]) || null;
+    const nov = parseFloat(data["Nov."]) || null;
+    const dec = parseFloat(data["Dec."]) || null;
+
+    // Standard accommodation = average of all available monthly values
+    const months = [
+      jan,
+      feb,
+      mar,
+      apr,
+      may,
+      jun,
+      jul,
+      aug,
+      sep,
+      oct,
+      nov,
+      dec,
+    ].filter((v) => typeof v === "number" && !Number.isNaN(v));
+    const standard =
+      months.length > 0
+        ? months.reduce((sum, v) => sum + v, 0) / months.length
+        : null;
+
+    insertStmt.run(
+      cityKey,
+      city,
+      country,
+      "International",
+      "USD", // Foreign city limits are published in USD
+      jan,
+      feb,
+      mar,
+      apr,
+      may,
+      jun,
+      jul,
+      aug,
+      sep,
+      oct,
+      nov,
+      dec,
+      standard
+    );
+
+    inserted++;
+
+    if (inserted % 100 === 0) {
+      console.log(`   ... ${inserted} cities migrated`);
+    }
+  } catch (error) {
+    errors++;
+    console.error(`   ⚠️  Error migrating ${row.city}:`, error.message);
+  }
+}
+
+// Migrate per-diem rates (meal and incidentals)
+console.log("\n📥 Migrating per-diem rates...");
+const rateEntries = sourceDb
+  .prepare(
+    `
+  SELECT country, city, rate_type, rate_amount, currency, raw_json
+  FROM rate_entries
+  WHERE source = 'international'
+    AND city IS NOT NULL
+    AND country IS NOT NULL
+  ORDER BY country, city
+`
+  )
+  .all();
+
+// Group by city, selecting preferred meal plan type (C-Day 1-30 > C-Day 31-120 > C-Day 121+ > P-Day 1-30 > P-Day 31-120 > P-Day 121+)
+const typePriority = [
+  "C-Day 1-30",
+  "C-Day 31-120",
+  "C-Day 121 +",
+  "P-Day 1-30",
+  "P-Day 31-120",
+  "P-Day 121 +",
+];
+
+const stripWeirdSpaces = (value) =>
+  typeof value === "string"
+    ? value
+        .replace(/[\u00ad\u200b\u200c\u200d]/g, "") // soft hyphen & zero-widths
+        .replace(/[\u2010\u2011\u2012\u2013]/g, "-")
+    : value;
+
+const normalizeType = (value) => {
+  if (!value) return "";
+  return stripWeirdSpaces(value)
+    .replace(/\s+/g, " ")
+    .replace(/Day (\d+)\s*\+/i, "Day $1 +")
+    .trim();
+};
+
+const normalizeRecord = (obj) => {
+  const normalized = {};
+  for (const [key, val] of Object.entries(obj || {})) {
+    const cleanKey = stripWeirdSpaces(key)
+      .replace(/\s+/g, " ")
+      .trim()
+      .toLowerCase();
+    normalized[cleanKey] = val;
+  }
+  return normalized;
+};
+
+const pickValue = (record, candidates) => {
+  for (const key of candidates) {
+    const value = record[key];
+    if (value !== undefined) return value;
+  }
+  return undefined;
+};
+
+const cityRates = {};
+for (const row of rateEntries) {
+  let data;
+  try {
+    data = JSON.parse(row.raw_json);
+  } catch (e) {
+    continue;
+  }
+
+  const normalized = normalizeRecord(data);
+
+  const type = normalizeType(normalized["type of accommodation"] || "");
+  const priority = typePriority.indexOf(type);
+  const key = `${row.country}_${row.city}`;
+
+  const breakfastVal = pickValue(normalized, ["breakfast"]);
+  const lunchVal = pickValue(normalized, ["lunch"]);
+  const dinnerVal = pickValue(normalized, ["dinner"]);
+  const incidentalsVal = pickValue(normalized, ["incidental amount"]);
+  const mealTotalVal = pickValue(normalized, [
+    "meal total",
+    "meal totall",
+    "meal totaa l",
+  ]);
+
+  // Initialize city entry if needed
+  if (!cityRates[key]) {
+    cityRates[key] = {
+      country: row.country,
+      city: row.city,
+      currency: row.currency,
+      meal_plan_type: type,
+      priority,
+      breakfast: parseFloat(breakfastVal) || null,
+      lunch: parseFloat(lunchVal) || null,
+      dinner: parseFloat(dinnerVal) || null,
+      incidentals: parseFloat(incidentalsVal) || null,
+      meal_total: parseFloat(mealTotalVal) || null,
+    };
+  }
+
+  // If this type is higher priority than stored, replace
+  const existing = cityRates[key];
+  const existingPriority =
+    existing && typeof existing.priority === "number"
+      ? existing.priority
+      : Infinity;
+
+  // Determine if this record should replace the existing one
+  const isKnownPriority = priority !== -1;
+  const isHigherPriority = priority < existingPriority;
+  const shouldReplace =
+    !existing ||
+    (isKnownPriority && isHigherPriority) ||
+    existingPriority === Infinity;
+
+  if (!shouldReplace) continue;
+
+  cityRates[key] = {
+    country: row.country,
+    city: row.city,
+    currency: row.currency,
+    meal_plan_type:
+      type ||
+      existing?.meal_plan_type ||
+      (isKnownPriority && priority >= 0 ? typePriority[priority] : null),
+    priority: isKnownPriority ? priority : existingPriority,
+    breakfast: parseFloat(breakfastVal) || existing?.breakfast || null,
+    lunch: parseFloat(lunchVal) || existing?.lunch || null,
+    dinner: parseFloat(dinnerVal) || existing?.dinner || null,
+    incidentals: parseFloat(incidentalsVal) || existing?.incidentals || null,
+    meal_total: parseFloat(mealTotalVal) || existing?.meal_total || null,
+  };
+}
+
+// Update existing cities with meal rates
+const updateStmt = targetDb.prepare(`
+  UPDATE travel_rates 
+  SET breakfast = ?, lunch = ?, dinner = ?, incidentals = ?,
+      meal_currency = COALESCE(meal_currency, ?),
+      meal_plan_type = COALESCE(?, meal_plan_type),
+      meal_total = COALESCE(?, meal_total)
+  WHERE city_key = ?
+`);
+
+let updated = 0;
+for (const [key, rates] of Object.entries(cityRates)) {
+  const cityKey = `${rates.country
+    .toLowerCase()
+    .replace(/\s+/g, "_")}_${rates.city.toLowerCase().replace(/\s+/g, "_")}`;
+
+  const result = updateStmt.run(
+    rates.breakfast,
+    rates.lunch,
+    rates.dinner,
+    rates.incidentals,
+    rates.currency,
+    rates.meal_plan_type ||
+      (typeof rates.priority === "number" && rates.priority >= 0
+        ? typePriority[rates.priority]
+        : null),
+    rates.meal_total,
+    cityKey
+  );
+
+  if (result.changes > 0) {
+    updated++;
+  }
+}
+
+// Final statistics
+console.log("\n✅ Migration complete!");
+console.log(`   📊 Accommodations inserted: ${inserted}`);
+console.log(`   📊 Per-diem rates updated: ${updated}`);
+console.log(`   ⚠️  Errors: ${errors}`);
+
+// Verify Riga
+console.log("\n🔍 Verifying Riga data:");
+const riga = targetDb
+  .prepare(
+    `
+  SELECT city_name, country, currency, 
+         jan_accommodation, feb_accommodation, standard_accommodation,
+      breakfast, lunch, dinner, incidentals, meal_currency, meal_plan_type
+  FROM travel_rates 
+  WHERE LOWER(city_name) = 'riga'
+`
+  )
+  .get();
+
+if (riga) {
+  console.log("   ✅ Riga found:");
+  console.log(`      Country: ${riga.country}`);
+  console.log(`      Accommodation currency: ${riga.currency}`);
+  console.log(
+    `      Accommodation (Jan): ${riga.currency} $${riga.jan_accommodation}`
+  );
+  console.log(
+    `      Accommodation (Standard): ${riga.currency} $${riga.standard_accommodation}`
+  );
+  if (riga.meal_currency)
+    console.log(`      Meal currency: ${riga.meal_currency}`);
+  if (riga.meal_plan_type)
+    console.log(`      Meal plan: ${riga.meal_plan_type}`);
+  if (riga.breakfast)
+    console.log(
+      `      Breakfast: ${riga.meal_currency || ""} $${riga.breakfast}`
+    );
+  if (riga.lunch)
+    console.log(`      Lunch: ${riga.meal_currency || ""} $${riga.lunch}`);
+  if (riga.dinner)
+    console.log(`      Dinner: ${riga.meal_currency || ""} $${riga.dinner}`);
+} else {
+  console.log("   ⚠️  Riga NOT found in database!");
+}
+
+// Show total count
+const total = targetDb
+  .prepare("SELECT COUNT(*) as count FROM travel_rates")
+  .get();
+console.log(`\n📊 Total cities in database: ${total.count}`);
+
+sourceDb.close();
+targetDb.close();
+
+console.log("\n✅ Done!");

scripts/peekMunichKeys.js

@@ -0,0 +1,13 @@
+const Database = require("better-sqlite3");
+const path = require("path");
+const db = new Database(
+  path.join(__dirname, "..", "data", "travel_rates_scraped.sqlite3")
+);
+const row = db
+  .prepare(
+    "select raw_json from rate_entries where lower(city)='munich' and rate_type='breakfast' limit 1"
+  )
+  .get();
+const data = JSON.parse(row.raw_json);
+console.log(Object.keys(data));
+console.log(data);

scripts/peekMunichType.js

@@ -0,0 +1,12 @@
+const Database = require("better-sqlite3");
+const path = require("path");
+const db = new Database(
+  path.join(__dirname, "..", "data", "travel_rates_scraped.sqlite3")
+);
+const row = db
+  .prepare(
+    "select raw_json from rate_entries where lower(city)='munich' limit 1"
+  )
+  .get();
+const data = JSON.parse(row.raw_json);
+console.log(data["Type of Accommodation"]);

scripts/queryMunich.js

@@ -0,0 +1,16 @@
+const Database = require("better-sqlite3");
+const path = require("path");
+const db = new Database(
+  path.join(__dirname, "..", "database", "travel_rates.db")
+);
+const row = db
+  .prepare(
+    `
+    SELECT city_name, country, currency, meal_currency, meal_plan_type, meal_total,
+      breakfast, lunch, dinner, incidentals
+  FROM travel_rates
+  WHERE lower(city_name) = 'munich'
+`
+  )
+  .get();
+console.log(row);

scripts/queryMunichRates.js

@@ -0,0 +1,16 @@
+const Database = require("better-sqlite3");
+const path = require("path");
+const db = new Database(
+  path.join(__dirname, "..", "data", "travel_rates_scraped.sqlite3")
+);
+const rows = db
+  .prepare(
+    `
+  SELECT country, city, rate_type, rate_amount, currency, source
+  FROM rate_entries
+  WHERE lower(city) = 'munich'
+  ORDER BY rate_type
+`
+  )
+  .all();
+console.log(rows);

scripts/rawMunichString.js

@@ -0,0 +1,11 @@
+const Database = require("better-sqlite3");
+const path = require("path");
+const db = new Database(
+  path.join(__dirname, "..", "data", "travel_rates_scraped.sqlite3")
+);
+const row = db
+  .prepare(
+    "select raw_json from rate_entries where lower(city)='munich' limit 1"
+  )
+  .get();
+console.log(row.raw_json);

scripts/updateRiga.js

@@ -0,0 +1,41 @@
+const db = require("../services/databaseService");
+
+async function updateRigaRate() {
+  await db.connect();
+
+  return new Promise((resolve, reject) => {
+    db.db.run(
+      `
+      UPDATE travel_rates 
+      SET jan_accommodation = 209,
+          feb_accommodation = 209,
+          mar_accommodation = 209,
+          apr_accommodation = 209,
+          may_accommodation = 209,
+          jun_accommodation = 209,
+          jul_accommodation = 209,
+          aug_accommodation = 209,
+          sep_accommodation = 209,
+          oct_accommodation = 209,
+          nov_accommodation = 209,
+          dec_accommodation = 209,
+          standard_accommodation = 209,
+          currency = 'CAD'
+      WHERE LOWER(city_name) = 'riga'
+    `,
+      [],
+      (err) => {
+        if (err) {
+          console.error("❌ Error:", err);
+          reject(err);
+        } else {
+          console.log("✅ Riga accommodation updated to CAD $209");
+          resolve();
+        }
+        db.close();
+      }
+    );
+  });
+}
+
+updateRigaRate().catch(console.error);