mirror of
https://github.com/mblanke/Gov_Travel_App.git
synced 2026-03-01 06:00:21 -05:00
feat: add scripts for database inspection and migration
- Implemented multiple scripts to check and inspect meal plans, scraped data, and accommodations for Munich and Riga. - Added a migration script to convert scraped data into the application's database format. - Introduced new database service methods for querying and updating travel rates. - Enhanced server configuration for serving static files in production. - Updated PostCSS configuration for consistency.
This commit is contained in:
Binary file not shown.
@@ -264,29 +264,30 @@ async function generateFlights(originCode, destCode, departureDate) {
|
||||
|
||||
selectedAirlines.forEach((airlineCode, idx) => {
|
||||
// Calculate realistic flight duration based on stop pattern
|
||||
// NO DIRECT FLIGHTS from North America to Eastern Europe - all require stops
|
||||
let stops, stopCodes;
|
||||
let totalDuration;
|
||||
|
||||
if (idx === 0) {
|
||||
// Direct flight (if available)
|
||||
stops = 0;
|
||||
stopCodes = [];
|
||||
totalDuration = 11 + Math.random() * 3; // 11-14 hours
|
||||
} else if (idx === 1) {
|
||||
// 1 stop
|
||||
// 1 stop via London
|
||||
stops = 1;
|
||||
stopCodes = ["LHR"];
|
||||
totalDuration = 13 + Math.random() * 2; // 13-15 hours
|
||||
} else if (idx === 2) {
|
||||
// 1 stop different city
|
||||
} else if (idx === 1) {
|
||||
// 1 stop via Paris
|
||||
stops = 1;
|
||||
stopCodes = ["CDG"];
|
||||
totalDuration = 13 + Math.random() * 2;
|
||||
totalDuration = 13 + Math.random() * 2; // 13-15 hours
|
||||
} else if (idx === 2) {
|
||||
// 1 stop via Frankfurt
|
||||
stops = 1;
|
||||
stopCodes = ["FRA"];
|
||||
totalDuration = 14 + Math.random() * 2; // 14-16 hours
|
||||
} else {
|
||||
// 2 stops
|
||||
stops = 2;
|
||||
stopCodes = ["FRA", "VIE"];
|
||||
totalDuration = 15 + Math.random() * 3; // 15-18 hours
|
||||
stopCodes = ["AMS", "WAW"]; // Amsterdam + Warsaw
|
||||
totalDuration = 16 + Math.random() * 2; // 16-18 hours
|
||||
}
|
||||
|
||||
// Generate realistic departure times (6am-10am)
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
export default {
|
||||
plugins: {
|
||||
'@tailwindcss/postcss': {},
|
||||
"@tailwindcss/postcss": {},
|
||||
},
|
||||
}
|
||||
};
|
||||
|
||||
12
scripts/checkMealPlan.js
Normal file
12
scripts/checkMealPlan.js
Normal file
@@ -0,0 +1,12 @@
|
||||
const Database = require("better-sqlite3");
|
||||
const path = require("path");
|
||||
const db = new Database(
|
||||
path.join(__dirname, "..", "database", "travel_rates.db")
|
||||
);
|
||||
console.log(
|
||||
db
|
||||
.prepare(
|
||||
"select city_name, meal_plan_type from travel_rates where lower(city_name)='munich'"
|
||||
)
|
||||
.get()
|
||||
);
|
||||
39
scripts/checkScrapedData.js
Normal file
39
scripts/checkScrapedData.js
Normal file
@@ -0,0 +1,39 @@
|
||||
const Database = require("better-sqlite3");
|
||||
const db = new Database("travel_rates.db");
|
||||
|
||||
// Check tables
|
||||
const tables = db
|
||||
.prepare("SELECT name FROM sqlite_master WHERE type='table'")
|
||||
.all();
|
||||
console.log(
|
||||
"Tables in database:",
|
||||
tables.map((t) => t.name)
|
||||
);
|
||||
|
||||
// Check rate entries
|
||||
const rateCount = db
|
||||
.prepare("SELECT COUNT(*) as count FROM rate_entries")
|
||||
.get();
|
||||
console.log("\nRate entries:", rateCount.count);
|
||||
|
||||
// Check accommodations
|
||||
const accommCount = db
|
||||
.prepare("SELECT COUNT(*) as count FROM accommodations")
|
||||
.get();
|
||||
console.log("Accommodations:", accommCount.count);
|
||||
|
||||
// Check Latvia/Riga
|
||||
const latvia = db
|
||||
.prepare(
|
||||
`
|
||||
SELECT city, jan, feb, mar, apr, may, jun, jul, aug, sep, oct, nov, dec
|
||||
FROM accommodations
|
||||
WHERE country = 'Latvia'
|
||||
`
|
||||
)
|
||||
.all();
|
||||
|
||||
console.log("\nLatvia accommodations:");
|
||||
console.log(JSON.stringify(latvia, null, 2));
|
||||
|
||||
db.close();
|
||||
61
scripts/debugMunichPlan.js
Normal file
61
scripts/debugMunichPlan.js
Normal file
@@ -0,0 +1,61 @@
|
||||
const Database = require("better-sqlite3");
|
||||
const path = require("path");
|
||||
const sourceDb = new Database(
|
||||
path.join(__dirname, "..", "data", "travel_rates_scraped.sqlite3")
|
||||
);
|
||||
const typePriority = [
|
||||
"C-Day 1-30",
|
||||
"C-Day 31-120",
|
||||
"C-Day 121 +",
|
||||
"P-Day 1-30",
|
||||
"P-Day 31-120",
|
||||
"P-Day 121 +",
|
||||
];
|
||||
const rateEntries = sourceDb
|
||||
.prepare(
|
||||
`
|
||||
SELECT country, city, rate_type, rate_amount, currency, raw_json
|
||||
FROM rate_entries
|
||||
WHERE lower(city) = 'munich'
|
||||
`
|
||||
)
|
||||
.all();
|
||||
const cityRates = {};
|
||||
for (const row of rateEntries) {
|
||||
const data = JSON.parse(row.raw_json);
|
||||
const type = (data["Type of Accommodation"] || "").trim();
|
||||
const priority = typePriority.indexOf(type);
|
||||
const key = `${row.country}_${row.city}`;
|
||||
const existing = cityRates[key];
|
||||
const existingPriority =
|
||||
existing && typeof existing.priority === "number"
|
||||
? existing.priority
|
||||
: Infinity;
|
||||
const isKnownPriority = priority !== -1;
|
||||
const isHigherPriority = priority < existingPriority;
|
||||
const shouldReplace =
|
||||
!existing ||
|
||||
(isKnownPriority && isHigherPriority) ||
|
||||
existingPriority === Infinity;
|
||||
if (!shouldReplace) continue;
|
||||
cityRates[key] = {
|
||||
country: row.country,
|
||||
city: row.city,
|
||||
currency: row.currency,
|
||||
meal_plan_type:
|
||||
type ||
|
||||
existing?.meal_plan_type ||
|
||||
(isKnownPriority ? typePriority[priority] : null),
|
||||
priority: isKnownPriority ? priority : existingPriority,
|
||||
breakfast: parseFloat(data.Breakfast) || existing?.breakfast || null,
|
||||
lunch: parseFloat(data.Lunch || data["Lunch"]) || existing?.lunch || null,
|
||||
dinner: parseFloat(data.Dinner) || existing?.dinner || null,
|
||||
incidentals:
|
||||
parseFloat(data["Incidental Amount"]) || existing?.incidentals || null,
|
||||
meal_total:
|
||||
parseFloat(data["Meal Total"] || data["Meal Totaa l"]) ||
|
||||
existing?.meal_total ||
|
||||
null,
|
||||
};
|
||||
}
|
||||
console.log(cityRates);
|
||||
38
scripts/inspectFreshData.js
Normal file
38
scripts/inspectFreshData.js
Normal file
@@ -0,0 +1,38 @@
|
||||
const Database = require("better-sqlite3");
|
||||
const sourceDb = new Database("data/travel_rates_scraped.sqlite3");
|
||||
|
||||
// Check schema
|
||||
console.log("\n=== ACCOMMODATIONS SCHEMA ===");
|
||||
const accomSchema = sourceDb
|
||||
.prepare(
|
||||
"SELECT sql FROM sqlite_master WHERE type='table' AND name='accommodations'"
|
||||
)
|
||||
.get();
|
||||
console.log(accomSchema.sql);
|
||||
|
||||
// Get Latvia data
|
||||
console.log("\n=== LATVIA ACCOMMODATIONS ===");
|
||||
const latvia = sourceDb
|
||||
.prepare(
|
||||
`
|
||||
SELECT * FROM accommodations
|
||||
WHERE source_url LIKE '%Latvia%' OR raw_json LIKE '%Latvia%'
|
||||
LIMIT 5
|
||||
`
|
||||
)
|
||||
.all();
|
||||
console.log(JSON.stringify(latvia, null, 2));
|
||||
|
||||
// Search for Riga specifically
|
||||
console.log("\n=== RIGA SEARCH ===");
|
||||
const riga = sourceDb
|
||||
.prepare(
|
||||
`
|
||||
SELECT * FROM accommodations
|
||||
WHERE city LIKE '%Riga%' OR raw_json LIKE '%Riga%'
|
||||
`
|
||||
)
|
||||
.all();
|
||||
console.log(JSON.stringify(riga, null, 2));
|
||||
|
||||
sourceDb.close();
|
||||
16
scripts/inspectMunichEntry.js
Normal file
16
scripts/inspectMunichEntry.js
Normal file
@@ -0,0 +1,16 @@
|
||||
const Database = require("better-sqlite3");
|
||||
const path = require("path");
|
||||
const db = new Database(
|
||||
path.join(__dirname, "..", "data", "travel_rates_scraped.sqlite3")
|
||||
);
|
||||
const row = db
|
||||
.prepare(
|
||||
`
|
||||
SELECT rate_type, rate_amount, currency, raw_json
|
||||
FROM rate_entries
|
||||
WHERE lower(city) = 'munich' AND rate_type = 'breakfast'
|
||||
LIMIT 1
|
||||
`
|
||||
)
|
||||
.get();
|
||||
console.log(row);
|
||||
16
scripts/inspectMunichKeyCodes.js
Normal file
16
scripts/inspectMunichKeyCodes.js
Normal file
@@ -0,0 +1,16 @@
|
||||
const Database = require("better-sqlite3");
|
||||
const path = require("path");
|
||||
const db = new Database(
|
||||
path.join(__dirname, "..", "data", "travel_rates_scraped.sqlite3")
|
||||
);
|
||||
const row = db
|
||||
.prepare(
|
||||
"select raw_json from rate_entries where lower(city)='munich' limit 1"
|
||||
)
|
||||
.get();
|
||||
const data = JSON.parse(row.raw_json);
|
||||
const keys = Object.keys(data);
|
||||
for (const k of keys) {
|
||||
const codes = Array.from(k).map((c) => c.charCodeAt(0));
|
||||
console.log(k, codes);
|
||||
}
|
||||
16
scripts/inspectMunichRaw.js
Normal file
16
scripts/inspectMunichRaw.js
Normal file
@@ -0,0 +1,16 @@
|
||||
const Database = require("better-sqlite3");
|
||||
const path = require("path");
|
||||
const db = new Database(
|
||||
path.join(__dirname, "..", "data", "travel_rates_scraped.sqlite3")
|
||||
);
|
||||
const row = db
|
||||
.prepare(
|
||||
`
|
||||
SELECT country, city, table_name, raw_html, raw_json
|
||||
FROM raw_tables
|
||||
WHERE country = 'Germany' AND city = 'Munich'
|
||||
LIMIT 1
|
||||
`
|
||||
)
|
||||
.get();
|
||||
console.log(row);
|
||||
14
scripts/inspectMunichRawTables.js
Normal file
14
scripts/inspectMunichRawTables.js
Normal file
@@ -0,0 +1,14 @@
|
||||
const Database = require("better-sqlite3");
|
||||
const path = require("path");
|
||||
const db = new Database(
|
||||
path.join(__dirname, "..", "data", "travel_rates_scraped.sqlite3")
|
||||
);
|
||||
const row = db
|
||||
.prepare(
|
||||
"select title, data_json from raw_tables where data_json like '%Munich%' limit 1"
|
||||
)
|
||||
.get();
|
||||
console.log(row?.title);
|
||||
if (row?.data_json) {
|
||||
console.log(row.data_json.slice(0, 2000));
|
||||
}
|
||||
14
scripts/listMunichTypes.js
Normal file
14
scripts/listMunichTypes.js
Normal file
@@ -0,0 +1,14 @@
|
||||
const Database = require("better-sqlite3");
|
||||
const path = require("path");
|
||||
const db = new Database(
|
||||
path.join(__dirname, "..", "data", "travel_rates_scraped.sqlite3")
|
||||
);
|
||||
const rows = db
|
||||
.prepare("select raw_json from rate_entries where lower(city)='munich'")
|
||||
.all();
|
||||
const types = new Set();
|
||||
for (const r of rows) {
|
||||
const data = JSON.parse(r.raw_json);
|
||||
types.add(data["Type of Accommodation"]);
|
||||
}
|
||||
console.log(types);
|
||||
391
scripts/migrateFreshData.js
Normal file
391
scripts/migrateFreshData.js
Normal file
@@ -0,0 +1,391 @@
|
||||
/**
|
||||
* Migration script to properly convert freshly scraped data to Node.js app format
|
||||
* Source: data/travel_rates_scraped.sqlite3 (Python scraper with raw_json)
|
||||
* Target: database/travel_rates.db (Node.js app schema)
|
||||
*/
|
||||
|
||||
const Database = require("better-sqlite3");
|
||||
const path = require("path");
|
||||
|
||||
const SOURCE_DB = path.join(
|
||||
__dirname,
|
||||
"..",
|
||||
"data",
|
||||
"travel_rates_scraped.sqlite3"
|
||||
);
|
||||
const TARGET_DB = path.join(__dirname, "..", "database", "travel_rates.db");
|
||||
|
||||
console.log("🚀 Starting fresh data migration...\n");
|
||||
|
||||
// Open databases
|
||||
const sourceDb = new Database(SOURCE_DB, { readonly: true });
|
||||
const targetDb = new Database(TARGET_DB);
|
||||
|
||||
// Initialize target schema (drop and recreate to ensure clean state)
|
||||
targetDb.exec(`
|
||||
DROP TABLE IF EXISTS travel_rates;
|
||||
|
||||
CREATE TABLE travel_rates (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
city_key TEXT UNIQUE NOT NULL,
|
||||
city_name TEXT NOT NULL,
|
||||
province TEXT,
|
||||
country TEXT,
|
||||
region TEXT,
|
||||
currency TEXT NOT NULL DEFAULT 'USD', -- accommodation currency (foreign = USD)
|
||||
meal_currency TEXT, -- per-diem currency
|
||||
meal_plan_type TEXT, -- e.g., C-Day 1-30
|
||||
meal_total REAL,
|
||||
jan_accommodation REAL,
|
||||
feb_accommodation REAL,
|
||||
mar_accommodation REAL,
|
||||
apr_accommodation REAL,
|
||||
may_accommodation REAL,
|
||||
jun_accommodation REAL,
|
||||
jul_accommodation REAL,
|
||||
aug_accommodation REAL,
|
||||
sep_accommodation REAL,
|
||||
oct_accommodation REAL,
|
||||
nov_accommodation REAL,
|
||||
dec_accommodation REAL,
|
||||
standard_accommodation REAL,
|
||||
breakfast REAL,
|
||||
lunch REAL,
|
||||
dinner REAL,
|
||||
incidentals REAL,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
`);
|
||||
|
||||
// Clear existing data (already done by DROP TABLE above)
|
||||
console.log("📋 Schema created with nullable meal fields");
|
||||
|
||||
let inserted = 0;
|
||||
let errors = 0;
|
||||
|
||||
// Migrate accommodations (international rates)
|
||||
console.log("📥 Migrating international accommodations...");
|
||||
const accommodations = sourceDb
|
||||
.prepare(
|
||||
`
|
||||
SELECT city, raw_json FROM accommodations
|
||||
WHERE raw_json IS NOT NULL
|
||||
`
|
||||
)
|
||||
.all();
|
||||
|
||||
const insertStmt = targetDb.prepare(`
|
||||
INSERT OR REPLACE INTO travel_rates (
|
||||
city_key, city_name, country, region, currency,
|
||||
jan_accommodation, feb_accommodation, mar_accommodation, apr_accommodation,
|
||||
may_accommodation, jun_accommodation, jul_accommodation, aug_accommodation,
|
||||
sep_accommodation, oct_accommodation, nov_accommodation, dec_accommodation,
|
||||
standard_accommodation
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
`);
|
||||
|
||||
for (const row of accommodations) {
|
||||
try {
|
||||
const data = JSON.parse(row.raw_json);
|
||||
const country = data.Country || "";
|
||||
const city = data.City || row.city || "";
|
||||
|
||||
if (!city || !country) continue;
|
||||
|
||||
const cityKey = `${country.toLowerCase().replace(/\s+/g, "_")}_${city
|
||||
.toLowerCase()
|
||||
.replace(/\s+/g, "_")}`;
|
||||
|
||||
// Parse monthly rates
|
||||
const jan = parseFloat(data["Jan."]) || null;
|
||||
const feb = parseFloat(data["Feb."]) || null;
|
||||
const mar = parseFloat(data["Mar."]) || null;
|
||||
const apr = parseFloat(data["Apr."]) || null;
|
||||
const may = parseFloat(data["May"]) || null;
|
||||
const jun = parseFloat(data["June"]) || null;
|
||||
const jul = parseFloat(data["July"]) || null;
|
||||
const aug = parseFloat(data["Aug."]) || null;
|
||||
const sep = parseFloat(data["Sept."]) || null;
|
||||
const oct = parseFloat(data["Oct."]) || null;
|
||||
const nov = parseFloat(data["Nov."]) || null;
|
||||
const dec = parseFloat(data["Dec."]) || null;
|
||||
|
||||
// Standard accommodation = average of all available monthly values
|
||||
const months = [
|
||||
jan,
|
||||
feb,
|
||||
mar,
|
||||
apr,
|
||||
may,
|
||||
jun,
|
||||
jul,
|
||||
aug,
|
||||
sep,
|
||||
oct,
|
||||
nov,
|
||||
dec,
|
||||
].filter((v) => typeof v === "number" && !Number.isNaN(v));
|
||||
const standard =
|
||||
months.length > 0
|
||||
? months.reduce((sum, v) => sum + v, 0) / months.length
|
||||
: null;
|
||||
|
||||
insertStmt.run(
|
||||
cityKey,
|
||||
city,
|
||||
country,
|
||||
"International",
|
||||
"USD", // Foreign city limits are published in USD
|
||||
jan,
|
||||
feb,
|
||||
mar,
|
||||
apr,
|
||||
may,
|
||||
jun,
|
||||
jul,
|
||||
aug,
|
||||
sep,
|
||||
oct,
|
||||
nov,
|
||||
dec,
|
||||
standard
|
||||
);
|
||||
|
||||
inserted++;
|
||||
|
||||
if (inserted % 100 === 0) {
|
||||
console.log(` ... ${inserted} cities migrated`);
|
||||
}
|
||||
} catch (error) {
|
||||
errors++;
|
||||
console.error(` ⚠️ Error migrating ${row.city}:`, error.message);
|
||||
}
|
||||
}
|
||||
|
||||
// Migrate per-diem rates (meal and incidentals)
|
||||
console.log("\n📥 Migrating per-diem rates...");
|
||||
const rateEntries = sourceDb
|
||||
.prepare(
|
||||
`
|
||||
SELECT country, city, rate_type, rate_amount, currency, raw_json
|
||||
FROM rate_entries
|
||||
WHERE source = 'international'
|
||||
AND city IS NOT NULL
|
||||
AND country IS NOT NULL
|
||||
ORDER BY country, city
|
||||
`
|
||||
)
|
||||
.all();
|
||||
|
||||
// Group by city, selecting preferred meal plan type (C-Day 1-30 > C-Day 31-120 > C-Day 121+ > P-Day 1-30 > P-Day 31-120 > P-Day 121+)
|
||||
const typePriority = [
|
||||
"C-Day 1-30",
|
||||
"C-Day 31-120",
|
||||
"C-Day 121 +",
|
||||
"P-Day 1-30",
|
||||
"P-Day 31-120",
|
||||
"P-Day 121 +",
|
||||
];
|
||||
|
||||
const stripWeirdSpaces = (value) =>
|
||||
typeof value === "string"
|
||||
? value
|
||||
.replace(/[\u00ad\u200b\u200c\u200d]/g, "") // soft hyphen & zero-widths
|
||||
.replace(/[\u2010\u2011\u2012\u2013]/g, "-")
|
||||
: value;
|
||||
|
||||
const normalizeType = (value) => {
|
||||
if (!value) return "";
|
||||
return stripWeirdSpaces(value)
|
||||
.replace(/\s+/g, " ")
|
||||
.replace(/Day (\d+)\s*\+/i, "Day $1 +")
|
||||
.trim();
|
||||
};
|
||||
|
||||
const normalizeRecord = (obj) => {
|
||||
const normalized = {};
|
||||
for (const [key, val] of Object.entries(obj || {})) {
|
||||
const cleanKey = stripWeirdSpaces(key)
|
||||
.replace(/\s+/g, " ")
|
||||
.trim()
|
||||
.toLowerCase();
|
||||
normalized[cleanKey] = val;
|
||||
}
|
||||
return normalized;
|
||||
};
|
||||
|
||||
const pickValue = (record, candidates) => {
|
||||
for (const key of candidates) {
|
||||
const value = record[key];
|
||||
if (value !== undefined) return value;
|
||||
}
|
||||
return undefined;
|
||||
};
|
||||
|
||||
const cityRates = {};
|
||||
for (const row of rateEntries) {
|
||||
let data;
|
||||
try {
|
||||
data = JSON.parse(row.raw_json);
|
||||
} catch (e) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const normalized = normalizeRecord(data);
|
||||
|
||||
const type = normalizeType(normalized["type of accommodation"] || "");
|
||||
const priority = typePriority.indexOf(type);
|
||||
const key = `${row.country}_${row.city}`;
|
||||
|
||||
const breakfastVal = pickValue(normalized, ["breakfast"]);
|
||||
const lunchVal = pickValue(normalized, ["lunch"]);
|
||||
const dinnerVal = pickValue(normalized, ["dinner"]);
|
||||
const incidentalsVal = pickValue(normalized, ["incidental amount"]);
|
||||
const mealTotalVal = pickValue(normalized, [
|
||||
"meal total",
|
||||
"meal totall",
|
||||
"meal totaa l",
|
||||
]);
|
||||
|
||||
// Initialize city entry if needed
|
||||
if (!cityRates[key]) {
|
||||
cityRates[key] = {
|
||||
country: row.country,
|
||||
city: row.city,
|
||||
currency: row.currency,
|
||||
meal_plan_type: type,
|
||||
priority,
|
||||
breakfast: parseFloat(breakfastVal) || null,
|
||||
lunch: parseFloat(lunchVal) || null,
|
||||
dinner: parseFloat(dinnerVal) || null,
|
||||
incidentals: parseFloat(incidentalsVal) || null,
|
||||
meal_total: parseFloat(mealTotalVal) || null,
|
||||
};
|
||||
}
|
||||
|
||||
// If this type is higher priority than stored, replace
|
||||
const existing = cityRates[key];
|
||||
const existingPriority =
|
||||
existing && typeof existing.priority === "number"
|
||||
? existing.priority
|
||||
: Infinity;
|
||||
|
||||
// Determine if this record should replace the existing one
|
||||
const isKnownPriority = priority !== -1;
|
||||
const isHigherPriority = priority < existingPriority;
|
||||
const shouldReplace =
|
||||
!existing ||
|
||||
(isKnownPriority && isHigherPriority) ||
|
||||
existingPriority === Infinity;
|
||||
|
||||
if (!shouldReplace) continue;
|
||||
|
||||
cityRates[key] = {
|
||||
country: row.country,
|
||||
city: row.city,
|
||||
currency: row.currency,
|
||||
meal_plan_type:
|
||||
type ||
|
||||
existing?.meal_plan_type ||
|
||||
(isKnownPriority && priority >= 0 ? typePriority[priority] : null),
|
||||
priority: isKnownPriority ? priority : existingPriority,
|
||||
breakfast: parseFloat(breakfastVal) || existing?.breakfast || null,
|
||||
lunch: parseFloat(lunchVal) || existing?.lunch || null,
|
||||
dinner: parseFloat(dinnerVal) || existing?.dinner || null,
|
||||
incidentals: parseFloat(incidentalsVal) || existing?.incidentals || null,
|
||||
meal_total: parseFloat(mealTotalVal) || existing?.meal_total || null,
|
||||
};
|
||||
}
|
||||
|
||||
// Update existing cities with meal rates
|
||||
const updateStmt = targetDb.prepare(`
|
||||
UPDATE travel_rates
|
||||
SET breakfast = ?, lunch = ?, dinner = ?, incidentals = ?,
|
||||
meal_currency = COALESCE(meal_currency, ?),
|
||||
meal_plan_type = COALESCE(?, meal_plan_type),
|
||||
meal_total = COALESCE(?, meal_total)
|
||||
WHERE city_key = ?
|
||||
`);
|
||||
|
||||
let updated = 0;
|
||||
for (const [key, rates] of Object.entries(cityRates)) {
|
||||
const cityKey = `${rates.country
|
||||
.toLowerCase()
|
||||
.replace(/\s+/g, "_")}_${rates.city.toLowerCase().replace(/\s+/g, "_")}`;
|
||||
|
||||
const result = updateStmt.run(
|
||||
rates.breakfast,
|
||||
rates.lunch,
|
||||
rates.dinner,
|
||||
rates.incidentals,
|
||||
rates.currency,
|
||||
rates.meal_plan_type ||
|
||||
(typeof rates.priority === "number" && rates.priority >= 0
|
||||
? typePriority[rates.priority]
|
||||
: null),
|
||||
rates.meal_total,
|
||||
cityKey
|
||||
);
|
||||
|
||||
if (result.changes > 0) {
|
||||
updated++;
|
||||
}
|
||||
}
|
||||
|
||||
// Final statistics
|
||||
console.log("\n✅ Migration complete!");
|
||||
console.log(` 📊 Accommodations inserted: ${inserted}`);
|
||||
console.log(` 📊 Per-diem rates updated: ${updated}`);
|
||||
console.log(` ⚠️ Errors: ${errors}`);
|
||||
|
||||
// Verify Riga
|
||||
console.log("\n🔍 Verifying Riga data:");
|
||||
const riga = targetDb
|
||||
.prepare(
|
||||
`
|
||||
SELECT city_name, country, currency,
|
||||
jan_accommodation, feb_accommodation, standard_accommodation,
|
||||
breakfast, lunch, dinner, incidentals, meal_currency, meal_plan_type
|
||||
FROM travel_rates
|
||||
WHERE LOWER(city_name) = 'riga'
|
||||
`
|
||||
)
|
||||
.get();
|
||||
|
||||
if (riga) {
|
||||
console.log(" ✅ Riga found:");
|
||||
console.log(` Country: ${riga.country}`);
|
||||
console.log(` Accommodation currency: ${riga.currency}`);
|
||||
console.log(
|
||||
` Accommodation (Jan): ${riga.currency} $${riga.jan_accommodation}`
|
||||
);
|
||||
console.log(
|
||||
` Accommodation (Standard): ${riga.currency} $${riga.standard_accommodation}`
|
||||
);
|
||||
if (riga.meal_currency)
|
||||
console.log(` Meal currency: ${riga.meal_currency}`);
|
||||
if (riga.meal_plan_type)
|
||||
console.log(` Meal plan: ${riga.meal_plan_type}`);
|
||||
if (riga.breakfast)
|
||||
console.log(
|
||||
` Breakfast: ${riga.meal_currency || ""} $${riga.breakfast}`
|
||||
);
|
||||
if (riga.lunch)
|
||||
console.log(` Lunch: ${riga.meal_currency || ""} $${riga.lunch}`);
|
||||
if (riga.dinner)
|
||||
console.log(` Dinner: ${riga.meal_currency || ""} $${riga.dinner}`);
|
||||
} else {
|
||||
console.log(" ⚠️ Riga NOT found in database!");
|
||||
}
|
||||
|
||||
// Show total count
|
||||
const total = targetDb
|
||||
.prepare("SELECT COUNT(*) as count FROM travel_rates")
|
||||
.get();
|
||||
console.log(`\n📊 Total cities in database: ${total.count}`);
|
||||
|
||||
sourceDb.close();
|
||||
targetDb.close();
|
||||
|
||||
console.log("\n✅ Done!");
|
||||
13
scripts/peekMunichKeys.js
Normal file
13
scripts/peekMunichKeys.js
Normal file
@@ -0,0 +1,13 @@
|
||||
const Database = require("better-sqlite3");
|
||||
const path = require("path");
|
||||
const db = new Database(
|
||||
path.join(__dirname, "..", "data", "travel_rates_scraped.sqlite3")
|
||||
);
|
||||
const row = db
|
||||
.prepare(
|
||||
"select raw_json from rate_entries where lower(city)='munich' and rate_type='breakfast' limit 1"
|
||||
)
|
||||
.get();
|
||||
const data = JSON.parse(row.raw_json);
|
||||
console.log(Object.keys(data));
|
||||
console.log(data);
|
||||
12
scripts/peekMunichType.js
Normal file
12
scripts/peekMunichType.js
Normal file
@@ -0,0 +1,12 @@
|
||||
const Database = require("better-sqlite3");
|
||||
const path = require("path");
|
||||
const db = new Database(
|
||||
path.join(__dirname, "..", "data", "travel_rates_scraped.sqlite3")
|
||||
);
|
||||
const row = db
|
||||
.prepare(
|
||||
"select raw_json from rate_entries where lower(city)='munich' limit 1"
|
||||
)
|
||||
.get();
|
||||
const data = JSON.parse(row.raw_json);
|
||||
console.log(data["Type of Accommodation"]);
|
||||
16
scripts/queryMunich.js
Normal file
16
scripts/queryMunich.js
Normal file
@@ -0,0 +1,16 @@
|
||||
const Database = require("better-sqlite3");
|
||||
const path = require("path");
|
||||
const db = new Database(
|
||||
path.join(__dirname, "..", "database", "travel_rates.db")
|
||||
);
|
||||
const row = db
|
||||
.prepare(
|
||||
`
|
||||
SELECT city_name, country, currency, meal_currency, meal_plan_type, meal_total,
|
||||
breakfast, lunch, dinner, incidentals
|
||||
FROM travel_rates
|
||||
WHERE lower(city_name) = 'munich'
|
||||
`
|
||||
)
|
||||
.get();
|
||||
console.log(row);
|
||||
16
scripts/queryMunichRates.js
Normal file
16
scripts/queryMunichRates.js
Normal file
@@ -0,0 +1,16 @@
|
||||
const Database = require("better-sqlite3");
|
||||
const path = require("path");
|
||||
const db = new Database(
|
||||
path.join(__dirname, "..", "data", "travel_rates_scraped.sqlite3")
|
||||
);
|
||||
const rows = db
|
||||
.prepare(
|
||||
`
|
||||
SELECT country, city, rate_type, rate_amount, currency, source
|
||||
FROM rate_entries
|
||||
WHERE lower(city) = 'munich'
|
||||
ORDER BY rate_type
|
||||
`
|
||||
)
|
||||
.all();
|
||||
console.log(rows);
|
||||
11
scripts/rawMunichString.js
Normal file
11
scripts/rawMunichString.js
Normal file
@@ -0,0 +1,11 @@
|
||||
const Database = require("better-sqlite3");
|
||||
const path = require("path");
|
||||
const db = new Database(
|
||||
path.join(__dirname, "..", "data", "travel_rates_scraped.sqlite3")
|
||||
);
|
||||
const row = db
|
||||
.prepare(
|
||||
"select raw_json from rate_entries where lower(city)='munich' limit 1"
|
||||
)
|
||||
.get();
|
||||
console.log(row.raw_json);
|
||||
41
scripts/updateRiga.js
Normal file
41
scripts/updateRiga.js
Normal file
@@ -0,0 +1,41 @@
|
||||
const db = require("../services/databaseService");
|
||||
|
||||
async function updateRigaRate() {
|
||||
await db.connect();
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
db.db.run(
|
||||
`
|
||||
UPDATE travel_rates
|
||||
SET jan_accommodation = 209,
|
||||
feb_accommodation = 209,
|
||||
mar_accommodation = 209,
|
||||
apr_accommodation = 209,
|
||||
may_accommodation = 209,
|
||||
jun_accommodation = 209,
|
||||
jul_accommodation = 209,
|
||||
aug_accommodation = 209,
|
||||
sep_accommodation = 209,
|
||||
oct_accommodation = 209,
|
||||
nov_accommodation = 209,
|
||||
dec_accommodation = 209,
|
||||
standard_accommodation = 209,
|
||||
currency = 'CAD'
|
||||
WHERE LOWER(city_name) = 'riga'
|
||||
`,
|
||||
[],
|
||||
(err) => {
|
||||
if (err) {
|
||||
console.error("❌ Error:", err);
|
||||
reject(err);
|
||||
} else {
|
||||
console.log("✅ Riga accommodation updated to CAD $209");
|
||||
resolve();
|
||||
}
|
||||
db.close();
|
||||
}
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
updateRigaRate().catch(console.error);
|
||||
13
server.js
13
server.js
@@ -81,12 +81,17 @@ app.use((req, res, next) => {
|
||||
});
|
||||
|
||||
// Serve React app (production build) or legacy static files
|
||||
if (process.env.NODE_ENV === 'production' && require('fs').existsSync(path.join(__dirname, 'dist', 'client'))) {
|
||||
if (
|
||||
process.env.NODE_ENV === "production" &&
|
||||
require("fs").existsSync(path.join(__dirname, "dist", "client"))
|
||||
) {
|
||||
// Serve React production build
|
||||
app.use(express.static(path.join(__dirname, 'dist', 'client'), {
|
||||
maxAge: '1d',
|
||||
app.use(
|
||||
express.static(path.join(__dirname, "dist", "client"), {
|
||||
maxAge: "1d",
|
||||
etag: true,
|
||||
}));
|
||||
})
|
||||
);
|
||||
} else {
|
||||
// Serve legacy static files from the current directory
|
||||
app.use(
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
const sqlite3 = require('sqlite3').verbose();
|
||||
const path = require('path');
|
||||
const sqlite3 = require("sqlite3").verbose();
|
||||
const path = require("path");
|
||||
|
||||
class DatabaseService {
|
||||
constructor() {
|
||||
this.dbPath = path.join(__dirname, '..', 'database', 'travel_rates.db');
|
||||
this.dbPath = path.join(__dirname, "..", "database", "travel_rates.db");
|
||||
this.db = null;
|
||||
}
|
||||
|
||||
@@ -11,10 +11,10 @@ class DatabaseService {
|
||||
return new Promise((resolve, reject) => {
|
||||
this.db = new sqlite3.Database(this.dbPath, (err) => {
|
||||
if (err) {
|
||||
console.error('❌ Database connection failed:', err);
|
||||
console.error("❌ Database connection failed:", err);
|
||||
reject(err);
|
||||
} else {
|
||||
console.log('✅ Database connected');
|
||||
console.log("✅ Database connected");
|
||||
resolve();
|
||||
}
|
||||
});
|
||||
@@ -47,10 +47,15 @@ class DatabaseService {
|
||||
const likeTerm = `${searchTerm.toLowerCase()}%`;
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
this.db.all(query, [term, term, term, term, exactTerm, exactTerm, likeTerm], (err, rows) => {
|
||||
this.db.all(
|
||||
query,
|
||||
[term, term, term, term, exactTerm, exactTerm, likeTerm],
|
||||
(err, rows) => {
|
||||
if (err) reject(err);
|
||||
else resolve(rows ? rows.map(row => this.formatTravelRate(row)) : []);
|
||||
});
|
||||
else
|
||||
resolve(rows ? rows.map((row) => this.formatTravelRate(row)) : []);
|
||||
}
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
@@ -81,7 +86,7 @@ class DatabaseService {
|
||||
city: rate.name,
|
||||
month: month,
|
||||
rate: rate.monthlyRates[monthIndex],
|
||||
currency: rate.currency
|
||||
currency: rate.currency,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -106,7 +111,7 @@ class DatabaseService {
|
||||
return new Promise((resolve, reject) => {
|
||||
this.db.all(query, [searchTerm, searchTerm], (err, rows) => {
|
||||
if (err) reject(err);
|
||||
else resolve(rows.map(row => this.formatTravelRate(row)));
|
||||
else resolve(rows.map((row) => this.formatTravelRate(row)));
|
||||
});
|
||||
});
|
||||
}
|
||||
@@ -124,23 +129,33 @@ class DatabaseService {
|
||||
currency: row.currency,
|
||||
accommodation: {
|
||||
monthly: [
|
||||
row.jan_accommodation, row.feb_accommodation, row.mar_accommodation,
|
||||
row.apr_accommodation, row.may_accommodation, row.jun_accommodation,
|
||||
row.jul_accommodation, row.aug_accommodation, row.sep_accommodation,
|
||||
row.oct_accommodation, row.nov_accommodation, row.dec_accommodation
|
||||
row.jan_accommodation,
|
||||
row.feb_accommodation,
|
||||
row.mar_accommodation,
|
||||
row.apr_accommodation,
|
||||
row.may_accommodation,
|
||||
row.jun_accommodation,
|
||||
row.jul_accommodation,
|
||||
row.aug_accommodation,
|
||||
row.sep_accommodation,
|
||||
row.oct_accommodation,
|
||||
row.nov_accommodation,
|
||||
row.dec_accommodation,
|
||||
],
|
||||
standard: row.standard_accommodation
|
||||
standard: row.standard_accommodation,
|
||||
},
|
||||
meals: {
|
||||
breakfast: row.breakfast,
|
||||
lunch: row.lunch,
|
||||
dinner: row.dinner,
|
||||
total: row.total_meals
|
||||
total: row.total_meals,
|
||||
},
|
||||
incidentals: row.incidentals,
|
||||
totalDailyAllowance: row.total_daily_allowance,
|
||||
fullDayCost: parseFloat(row.standard_accommodation || row.jan_accommodation) + parseFloat(row.total_daily_allowance),
|
||||
isInternational: row.is_international === 1
|
||||
fullDayCost:
|
||||
parseFloat(row.standard_accommodation || row.jan_accommodation) +
|
||||
parseFloat(row.total_daily_allowance),
|
||||
isInternational: row.is_international === 1,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -167,11 +182,11 @@ class DatabaseService {
|
||||
row.sep_accommodation || row.sep_rate,
|
||||
row.oct_accommodation || row.oct_rate,
|
||||
row.nov_accommodation || row.nov_rate,
|
||||
row.dec_accommodation || row.dec_rate
|
||||
row.dec_accommodation || row.dec_rate,
|
||||
],
|
||||
standardRate: row.standard_accommodation || row.standard_rate,
|
||||
isInternational: row.is_international === 1,
|
||||
effectiveDate: row.effective_date
|
||||
effectiveDate: row.effective_date,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -188,7 +203,7 @@ class DatabaseService {
|
||||
return new Promise((resolve, reject) => {
|
||||
this.db.all(query, [region], (err, rows) => {
|
||||
if (err) reject(err);
|
||||
else resolve(rows.map(row => this.formatAccommodationRate(row)));
|
||||
else resolve(rows.map((row) => this.formatAccommodationRate(row)));
|
||||
});
|
||||
});
|
||||
}
|
||||
@@ -206,7 +221,7 @@ class DatabaseService {
|
||||
return new Promise((resolve, reject) => {
|
||||
this.db.all(query, [country], (err, rows) => {
|
||||
if (err) reject(err);
|
||||
else resolve(rows.map(row => this.formatAccommodationRate(row)));
|
||||
else resolve(rows.map((row) => this.formatAccommodationRate(row)));
|
||||
});
|
||||
});
|
||||
}
|
||||
@@ -220,7 +235,7 @@ class DatabaseService {
|
||||
return new Promise((resolve, reject) => {
|
||||
this.db.all(query, [], (err, rows) => {
|
||||
if (err) reject(err);
|
||||
else resolve(rows.map(row => row.region));
|
||||
else resolve(rows.map((row) => row.region));
|
||||
});
|
||||
});
|
||||
}
|
||||
@@ -234,7 +249,7 @@ class DatabaseService {
|
||||
return new Promise((resolve, reject) => {
|
||||
this.db.all(query, [], (err, rows) => {
|
||||
if (err) reject(err);
|
||||
else resolve(rows.map(row => row.country));
|
||||
else resolve(rows.map((row) => row.country));
|
||||
});
|
||||
});
|
||||
}
|
||||
@@ -332,7 +347,7 @@ class DatabaseService {
|
||||
close() {
|
||||
if (this.db) {
|
||||
this.db.close();
|
||||
console.log('✅ Database connection closed');
|
||||
console.log("✅ Database connection closed");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user