Files
vendoo/lib/operations-center.mjs
T
Masterluke77andGitHub 6f48827f4d Vendoo 1.41.2 – Git Ignore Boundary & Module Tracking Hotfix (#18)
* docs: prepare Vendoo 1.41.2 git-ignore boundary hotfix

* fix: track native source modules with root-anchored runtime ignores
2026-07-09 17:09:00 +02:00

402 lines
23 KiB
JavaScript

import Database from 'better-sqlite3';
import { createHash, randomUUID } from 'crypto';
import {
existsSync, mkdirSync, readFileSync, writeFileSync, readdirSync, statSync,
unlinkSync, copyFileSync, rmSync,
} from 'fs';
import { join, relative, dirname, basename, resolve, sep } from 'path';
import { db } from './db.mjs';
import { APP_ROOT, DATA_ROOT, DB_PATH, UPLOADS_DIR, BACKUP_DIR, OPERATIONS_DIR, CONFIG_PATH, MODULES_DIR, MODULE_STATE_PATH, APP_VERSION } from './runtime-paths.mjs';
import { createZip } from './zip.mjs';
import { readZip } from './archive.mjs';
const AUTO_BACKUP_DIR = join(BACKUP_DIR, 'automatic');
const MANUAL_BACKUP_DIR = join(BACKUP_DIR, 'manual');
const STAGING_DIR = join(OPERATIONS_DIR, 'staging');
const UPDATE_DIR = join(OPERATIONS_DIR, 'updates');
const PENDING_FILE = join(OPERATIONS_DIR, 'pending-operation.json');
const LAST_OPERATION_FILE = join(OPERATIONS_DIR, 'last-operation.json');
for (const directory of [BACKUP_DIR, AUTO_BACKUP_DIR, MANUAL_BACKUP_DIR, STAGING_DIR, UPDATE_DIR, dirname(PENDING_FILE)]) {
mkdirSync(directory, { recursive: true });
}
function nowIso() { return new Date().toISOString(); }
function localDateKey(date = new Date()) { return `${date.getFullYear()}-${String(date.getMonth() + 1).padStart(2, '0')}-${String(date.getDate()).padStart(2, '0')}`; }
function safeJson(value, fallback = {}) { try { return JSON.parse(value); } catch { return fallback; } }
function hash(data) { return createHash('sha256').update(data).digest('hex'); }
function cleanLabel(value) { return String(value || '').trim().replace(/[^a-zA-Z0-9äöüÄÖÜß._ -]+/g, '').slice(0, 60); }
function portable(path) { return String(path || '').replace(/\\/g, '/'); }
function fileSize(path) { try { return statSync(path).size; } catch { return 0; } }
function isInsideAllowedRoot(path) {
const resolvedPath = resolve(path);
return [APP_ROOT, DATA_ROOT].some(root => resolvedPath === resolve(root) || resolvedPath.startsWith(resolve(root) + sep));
}
function ensureInsideAllowedRoot(path) { if (!isInsideAllowedRoot(path)) throw new Error('Pfad liegt außerhalb der erlaubten Vendoo-Verzeichnisse.'); return path; }
function walkFiles(base, prefix = '') {
const result = [];
if (!existsSync(base)) return result;
for (const name of readdirSync(base)) {
const absolute = join(base, name);
const relativeName = portable(join(prefix, name));
const stat = statSync(absolute);
if (stat.isDirectory()) result.push(...walkFiles(absolute, relativeName));
else if (stat.isFile()) result.push({ absolute, name: relativeName, size: stat.size });
}
return result;
}
function getOperationSettings() {
const rows = db.prepare('SELECT key, value FROM operation_settings').all();
const values = Object.fromEntries(rows.map(row => [row.key, row.value]));
return {
auto_backup_enabled: values.auto_backup_enabled === '1',
auto_backup_hour: Math.max(0, Math.min(23, Number(values.auto_backup_hour ?? 3))),
auto_backup_retention: Math.max(1, Math.min(30, Number(values.auto_backup_retention ?? 7))),
include_uploads_automatic: values.include_uploads_automatic !== '0',
update_manifest_url: values.update_manifest_url || process.env.VENDOO_UPDATE_MANIFEST_URL || '',
last_auto_backup_date: values.last_auto_backup_date || '',
};
}
export function updateOperationSettings(patch = {}) {
const allowed = new Set(['auto_backup_enabled', 'auto_backup_hour', 'auto_backup_retention', 'include_uploads_automatic', 'update_manifest_url']);
const stmt = db.prepare(`INSERT INTO operation_settings (key, value, updated_at) VALUES (?, ?, datetime('now'))
ON CONFLICT(key) DO UPDATE SET value = excluded.value, updated_at = datetime('now')`);
const tx = db.transaction(() => {
for (const [key, raw] of Object.entries(patch || {})) {
if (!allowed.has(key)) continue;
let value = raw;
if (['auto_backup_enabled', 'include_uploads_automatic'].includes(key)) value = raw ? '1' : '0';
if (key === 'auto_backup_hour') value = String(Math.max(0, Math.min(23, Number(raw || 0))));
if (key === 'auto_backup_retention') value = String(Math.max(1, Math.min(30, Number(raw || 7))));
if (key === 'update_manifest_url') value = String(raw || '').trim().slice(0, 1000);
stmt.run(key, String(value));
}
});
tx();
return getOperationSettings();
}
async function createDatabaseSnapshot(destination) {
try { db.pragma('wal_checkpoint(PASSIVE)'); } catch {}
await db.backup(destination);
const checkDb = new Database(destination, { readonly: true });
try {
const integrity = checkDb.pragma('integrity_check', { simple: true });
if (integrity !== 'ok') throw new Error(`SQLite-Integritätsprüfung: ${integrity}`);
} finally { checkDb.close(); }
}
function backupFileRows() {
return db.prepare('SELECT * FROM operation_backups ORDER BY created_at DESC, id DESC').all().map(row => {
const { file_path: filePath, options_json: _optionsJson, manifest_json: _manifestJson, ...safeRow } = row;
return {
...safeRow,
options: safeJson(row.options_json),
manifest: safeJson(row.manifest_json),
exists: existsSync(filePath),
};
});
}
function rotateAutomaticBackups(retention) {
const rows = db.prepare("SELECT * FROM operation_backups WHERE kind = 'automatic' ORDER BY created_at DESC, id DESC").all();
for (const row of rows.slice(retention)) {
try { if (existsSync(row.file_path)) unlinkSync(row.file_path); } catch {}
db.prepare('DELETE FROM operation_backups WHERE id = ?').run(row.id);
}
}
export async function createBackup({ kind = 'manual', includeUploads = true, includeConfig = false, label = '', automatic = false } = {}) {
const id = randomUUID();
const createdAt = nowIso();
const safeKind = automatic ? 'automatic' : (kind === 'database' ? 'database' : 'manual');
const directory = automatic ? AUTO_BACKUP_DIR : MANUAL_BACKUP_DIR;
const stamp = createdAt.replace(/[:.]/g, '-');
const suffix = cleanLabel(label) ? `-${cleanLabel(label).replace(/\s+/g, '-')}` : '';
const filename = `vendoo-${safeKind}-${stamp}${suffix}.zip`;
const outputPath = join(directory, filename);
const tempDb = join(STAGING_DIR, `backup-${id}.db`);
await createDatabaseSnapshot(tempDb);
const sourceFiles = [{ absolute: tempDb, name: 'db/vendoo.db', size: fileSize(tempDb), role: 'database' }];
if (includeUploads) sourceFiles.push(...walkFiles(UPLOADS_DIR, 'uploads').map(file => ({ ...file, role: 'upload' })));
sourceFiles.push(...walkFiles(MODULES_DIR, 'modules').map(file => ({ ...file, role: 'module-package' })));
if (existsSync(MODULE_STATE_PATH)) sourceFiles.push({ absolute: MODULE_STATE_PATH, name: 'module-state/modules-state.json', size: fileSize(MODULE_STATE_PATH), role: 'module-state' });
if (includeConfig) {
const configRoot = dirname(CONFIG_PATH);
sourceFiles.push(...walkFiles(configRoot, 'config').map(file => ({ ...file, role: 'config-secret' })));
if (existsSync(join(APP_ROOT, '.env'))) sourceFiles.push({ absolute: join(APP_ROOT, '.env'), name: 'config/.env', size: fileSize(join(APP_ROOT, '.env')), role: 'config-secret' });
}
for (const name of ['package.json', '.env.example']) {
const absolute = join(APP_ROOT, name);
if (existsSync(absolute)) sourceFiles.push({ absolute, name: `metadata/${name}`, size: fileSize(absolute), role: 'metadata' });
}
const uniqueSourceFiles = [...new Map(sourceFiles.map(file => [file.name, file])).values()];
sourceFiles.length = 0;
sourceFiles.push(...uniqueSourceFiles);
const maxBytes = 2 * 1024 * 1024 * 1024;
const totalBytes = sourceFiles.reduce((sum, file) => sum + file.size, 0);
if (totalBytes > maxBytes) {
rmSync(tempDb, { force: true });
throw new Error('Backup überschreitet 2 GB. Bitte Uploads separat sichern oder alte Dateien bereinigen.');
}
const payloadFiles = [];
const manifestFiles = [];
for (const file of sourceFiles) {
const data = readFileSync(ensureInsideAllowedRoot(file.absolute));
payloadFiles.push({ name: file.name, data });
manifestFiles.push({ path: file.name, role: file.role, size: data.length, sha256: hash(data) });
}
const manifest = {
format: 'vendoo-backup-v1', version: APP_VERSION, id, kind: safeKind, created_at: createdAt,
include_uploads: Boolean(includeUploads), include_config: Boolean(includeConfig),
file_count: manifestFiles.length, total_uncompressed_bytes: totalBytes, files: manifestFiles,
};
payloadFiles.unshift({ name: 'backup-manifest.json', data: Buffer.from(JSON.stringify(manifest, null, 2), 'utf8') });
const archive = createZip(payloadFiles);
writeFileSync(outputPath, archive);
rmSync(tempDb, { force: true });
const verification = verifyBackupBuffer(archive);
if (!verification.ok) {
rmSync(outputPath, { force: true });
throw new Error(`Backup-Prüfung fehlgeschlagen: ${verification.errors.join('; ')}`);
}
db.prepare(`INSERT INTO operation_backups
(id, kind, label, file_name, file_path, file_size, sha256, status, verified_at, options_json, manifest_json, created_at)
VALUES (?, ?, ?, ?, ?, ?, ?, 'verified', datetime('now'), ?, ?, datetime('now'))`).run(
id, safeKind, cleanLabel(label), filename, outputPath, fileSize(outputPath), hash(archive),
JSON.stringify({ includeUploads: Boolean(includeUploads), includeConfig: Boolean(includeConfig) }), JSON.stringify(manifest),
);
if (automatic) rotateAutomaticBackups(getOperationSettings().auto_backup_retention);
return { id, filename, file_size: fileSize(outputPath), sha256: hash(archive), manifest, verification };
}
export function verifyBackupBuffer(buffer) {
const errors = [];
const warnings = [];
let manifest = null;
try {
const { files } = readZip(buffer, { maxEntries: 20000, maxUncompressedBytes: 3 * 1024 * 1024 * 1024 });
const manifestBuffer = files.get('backup-manifest.json');
if (!manifestBuffer) throw new Error('backup-manifest.json fehlt.');
manifest = JSON.parse(manifestBuffer.toString('utf8'));
if (manifest.format !== 'vendoo-backup-v1') errors.push('Unbekanntes Backup-Format.');
for (const entry of manifest.files || []) {
const data = files.get(entry.path);
if (!data) { errors.push(`Datei fehlt: ${entry.path}`); continue; }
if (data.length !== Number(entry.size)) errors.push(`Dateigröße stimmt nicht: ${entry.path}`);
if (hash(data) !== entry.sha256) errors.push(`Prüfsumme stimmt nicht: ${entry.path}`);
}
const dbData = files.get('db/vendoo.db');
if (!dbData) errors.push('Datenbank fehlt im Backup.');
else {
const temp = join(STAGING_DIR, `verify-${randomUUID()}.db`);
writeFileSync(temp, dbData);
try {
const verifyDb = new Database(temp, { readonly: true });
const integrity = verifyDb.pragma('integrity_check', { simple: true });
verifyDb.close();
if (integrity !== 'ok') errors.push(`SQLite-Integrität: ${integrity}`);
} catch (error) { errors.push(`Datenbank kann nicht geöffnet werden: ${error.message}`); }
finally { rmSync(temp, { force: true }); }
}
if (manifest.include_config) warnings.push('Dieses Backup enthält verschlüsselte Secrets, Master-Key und gegebenenfalls .env. Sicher aufbewahren.');
} catch (error) { errors.push(error.message); }
return { ok: errors.length === 0, errors, warnings, manifest };
}
export function verifyStoredBackup(id) {
const row = db.prepare('SELECT * FROM operation_backups WHERE id = ?').get(id);
if (!row || !existsSync(row.file_path)) throw new Error('Backup wurde nicht gefunden.');
const buffer = readFileSync(row.file_path);
const result = verifyBackupBuffer(buffer);
db.prepare(`UPDATE operation_backups SET status = ?, verified_at = datetime('now'), sha256 = ?, file_size = ?, manifest_json = ? WHERE id = ?`).run(
result.ok ? 'verified' : 'invalid', hash(buffer), buffer.length, JSON.stringify(result.manifest || {}), id,
);
return result;
}
export function listBackups() { return backupFileRows(); }
export function getBackupPath(id) {
const row = db.prepare('SELECT * FROM operation_backups WHERE id = ?').get(id);
if (!row || !existsSync(row.file_path)) throw new Error('Backup wurde nicht gefunden.');
return row;
}
export function deleteBackup(id) {
const row = db.prepare('SELECT * FROM operation_backups WHERE id = ?').get(id);
if (!row) return { deleted: 0 };
try { if (existsSync(row.file_path)) unlinkSync(row.file_path); } catch {}
const result = db.prepare('DELETE FROM operation_backups WHERE id = ?').run(id);
return { deleted: Number(result.changes || 0) };
}
function writePendingOperation(operation) {
if (existsSync(PENDING_FILE)) throw new Error('Es ist bereits eine ausstehende Systemoperation vorhanden.');
writeFileSync(PENDING_FILE, JSON.stringify({ ...operation, created_at: nowIso(), app_version: APP_VERSION }, null, 2), 'utf8');
return { ...operation, pending_file: PENDING_FILE };
}
export async function stageRestore(buffer, originalName = 'backup.zip') {
if (existsSync(PENDING_FILE)) throw new Error('Es ist bereits eine ausstehende Systemoperation vorhanden.');
const verification = verifyBackupBuffer(buffer);
if (!verification.ok) throw new Error(`Backup ist ungültig: ${verification.errors.join('; ')}`);
const safety = await createBackup({ kind: 'manual', includeUploads: true, includeConfig: false, label: 'vor-wiederherstellung' });
const id = randomUUID();
const archivePath = join(STAGING_DIR, `restore-${id}.zip`);
writeFileSync(archivePath, buffer);
return writePendingOperation({ id, type: 'restore', archive_path: archivePath, original_name: basename(originalName), safety_backup_id: safety.id, verified: true, manifest: verification.manifest });
}
function versionParts(version) { return String(version || '0').replace(/^v/, '').split(/[.-]/).map(part => Number(part) || 0); }
function compareVersions(left, right) {
const a = versionParts(left); const b = versionParts(right);
for (let i = 0; i < Math.max(a.length, b.length); i++) { if ((a[i] || 0) !== (b[i] || 0)) return (a[i] || 0) > (b[i] || 0) ? 1 : -1; }
return 0;
}
function inspectUpdateArchive(buffer) {
const { entries, files } = readZip(buffer, { maxEntries: 30000, maxUncompressedBytes: 4 * 1024 * 1024 * 1024 });
const packageNames = [...files.keys()].filter(name => name === 'package.json' || name.endsWith('/vendoo/package.json') || /\/package\.json$/.test(name));
if (!packageNames.length) throw new Error('Kein Vendoo-package.json im Update gefunden.');
packageNames.sort((a, b) => a.split('/').length - b.split('/').length);
const packagePath = packageNames[0];
const packageData = JSON.parse(files.get(packagePath).toString('utf8'));
if (packageData.name !== 'vendoo') throw new Error('Das Paket ist kein Vendoo-Update.');
const rootPrefix = packagePath.slice(0, -'package.json'.length);
const relativeNames = entries.filter(entry => entry.name.startsWith(rootPrefix)).map(entry => entry.name.slice(rootPrefix.length));
const forbidden = relativeNames.filter(name =>
name === '.env' || name.startsWith('db/vendoo.db') || name.startsWith('uploads/') ||
name.startsWith('node_modules/') || name.startsWith('backups/') || name.startsWith('operations/') ||
name.startsWith('logs/') || name.startsWith('local-ai/comfyui/') || name.startsWith('local-ai/downloads/')
);
if (forbidden.length) throw new Error(`Update enthält verbotene Laufzeitdaten: ${forbidden.slice(0, 5).join(', ')}`);
for (const required of ['server.mjs', 'public/index.html', 'public/app.js', 'public/style.css', 'setup.ps1']) {
if (!files.has(rootPrefix + required)) throw new Error(`Pflichtdatei fehlt im Update: ${required}`);
}
return { version: packageData.version, package: packageData, root_prefix: rootPrefix, entries: relativeNames.length };
}
export async function stageUpdate(buffer, originalName = 'vendoo-update.zip') {
if (existsSync(PENDING_FILE)) throw new Error('Es ist bereits eine ausstehende Systemoperation vorhanden.');
const inspection = inspectUpdateArchive(buffer);
if (compareVersions(inspection.version, APP_VERSION) <= 0) throw new Error(`Update-Version ${inspection.version} ist nicht neuer als ${APP_VERSION}.`);
const safety = await createBackup({ kind: 'manual', includeUploads: true, includeConfig: false, label: `vor-update-${inspection.version}` });
const id = randomUUID();
const archivePath = join(UPDATE_DIR, `vendoo-update-${inspection.version}-${id}.zip`);
writeFileSync(archivePath, buffer);
db.prepare(`INSERT INTO update_operations (id, version_from, version_to, source_name, archive_path, sha256, status, details_json, created_at)
VALUES (?, ?, ?, ?, ?, ?, 'staged', ?, datetime('now'))`).run(
id, APP_VERSION, inspection.version, basename(originalName), archivePath, hash(buffer), JSON.stringify({ ...inspection, safety_backup_id: safety.id }),
);
return writePendingOperation({ id, type: 'update', archive_path: archivePath, original_name: basename(originalName), version_from: APP_VERSION, version_to: inspection.version, root_prefix: inspection.root_prefix, safety_backup_id: safety.id, verified: true });
}
function syncLastOperationState() {
if (!existsSync(LAST_OPERATION_FILE)) return;
const last = safeJson(readFileSync(LAST_OPERATION_FILE, 'utf8'), null);
if (!last?.id || last.type !== 'update') return;
try {
db.prepare(`UPDATE update_operations SET status = ?, applied_at = COALESCE(applied_at, ?), error_message = ? WHERE id = ?`).run(
last.status || 'applied', last.applied_at || null, last.error_message || null, last.id,
);
} catch {}
}
export function getOperationsStatus() {
syncLastOperationState();
const pending = existsSync(PENDING_FILE) ? safeJson(readFileSync(PENDING_FILE, 'utf8'), null) : null;
const last = existsSync(LAST_OPERATION_FILE) ? safeJson(readFileSync(LAST_OPERATION_FILE, 'utf8'), null) : null;
const latestUpdate = db.prepare('SELECT * FROM update_operations ORDER BY created_at DESC LIMIT 1').get() || null;
const migrations = db.prepare('SELECT * FROM system_migrations ORDER BY applied_at DESC, id DESC').all();
let safeUpdate = null;
if (latestUpdate) {
const { archive_path: _archivePath, details_json: _detailsJson, ...row } = latestUpdate;
safeUpdate = { ...row, details: safeJson(latestUpdate.details_json) };
}
return { version: APP_VERSION, pending, last, latest_update: safeUpdate, migrations, settings: getOperationSettings() };
}
export function cancelPendingOperation() {
if (!existsSync(PENDING_FILE)) return { cancelled: false };
const pending = safeJson(readFileSync(PENDING_FILE, 'utf8'), {});
rmSync(PENDING_FILE, { force: true });
return { cancelled: true, pending };
}
export async function checkForUpdates() {
const settings = getOperationSettings();
const url = String(settings.update_manifest_url || '').trim();
if (!url) return { configured: false, current_version: APP_VERSION, message: 'Keine Update-Manifest-URL konfiguriert.' };
if (!/^https?:\/\//i.test(url)) throw new Error('Update-Manifest muss über HTTP oder HTTPS erreichbar sein.');
const controller = new AbortController();
const timeout = setTimeout(() => controller.abort(), 8000);
try {
const response = await fetch(url, { signal: controller.signal, headers: { Accept: 'application/json' } });
if (!response.ok) throw new Error(`Update-Manifest antwortet mit HTTP ${response.status}.`);
const manifest = await response.json();
const latest = String(manifest.version || manifest.latest_version || '');
if (!latest) throw new Error('Update-Manifest enthält keine Version.');
return { configured: true, current_version: APP_VERSION, latest_version: latest, update_available: compareVersions(latest, APP_VERSION) > 0, manifest };
} finally { clearTimeout(timeout); }
}
export function recordExtensionHeartbeat({ clientId, browser, version, platform, capabilities, serverUrl, userAgent } = {}) {
const id = String(clientId || '').trim().slice(0, 120) || randomUUID();
db.prepare(`INSERT INTO extension_clients
(client_id, browser, version, platform, server_url, user_agent, capabilities_json, last_seen, first_seen)
VALUES (?, ?, ?, ?, ?, ?, ?, datetime('now'), datetime('now'))
ON CONFLICT(client_id) DO UPDATE SET browser=excluded.browser, version=excluded.version, platform=excluded.platform,
server_url=excluded.server_url, user_agent=excluded.user_agent, capabilities_json=excluded.capabilities_json, last_seen=datetime('now')`).run(
id, String(browser || 'unknown').slice(0, 40), String(version || '').slice(0, 30), String(platform || '').slice(0, 40),
String(serverUrl || '').slice(0, 500), String(userAgent || '').slice(0, 1000), JSON.stringify(capabilities || {}),
);
return { ok: true, client_id: id, server_version: APP_VERSION, time: nowIso() };
}
function manifestVersion(path) {
try { return JSON.parse(readFileSync(path, 'utf8')).version || null; } catch { return null; }
}
export function getExtensionDiagnostics() {
const packages = {
chrome: { path: join(APP_ROOT, 'extensions', 'chrome', 'manifest.json') },
edge: { path: join(APP_ROOT, 'extensions', 'edge', 'manifest.json') },
firefox: { path: join(APP_ROOT, 'extensions', 'firefox', 'manifest.json') },
safari: { path: join(APP_ROOT, 'extensions', 'safari', 'web-extension', 'manifest.json') },
};
for (const item of Object.values(packages)) Object.assign(item, { exists: existsSync(item.path), version: manifestVersion(item.path) });
const clients = db.prepare(`SELECT *, CAST((julianday('now') - julianday(last_seen)) * 86400 AS INTEGER) AS age_seconds
FROM extension_clients ORDER BY last_seen DESC LIMIT 30`).all().map(row => ({ ...row, capabilities: safeJson(row.capabilities_json) }));
return { expected_version: '1.9.0', packages, clients, connected: clients.filter(client => Number(client.age_seconds) <= 180).length };
}
let automaticTimer = null;
export function startAutomaticBackupLoop() {
if (automaticTimer) return;
const tick = async () => {
const settings = getOperationSettings();
if (!settings.auto_backup_enabled) return;
const now = new Date();
const date = localDateKey(now);
if (now.getHours() !== settings.auto_backup_hour || settings.last_auto_backup_date === date) return;
try {
await createBackup({ automatic: true, includeUploads: settings.include_uploads_automatic, includeConfig: false, label: 'automatisch' });
db.prepare(`INSERT INTO operation_settings (key, value, updated_at) VALUES ('last_auto_backup_date', ?, datetime('now'))
ON CONFLICT(key) DO UPDATE SET value = excluded.value, updated_at = datetime('now')`).run(date);
} catch (error) { console.error('Automatisches Backup fehlgeschlagen:', error.message); }
};
automaticTimer = setInterval(tick, 15 * 60 * 1000);
automaticTimer.unref?.();
const initialTimer = setTimeout(tick, 5000);
initialTimer.unref?.();
}
export const operationsPaths = { APP_ROOT, DATA_ROOT, DB_PATH, UPLOADS_DIR, BACKUP_DIR, OPERATIONS_DIR, PENDING_FILE, LAST_OPERATION_FILE };