/*
 * Priority Forge - Cross-project task prioritization
 * Copyright (C) 2026 Priority Forge Contributors
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, version 3.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program. If not, see <https://www.gnu.org/licenses/>.
 */

import * as fs from 'fs';
import * as fsPromises from 'fs/promises';
import * as path from 'path';
import { v4 as uuidv4 } from 'uuid';
import chokidar, { FSWatcher } from 'chokidar';
import { Mutex } from 'async-mutex';
import {
  Project,
  Task,
  WeightedTask,
  DataGap,
  Decision,
  TaskCompletionRecord,
  PriorityChangeEvent,
  TaskSelectionEvent,
  QueueRebalanceEvent,
  DragReorderEvent,
  OnlineLearnerState,
  ProgressDatabase,
  GlobalMLDatabase,
  DEFAULT_GLOBAL_ML_DATABASE,
  CreateProjectDTO,
  UpdateProjectDTO,
  CreateTaskDTO,
  UpdateTaskDTO,
  CreateDataGapDTO,
  UpdateDataGapDTO,
  CreateDecisionDTO,
  Priority,
  HeuristicWeights,
  DEFAULT_HEURISTIC_WEIGHTS,
  DEFAULT_ONLINE_LEARNER_STATE,
  UpdateHeuristicWeightsDTO,
  LogDragReorderDTO,
  UpdateOnlineLearnerDTO,
  TaskWeights,
  Effort,
  Workspace,
  CreateWorkspaceDTO,
  WorkspaceMetadata,
  WorkspaceFreshness,
  WorkspaceFreshnessThresholds,
  DEFAULT_WORKSPACE_FRESHNESS_THRESHOLDS,
} from '../types/schema';
import { StorageInterface } from './interface';
import { MinHeap, toWeightedTask, recalculateAllScores, getDefaultWeights } from '../heap';
import {
  TaskAlreadyExistsError,
  VersionConflictError,
  TaskNotClaimableError,
  TaskAlreadyClaimedError,
} from './errors';

// Data dir resolution. Production default is the repo-local `data/`. Tests set
// PRIORITY_FORGE_DATA_DIR to a throwaway dir so they never read/write the user's
// live data; under vitest we additionally namespace by worker so parallel test
// files don't race the same JSON files (the corruption mode we hit otherwise).
function resolveDataDir(): string {
  const override = process.env.PRIORITY_FORGE_DATA_DIR;
  if (!override) return path.join(__dirname, '../../data');
  const worker = process.env.VITEST_POOL_ID || process.env.VITEST_WORKER_ID;
  return worker ? path.join(override, `w${worker}`) : override;
}

const DATA_DIR = resolveDataDir();
const WORKSPACES_DIR = path.join(DATA_DIR, 'workspaces');
const WORKSPACES_META_FILE = path.join(DATA_DIR, 'workspaces.json');
const GLOBAL_ML_FILE = path.join(DATA_DIR, 'ml-training.json');

function getEmptyDatabase(): ProgressDatabase {
  return {
    version: 'v4',
    lastUpdated: new Date().toISOString(),
    projects: [],
    tasks: [],
    dataGaps: [],
    decisions: [],
    heuristicWeights: { ...DEFAULT_HEURISTIC_WEIGHTS },
  };
}

// In-memory context switch counter (resets on restart, persisted on task completion)
const contextSwitchCounts: Map<string, number> = new Map();

export class JsonStorage implements StorageInterface {
  private db: ProgressDatabase;
  private globalML: GlobalMLDatabase;  // V4: Global ML training data (shared across workspaces)
  private currentWorkspaceId: string | null = null;
  
  /**
   * V3.1: Map-based task storage for O(1) lookups and guaranteed uniqueness
   * The Map is the source of truth; db.tasks is only used for JSON serialization
   */
  private taskMap: Map<string, WeightedTask> = new Map();
  
  private taskHeap: MinHeap<WeightedTask>;
  private onWrite: (() => Promise<void>) | null = null;
  
  /**
   * V4.1: File watcher for external modification detection
   * Uses chokidar for reliable cross-platform watching (handles atomic file ops like mv)
   */
  private fileWatcher: FSWatcher | null = null;
  private lastWriteTime: number = 0;  // Track our own writes to avoid reload loops
  private static readonly WRITE_GRACE_PERIOD_MS = 1500;  // Ignore file changes within 1.5s of our write

  /**
   * Concurrency — per-file write mutexes (CONC-001).
   *
   * `save()` and `saveGlobalML()` acquire these internally so concurrent HTTP
   * requests can't produce torn writes. async-mutex is NOT reentrant, so never
   * call save()/saveGlobalML() from inside a `runExclusiveWorkspace` /
   * `runExclusiveGlobalML` block — use the `_saveLocked` / `_saveGlobalMLLocked`
   * variants there instead.
   *
   * For broader critical sections (check-then-write patterns that must be
   * atomic relative to other writers), use runExclusiveWorkspace/GlobalML
   * and call the `*Locked` variants inside. CONC-002+ depend on this pattern.
   */
  private readonly workspaceMutex = new Mutex();
  private readonly globalMLMutex = new Mutex();

  /**
   * Test instrumentation — exposed publicly so concurrency tests can verify
   * serialization. Maintained in the `*Locked` variants. Cheap (4 integer ops
   * per save), left in production for diagnostics.
   */
  public readonly _writeInFlight = { workspace: 0, globalML: 0 };
  public readonly _maxWriteInFlight = { workspace: 0, globalML: 0 };

  constructor() {
    // Ensure workspace directory exists
    if (!fs.existsSync(WORKSPACES_DIR)) {
      fs.mkdirSync(WORKSPACES_DIR, { recursive: true });
    }
    // Load global ML training data
    this.globalML = this.loadGlobalML();
    // Load current workspace
    this.currentWorkspaceId = this.loadCurrentWorkspaceId();
    this.db = this.load();
    // Initialize Map from loaded tasks (with deduplication)
    this.initializeTaskMap();
    this.taskHeap = new MinHeap(this.getTaskArray());
    // Start watching for external file changes
    this.startFileWatcher();
  }
  
  /**
   * V4.1: Start watching the current workspace's progress.json for external changes
   * Uses chokidar which properly handles:
   * - Atomic file replacements (mv, rsync, etc.)
   * - FSEvents quirks on macOS
   * - Recursive directory watching
   */
  private startFileWatcher(): void {
    this.stopFileWatcher();  // Clean up any existing watcher
    
    const dbFile = this.getWorkspaceDbFile();
    if (!fs.existsSync(dbFile)) return;
    
    try {
      this.fileWatcher = chokidar.watch(dbFile, {
        persistent: true,
        awaitWriteFinish: {
          stabilityThreshold: 300,  // Wait 300ms for write to finish
          pollInterval: 100
        },
        ignoreInitial: true,  // Don't fire on initial add
        usePolling: false,    // Use native OS watchers (inotify on Linux, FSEvents on macOS)
        atomic: true          // Handle atomic writes (mv, etc.)
      });
      
      this.fileWatcher.on('change', (changedPath) => {
        // Check if this was our own write (within grace period)
        const now = Date.now();
        const timeSinceWrite = now - this.lastWriteTime;
        if (timeSinceWrite < JsonStorage.WRITE_GRACE_PERIOD_MS) {
          console.log(`👁️  Ignoring change (own write ${timeSinceWrite}ms ago)`);
          return;  // Ignore - this was our own write
        }
        
        console.log(`📂 External file change detected (${timeSinceWrite}ms since last write), reloading...`);
        this.reloadFromDisk();
      });
      
      this.fileWatcher.on('error', (err) => {
        console.warn('⚠️  File watcher error:', err);
      });
      
      console.log('👁️  File watcher active for external changes (chokidar)');
    } catch (err) {
      console.warn('⚠️  Could not start file watcher:', err);
    }
  }
  
  /**
   * V4.1: Stop the file watcher (called on workspace switch or shutdown)
   */
  private stopFileWatcher(): void {
    if (this.fileWatcher) {
      this.fileWatcher.close();
      this.fileWatcher = null;
    }
  }
  
  /**
   * V4.1: Reload database from disk (called when external changes detected)
   */
  private reloadFromDisk(): void {
    try {
      this.db = this.load();
      this.initializeTaskMap();
      this.taskHeap = new MinHeap(this.getTaskArray());
      console.log(`✅ Database reloaded: ${this.taskMap.size} tasks`);
    } catch (err) {
      console.error('❌ Failed to reload database:', err);
    }
  }

  /**
   * V4: Load current workspace ID from metadata
   */
  private loadCurrentWorkspaceId(): string | null {
    if (!fs.existsSync(WORKSPACES_META_FILE)) {
      return null;
    }
    try {
      const metadata = JSON.parse(fs.readFileSync(WORKSPACES_META_FILE, 'utf-8')) as WorkspaceMetadata;
      return metadata.currentWorkspaceId || null;
    } catch {
      return null;
    }
  }
  
  /**
   * V4: Get workspace database file path
   * V4.1: Made workspaceId optional - defaults to current workspace
   */
  private getWorkspaceDbFile(workspaceId?: string): string {
    const effectiveId = workspaceId || this.currentWorkspaceId || 'default';
    const workspaceDir = path.join(WORKSPACES_DIR, effectiveId);
    if (!fs.existsSync(workspaceDir)) {
      fs.mkdirSync(workspaceDir, { recursive: true });
    }
    return path.join(workspaceDir, 'progress.json');
  }
  
  /**
   * V4: Load workspace metadata
   */
  private loadWorkspaceMetadata(): WorkspaceMetadata {
    if (!fs.existsSync(WORKSPACES_META_FILE)) {
      return {
        workspaces: [],
        currentWorkspaceId: null,
      };
    }
    try {
      return JSON.parse(fs.readFileSync(WORKSPACES_META_FILE, 'utf-8')) as WorkspaceMetadata;
    } catch {
      return {
        workspaces: [],
        currentWorkspaceId: null,
      };
    }
  }
  
  /**
   * V4: Save workspace metadata
   */
  private saveWorkspaceMetadata(metadata: WorkspaceMetadata): void {
    fs.writeFileSync(WORKSPACES_META_FILE, JSON.stringify(metadata, null, 2));
  }

  /**
   * V4: Load global ML training data
   * This data is shared across ALL workspaces to ensure training continuity
   */
  private loadGlobalML(): GlobalMLDatabase {
    if (!fs.existsSync(GLOBAL_ML_FILE)) {
      const empty: GlobalMLDatabase = { ...DEFAULT_GLOBAL_ML_DATABASE };
      fs.writeFileSync(GLOBAL_ML_FILE, JSON.stringify(empty, null, 2));
      return empty;
    }
    try {
      const loaded = JSON.parse(fs.readFileSync(GLOBAL_ML_FILE, 'utf-8')) as GlobalMLDatabase;
      // V4.2: Default decisions array for existing databases without it
      loaded.decisions = loaded.decisions || [];
      return loaded;
    } catch {
      console.error('Failed to load global ML data, using defaults');
      return { ...DEFAULT_GLOBAL_ML_DATABASE };
    }
  }

  /**
   * V4: Save global ML training data.
   *
   * Serialized by `globalMLMutex` (CONC-001). Most callers should use this.
   * Callers that need a broader critical section (e.g. check-then-write)
   * acquire the mutex via `runExclusiveGlobalML()` and call `_saveGlobalMLLocked`
   * directly — never nest calls to this method (async-mutex is not reentrant).
   */
  private async saveGlobalML(): Promise<void> {
    return this.globalMLMutex.runExclusive(() => this._saveGlobalMLLocked());
  }

  /**
   * Lock-required variant of saveGlobalML. Assumes the caller holds
   * `globalMLMutex`. Use via `runExclusiveGlobalML(() => this._saveGlobalMLLocked())`.
   */
  private async _saveGlobalMLLocked(): Promise<void> {
    this._writeInFlight.globalML++;
    if (this._writeInFlight.globalML > this._maxWriteInFlight.globalML) {
      this._maxWriteInFlight.globalML = this._writeInFlight.globalML;
    }
    try {
      this.globalML.lastUpdated = new Date().toISOString();
      const tmpFile = GLOBAL_ML_FILE + '.tmp';
      await fsPromises.writeFile(tmpFile, JSON.stringify(this.globalML, null, 2));
      await fsPromises.rename(tmpFile, GLOBAL_ML_FILE);
    } finally {
      this._writeInFlight.globalML--;
    }
  }

  /**
   * Acquire the globalML write mutex and run `fn` inside it. Use when a
   * critical section needs to span read-check-write or multiple mutations.
   * Inside `fn`, call `_saveGlobalMLLocked()` (NOT `saveGlobalML()`).
   */
  public async runExclusiveGlobalML<T>(fn: () => Promise<T>): Promise<T> {
    return this.globalMLMutex.runExclusive(fn);
  }

  /**
   * Initialize the task Map from db.tasks array
   * Deduplicates by keeping the most "complete" version of each task
   */
  private initializeTaskMap(): void {
    this.taskMap.clear();
    let duplicatesFound = 0;
    
    for (const task of this.db.tasks) {
      const existing = this.taskMap.get(task.id);
      if (!existing) {
        this.taskMap.set(task.id, task);
      } else {
        duplicatesFound++;
        // Keep the completed one, or the newer one if same status
        if (task.status === 'complete' && existing.status !== 'complete') {
          this.taskMap.set(task.id, task);
        } else if (existing.status !== 'complete' && new Date(task.updatedAt) > new Date(existing.updatedAt)) {
          this.taskMap.set(task.id, task);
        }
        // Otherwise keep existing (first one wins)
      }
    }
    
    if (duplicatesFound > 0) {
      console.log(`🔧 V3.1: Deduplicated ${duplicatesFound} tasks during load`);
      // Sync db.tasks with deduplicated Map
      this.syncTasksArray();
    }
  }

  /**
   * Sync db.tasks array from taskMap (for JSON serialization)
   */
  private syncTasksArray(): void {
    this.db.tasks = Array.from(this.taskMap.values());
  }

  /**
   * Get tasks as array (from Map)
   */
  private getTaskArray(): WeightedTask[] {
    return Array.from(this.taskMap.values());
  }

  setOnWriteCallback(callback: () => Promise<void>) {
    this.onWrite = callback;
  }

  /**
   * V3: Log a queue rebalance event for ML training
   * Captures before/after state when dependency graph changes
   */
  private logRebalanceEvent(
    trigger: QueueRebalanceEvent['trigger'],
    tasksBefore: WeightedTask[],
    tasksAfter: WeightedTask[],
    triggerTaskId?: string
  ): void {
    // Build rank maps
    const rankBefore = new Map<string, { rank: number; score: number }>();
    const rankAfter = new Map<string, { rank: number; score: number }>();
    
    tasksBefore
      .filter(t => t.status !== 'complete')
      .sort((a, b) => a.priorityScore - b.priorityScore)
      .forEach((t, idx) => rankBefore.set(t.id, { rank: idx, score: t.priorityScore }));
    
    tasksAfter
      .filter(t => t.status !== 'complete')
      .sort((a, b) => a.priorityScore - b.priorityScore)
      .forEach((t, idx) => rankAfter.set(t.id, { rank: idx, score: t.priorityScore }));

    // Find significant changes (rank changed by more than 2)
    const significantChanges: QueueRebalanceEvent['significantChanges'] = [];
    for (const [taskId, before] of rankBefore) {
      const after = rankAfter.get(taskId);
      if (after && Math.abs(before.rank - after.rank) > 2) {
        significantChanges.push({
          taskId,
          rankBefore: before.rank,
          rankAfter: after.rank,
          scoreBefore: before.score,
          scoreAfter: after.score,
        });
      }
    }

    // Only log if there were significant changes
    if (significantChanges.length === 0 && trigger !== 'weights_changed') {
      return;
    }

    const topBefore = [...rankBefore.entries()]
      .sort((a, b) => a[1].rank - b[1].rank)
      .slice(0, 3)
      .map(([id]) => id);
    
    const topAfter = [...rankAfter.entries()]
      .sort((a, b) => a[1].rank - b[1].rank)
      .slice(0, 3)
      .map(([id]) => id);

    const event: QueueRebalanceEvent = {
      id: uuidv4(),
      trigger,
      triggerTaskId,
      timestamp: new Date().toISOString(),
      queueSizeBefore: rankBefore.size,
      queueSizeAfter: rankAfter.size,
      significantChanges,
      topTasksBefore: topBefore,
      topTasksAfter: topAfter,
      // V4: Tag with workspace for ML filtering
      workspaceId: this.currentWorkspaceId || undefined,
      // V4.4: cross-forge continuity — link via the triggering task if it's a projection
      sharedTaskId: triggerTaskId ? this.taskMap.get(triggerTaskId)?.sharedTaskId : undefined,
    };

    // V4: Write to global ML file
    this.globalML.queueRebalanceEvents.push(event);
    
    console.log(`📊 V3: Logged rebalance event (${trigger}): ${significantChanges.length} significant changes`);
  }

  private load(): ProgressDatabase {
    if (!fs.existsSync(DATA_DIR)) {
      fs.mkdirSync(DATA_DIR, { recursive: true });
    }

    const dbFile = this.getWorkspaceDbFile(this.currentWorkspaceId!);

    if (!fs.existsSync(dbFile)) {
      const empty = getEmptyDatabase();
      fs.writeFileSync(dbFile, JSON.stringify(empty, null, 2));
      return empty;
    }
    return JSON.parse(fs.readFileSync(dbFile, 'utf-8')) as ProgressDatabase;
  }

  /**
   * Workspace progress.json save.
   *
   * Serialized by `workspaceMutex` (CONC-001). Callers that need a broader
   * critical section (check-then-write) acquire the mutex via
   * `runExclusiveWorkspace()` and call `_saveLocked` inside — never nest
   * calls to this method (async-mutex is not reentrant).
   */
  private async save(): Promise<void> {
    return this.workspaceMutex.runExclusive(() => this._saveLocked());
  }

  /**
   * Lock-required variant of save. Assumes the caller holds `workspaceMutex`.
   * Use via `runExclusiveWorkspace(() => this._saveLocked())`.
   */
  private async _saveLocked(): Promise<void> {
    this._writeInFlight.workspace++;
    if (this._writeInFlight.workspace > this._maxWriteInFlight.workspace) {
      this._maxWriteInFlight.workspace = this._writeInFlight.workspace;
    }
    try {
      // Sync array from Map before saving
      this.syncTasksArray();
      this.db.lastUpdated = new Date().toISOString();

      const dbFile = this.getWorkspaceDbFile(this.currentWorkspaceId!);

      // V4.1: Track our write time to avoid reload loops from file watcher
      this.lastWriteTime = Date.now();
      const tmpFile = dbFile + '.tmp';
      await fsPromises.writeFile(tmpFile, JSON.stringify(this.db, null, 2));
      await fsPromises.rename(tmpFile, dbFile);
      if (this.onWrite) {
        await this.onWrite();
      }
    } finally {
      this._writeInFlight.workspace--;
    }
  }

  /**
   * Acquire the workspace write mutex and run `fn` inside it. Use when a
   * critical section needs to span read-check-write or multiple mutations.
   * Inside `fn`, call `_saveLocked()` (NOT `save()`).
   */
  public async runExclusiveWorkspace<T>(fn: () => Promise<T>): Promise<T> {
    return this.workspaceMutex.runExclusive(fn);
  }

  /**
   * CONC-006: atomic dual-file write. Acquires both mutexes in the fixed
   * order (workspace → globalML) and writes both files before releasing
   * either. Replaces the
   *   await this.saveGlobalML();
   *   await this.save();
   * pattern which left a window where another request could interleave its
   * own saves between the two calls, leaving the two files in a diverged
   * state (e.g., globalML has the new priority-change event but progress.json
   * hasn't committed the task's priority change yet).
   *
   * DEADLOCK SAFETY: the lock order is workspace → globalML, always. No
   * other code path in this file acquires globalML → workspace. Never call
   * `save()` from inside a globalML-holding critical section.
   *
   * If the caller already holds workspaceMutex (i.e., is inside a
   * runExclusiveWorkspace block), use `_saveBothFromWorkspaceContext()`
   * instead — this method would deadlock on the non-reentrant mutex.
   */
  public async saveBoth(): Promise<void> {
    return this.workspaceMutex.runExclusive(async () => {
      return this.globalMLMutex.runExclusive(async () => {
        await this._saveGlobalMLLocked();
        await this._saveLocked();
      });
    });
  }

  /**
   * CONC-006: variant of `saveBoth` for callers that already hold
   * workspaceMutex (e.g., inside an existing runExclusiveWorkspace call).
   * Acquires globalMLMutex on top, then writes both files. Respects the
   * fixed workspace → globalML lock order.
   */
  private async _saveBothFromWorkspaceContext(): Promise<void> {
    return this.globalMLMutex.runExclusive(async () => {
      await this._saveGlobalMLLocked();
      await this._saveLocked();
    });
  }

  /**
   * Rebuild heap from current task Map (use after bulk operations)
   */
  private rebuildHeap(): void {
    this.taskHeap = new MinHeap(this.getTaskArray());
  }

  async getAll(): Promise<ProgressDatabase> {
    // Sync before returning
    this.syncTasksArray();
    return this.db;
  }

  // Projects
  async getProjects(): Promise<Project[]> {
    return this.db.projects;
  }

  async getProject(id: string): Promise<Project | null> {
    return this.db.projects.find(p => p.id === id) || null;
  }

  async createProject(data: CreateProjectDTO): Promise<Project> {
    const now = new Date().toISOString();
    const project: Project = {
      id: uuidv4(),
      name: data.name,
      path: data.path,
      status: data.status || 'active',
      primaryFocus: data.primaryFocus,
      createdAt: now,
      updatedAt: now,
    };
    this.db.projects.push(project);
    await this.save();
    return project;
  }

  async updateProject(id: string, data: UpdateProjectDTO): Promise<Project | null> {
    const idx = this.db.projects.findIndex(p => p.id === id);
    if (idx === -1) return null;
    
    this.db.projects[idx] = {
      ...this.db.projects[idx],
      ...data,
      updatedAt: new Date().toISOString(),
    };
    await this.save();
    return this.db.projects[idx];
  }

  async deleteProject(id: string): Promise<boolean> {
    const idx = this.db.projects.findIndex(p => p.id === id);
    if (idx === -1) return false;
    this.db.projects.splice(idx, 1);
    await this.save();
    return true;
  }

  // Tasks - V3.1 with Map-based storage
  //
  // CONC-005 read-safety analysis:
  //   MinHeap.toSortedArray() does `[...this.heap].sort(...)` — the spread is
  //   a synchronous atomic copy at the JS microtask level, so even if a
  //   concurrent writer replaces `this.taskHeap` mid-call, we're working on a
  //   snapshot of the prior heap. The returned array contains references to
  //   WeightedTask objects, which are only mutated through new-object
  //   replacement (createTask / updateTask spread `{...existing, ...data}`
  //   and call taskMap.set with the new object). So references captured in
  //   `all` reflect a consistent point-in-time state. Filter chain below is
  //   race-free on that snapshot.
  async getTasks(includeCompleted: boolean = false): Promise<WeightedTask[]> {
    // Return sorted by priority score (lowest first = highest priority)
    const all = this.taskHeap.toSortedArray();
    if (includeCompleted) {
      return all;
    }
    // Filter out completed/cancelled tasks by default
    const finishedStatuses = ['complete', 'completed', 'cancelled'];
    return all.filter(t => !finishedStatuses.includes(t.status));
  }

  /**
   * Get all tasks including completed (for history/archival purposes)
   */
  async getAllTasks(): Promise<WeightedTask[]> {
    return this.taskHeap.toSortedArray();
  }

  /**
   * Get completed/cancelled tasks only
   */
  async getCompletedTasks(): Promise<WeightedTask[]> {
    const finishedStatuses = ['complete', 'completed', 'cancelled'];
    return this.taskHeap.toSortedArray().filter(t => finishedStatuses.includes(t.status));
  }

  async getTask(id: string): Promise<WeightedTask | null> {
    // O(1) lookup via Map
    return this.taskMap.get(id) || null;
  }

  async getTasksByPriority(priority: Priority): Promise<WeightedTask[]> {
    return this.getTaskArray()
      .filter(t => t.priority === priority)
      .sort((a, b) => a.priorityScore - b.priorityScore);
  }

  async getTasksByProject(projectId: string): Promise<WeightedTask[]> {
    return this.getTaskArray()
      .filter(t => t.project === projectId)
      .sort((a, b) => a.priorityScore - b.priorityScore);
  }

  /**
   * V2: Get the single highest priority task (excludes completed/cancelled tasks)
   *
   * CONC-005: returns a shallow defensive copy (spread) so callers can't
   * mutate the object's fields and corrupt internal heap/taskMap state. The
   * heap itself stores references; writers always replace objects (never
   * mutate in place), so read-side copies aren't needed for correctness,
   * just for defense against misbehaving callers.
   */
  async getTopPriority(): Promise<WeightedTask | null> {
    const finishedStatuses = ['complete', 'completed', 'cancelled'];
    // O(1) fast path: heap top is almost always actionable since completed tasks are removed
    const top = this.taskHeap.peek();
    if (top && !finishedStatuses.includes(top.status)) {
      return { ...top };
    }
    // Fallback: stale completed task at heap top, sort to find first actionable
    const sorted = this.taskHeap.toSortedArray();
    const found = sorted.find(t => !finishedStatuses.includes(t.status));
    return found ? { ...found } : null;  // CONC-005: defensive shallow copy
  }

  /**
   * V2: Pop the highest priority task (removes from queue)
   */
  async popTopPriority(): Promise<WeightedTask | null> {
    const task = this.taskHeap.pop();
    if (task) {
      this.taskMap.delete(task.id);
      await this.save();
    }
    return task;
  }

  async createTask(data: CreateTaskDTO): Promise<WeightedTask> {
    // CONC-002: wrap entire check-then-write in the workspace mutex so two
    // concurrent creates with the same custom id can't both pass the uniqueness
    // check and then silently overwrite each other in taskMap. One wins with
    // 201; the other throws TaskAlreadyExistsError (→ 409 at the HTTP layer).
    return this.workspaceMutex.runExclusive(async () => {
      const now = new Date().toISOString();
      const taskId = data.id || uuidv4();

      // V3.1 + CONC-002: check for duplicate ID inside the mutex so the
      // check-and-insert is atomic relative to other workspaceMutex writers.
      if (this.taskMap.has(taskId)) {
        throw new TaskAlreadyExistsError(taskId);
      }

      // Snapshot before state for rebalance logging
      const tasksBefore = this.getTaskArray();

      // Create base task
      const baseTask: Task = {
        id: taskId,
        priority: data.priority,
        task: data.task,
        project: data.project,
        status: data.status || 'not_started',
        blocking: data.blocking,
        dependencies: data.dependencies,
        notes: data.notes,
        createdAt: now,
        updatedAt: now,
      };

      // Convert to weighted task with computed scores
      const weightedTask = toWeightedTask(
        {
          ...baseTask,
          deadline: data.deadline,
          effort: data.effort,
          weights: data.weights ? { ...getDefaultWeights(), ...data.weights } : undefined,
          version: 1, // CONC-003: new tasks start at version 1
          // Plan 2: preserve sharedTaskId link when present (projection of
          // a hub-canonical SharedTask).
          sharedTaskId: data.sharedTaskId,
        } as WeightedTask,
        this.getTaskArray(),
        this.globalML.heuristicWeights
      );

      // Add to Map (guaranteed unique due to check above, now atomic via mutex)
      this.taskMap.set(weightedTask.id, weightedTask);

      // IMPORTANT: Recalculate ALL task weights since dependencies/blocking may have changed
      // e.g., if new task depends on existing task, that task's blockingCount increases
      if (data.dependencies?.length || data.blocking) {
        const recalculated = recalculateAllScores(this.getTaskArray(), this.globalML.heuristicWeights);
        // Update Map with recalculated tasks
        for (const task of recalculated) {
          this.taskMap.set(task.id, task);
        }
        this.rebuildHeap();

        // Log rebalance event
        this.logRebalanceEvent('task_created', tasksBefore, this.getTaskArray(), weightedTask.id);
      } else {
        this.taskHeap.push(weightedTask);
      }

      // CONC-002: use _saveLocked since we already hold workspaceMutex (mutex is not reentrant)
      await this._saveLocked();

      // Return the updated version from Map (may have been recalculated)
      return this.taskMap.get(weightedTask.id) || weightedTask;
    });
  }

  async updateTask(id: string, data: UpdateTaskDTO, expectedVersion?: number): Promise<WeightedTask | null> {
    // CONC-003: wrap entire read-modify-write in the workspace mutex. Critical
    // for two reasons:
    //   1. The optimistic-concurrency check (expectedVersion) must be atomic
    //      with the subsequent write — otherwise two writers could both pass
    //      the check and both write, losing one update.
    //   2. Multi-field merges (e.g. agent A sets notes, agent B sets status)
    //      can still race even without expectedVersion. Serializing inside
    //      the mutex ensures one merge completes fully before the next reads.
    // Uses `_saveLocked` for the final write since we already hold the mutex
    // (async-mutex is not reentrant). `saveGlobalML` on the priority-change
    // path stays as the public version because it acquires a *different*
    // mutex (globalMLMutex) — no deadlock.
    return this.workspaceMutex.runExclusive(async () => {
      const existingTask = this.taskMap.get(id);
      if (!existingTask) return null;

      // CONC-003: optimistic concurrency — if the caller specified the version
      // they expected to be updating, reject on mismatch. Existing persisted
      // tasks (pre-CONC-003) have undefined version; treat as 1 so If-Match: "1"
      // still works.
      const currentVersion = existingTask.version ?? 1;
      if (expectedVersion !== undefined && expectedVersion !== currentVersion) {
        throw new VersionConflictError(id, expectedVersion, currentVersion);
      }

      // Snapshot before state for rebalance logging
      const tasksBefore = this.getTaskArray();

      const oldPriority = existingTask.priority;
      const oldScore = existingTask.priorityScore;
      const oldDependencies = existingTask.dependencies || [];
      const oldBlocking = existingTask.blocking;
      const oldStatus = existingTask.status;

      // Get queue position before update
      const sortedBefore = this.taskHeap.toSortedArray();
      const queuePositionBefore = sortedBefore.findIndex(t => t.id === id);

      // V4.0: Capture startedAt when transitioning to in_progress
      let startedAt = existingTask.startedAt;
      if (data.status === 'in_progress' && oldStatus !== 'in_progress' && !startedAt) {
        startedAt = new Date().toISOString();
        console.log(`📊 V4.0: Task ${id} started work at ${startedAt}`);
        // V4.2: Auto-log task selection when work begins — ensures every started
        // task is captured as a selection event without relying on the agent to
        // call log_task_selection separately.
        const selectionEvent = await this.logTaskSelection(id);
        // V4.2: Auto-log skip decision when user picks a non-top task
        if (selectionEvent && !selectionEvent.wasTopSelected) {
          const topTaskName = this.taskMap.get(selectionEvent.topTaskId)?.task || selectionEvent.topTaskId;
          const skipDecision: Decision = {
            id: uuidv4(),
            date: new Date().toISOString().split('T')[0],
            decision: `Selected "${existingTask.task}" (rank ${selectionEvent.selectedTaskRank + 1}) over recommended "${topTaskName}" (rank 1)`,
            rationale: `User chose task ${selectionEvent.selectedTaskRank} positions below top-ranked. Score diff: ${selectionEvent.selectedTaskScore - selectionEvent.topTaskScore}.`,
            createdAt: new Date().toISOString(),
            decisionType: 'skip',
            relatedTaskId: id,
            skippedTaskId: selectionEvent.topTaskId,
            selectedTaskId: id,
            rankDelta: selectionEvent.selectedTaskRank,
            selectionEventId: selectionEvent.id,
            workspaceId: this.currentWorkspaceId || undefined,
            // V4.4: cross-forge continuity across release→reclaim id churn
            sharedTaskId: existingTask.sharedTaskId,
          };
          this.db.decisions.push(skipDecision);
          this.globalML.decisions.push(skipDecision);
          console.log(`📊 V4.2: Auto-logged skip decision (rank ${selectionEvent.selectedTaskRank + 1}, skipped "${topTaskName}")`);
        }
      }

      // Merge updates + CONC-003 version bump
      const updatedBase: WeightedTask = {
        ...existingTask,
        ...data,
        startedAt,  // V4.0: Preserve or set startedAt
        weights: data.weights
          ? { ...existingTask.weights, ...data.weights }
          : existingTask.weights,
        updatedAt: new Date().toISOString(),
        version: currentVersion + 1, // CONC-003: bump on every mutation
      };

      // Check if dependency graph changed (requires full recalculation)
      const dependenciesChanged =
        JSON.stringify(data.dependencies) !== JSON.stringify(oldDependencies) ||
        data.blocking !== oldBlocking;

      // Update in Map
      this.taskMap.set(id, updatedBase);

      // Recalculate scores
      if (dependenciesChanged) {
        // Full recalculation needed - dependency graph changed
        const recalculated = recalculateAllScores(this.getTaskArray(), this.globalML.heuristicWeights);
        for (const task of recalculated) {
          this.taskMap.set(task.id, task);
        }
        this.rebuildHeap();

        // Log rebalance event
        this.logRebalanceEvent('task_updated', tasksBefore, this.getTaskArray(), id);
      } else {
        // Just recalculate this task
        const updatedTask = toWeightedTask(
          updatedBase,
          this.getTaskArray(),
          this.globalML.heuristicWeights
        );
        // Preserve the bumped version through recalculation
        updatedTask.version = updatedBase.version;
        this.taskMap.set(id, updatedTask);
        this.taskHeap.update(id, updatedTask);
      }

      const finalTask = this.taskMap.get(id)!;

      // V3: Log priority change event if priority changed
      const priorityChanged = data.priority !== undefined && data.priority !== oldPriority;
      if (priorityChanged) {
        const sortedAfter = this.taskHeap.toSortedArray();
        const queuePositionAfter = sortedAfter.findIndex(t => t.id === id);

        const changeEvent: PriorityChangeEvent = {
          id: uuidv4(),
          taskId: id,
          oldPriority,
          newPriority: data.priority!,
          oldScore,
          newScore: finalTask.priorityScore,
          timestamp: new Date().toISOString(),
          queuePositionBefore,
          queuePositionAfter,
          // V4: Tag with workspace for ML filtering
          workspaceId: this.currentWorkspaceId || undefined,
          // V4.4: cross-forge continuity across release→reclaim id churn
          sharedTaskId: finalTask.sharedTaskId,
        };

        this.globalML.priorityChangeEvents.push(changeEvent);
        console.log(`📊 V3: Logged priority change for ${id}: ${oldPriority} → ${data.priority}`);
      }

      // CONC-006: atomic dual-write when priority changed — ensures globalML
      // priority-change event and the task's new priority land on disk
      // together. When no ML event fired, workspace-only save is enough.
      if (priorityChanged) {
        await this._saveBothFromWorkspaceContext();
      } else {
        await this._saveLocked();
      }
      return finalTask;
    });
  }

  /**
   * CONC-004: atomically claim a task for an agent.
   *
   * Transitions not_started / waiting / blocked → in_progress, setting
   * `claimedBy = agentId`, `startedAt = now`, and bumping the version. The
   * entire check-and-transition is inside workspaceMutex so two agents
   * can't both claim the same task.
   *
   * Semantics:
   *   - Task missing → returns null (caller maps to 404)
   *   - Task is complete / completed / cancelled → TaskNotClaimableError (409)
   *   - Task is already in_progress with SAME agentId → idempotent no-op,
   *     returns the current task (safe to retry on network failures)
   *   - Task is already in_progress with DIFFERENT agentId → TaskAlreadyClaimedError (409)
   *   - Task is not_started / waiting / blocked → claim succeeds
   *
   * Also logs a TaskSelectionEvent (matches the V4.2 protocol where every
   * transition to in_progress is captured for ML training).
   */
  // Plan 2: lookup by hub-canonical sharedTaskId. Linear scan is fine
  // here — the cardinality of sharedTaskId-linked tasks is small (one
  // active projection per claim), and the alternative (a secondary index)
  // would couple the storage to a feature that's still settling.
  async getTaskBySharedTaskId(sharedTaskId: string): Promise<WeightedTask | null> {
    for (const task of this.taskMap.values()) {
      if (task.sharedTaskId === sharedTaskId) return task;
    }
    return null;
  }

  // Plan 2: stamp sharedTaskId onto an existing local Task. Acquired
  // inside the workspace mutex so the read-modify-write is atomic against
  // any concurrent updateTask / claimTask on the same row.
  async linkSharedTask(localTaskId: string, sharedTaskId: string): Promise<WeightedTask | null> {
    return this.workspaceMutex.runExclusive(async () => {
      const existing = this.taskMap.get(localTaskId);
      if (!existing) return null;
      const updated: WeightedTask = {
        ...existing,
        sharedTaskId,
        version: (existing.version ?? 1) + 1,
        updatedAt: new Date().toISOString(),
      };
      this.taskMap.set(localTaskId, updated);
      await this._saveLocked();
      return updated;
    });
  }

  async claimTask(id: string, agentId: string): Promise<WeightedTask | null> {
    return this.workspaceMutex.runExclusive(async () => {
      const existing = this.taskMap.get(id);
      if (!existing) return null;

      const terminalStatuses = new Set(['complete', 'completed', 'cancelled']);
      if (terminalStatuses.has(existing.status)) {
        throw new TaskNotClaimableError(id, existing.status);
      }

      if (existing.status === 'in_progress') {
        if (existing.claimedBy === agentId) {
          // Idempotent: same agent re-claiming its own task. Common on
          // network retries — silently succeed with the current state.
          return existing;
        }
        if (existing.claimedBy !== undefined) {
          throw new TaskAlreadyClaimedError(id, existing.claimedBy);
        }
        // Task is in_progress but unclaimed (claimed via legacy update_task
        // flow without agentId). Allow this agent to claim it — resolves
        // the legacy-vs-claim ambiguity in favor of structured claiming.
      }

      // Transition to in_progress + claim
      const now = new Date().toISOString();
      const startedAt = existing.startedAt ?? now;
      const previousStatus = existing.status;

      const claimed: WeightedTask = {
        ...existing,
        status: 'in_progress',
        claimedBy: agentId,
        startedAt,
        updatedAt: now,
        version: (existing.version ?? 1) + 1,
      };
      this.taskMap.set(id, claimed);
      this.taskHeap.update(id, claimed);

      // V4.2: log a selection event (matches the update_task(status:
      // 'in_progress') auto-logging pattern). ML pipeline uses this to learn
      // which tasks agents pick and in what order.
      if (previousStatus !== 'in_progress') {
        await this.logTaskSelection(id);
      }

      await this._saveLocked();
      return claimed;
    });
  }

  async deleteTask(id: string): Promise<boolean> {
    if (!this.taskMap.has(id)) return false;
    
    // Snapshot before state for rebalance logging
    const tasksBefore = this.getTaskArray();
    
    const deletedTask = this.taskMap.get(id)!;
    const hadDependents = this.getTaskArray().some(t => 
      t.dependencies?.includes(id) || t.blocking === id
    );
    
    // Remove from Map
    this.taskMap.delete(id);
    contextSwitchCounts.delete(id);
    
    // Recalculate if deleted task was blocking others
    if (hadDependents || deletedTask.blocking || deletedTask.dependencies?.length) {
      const recalculated = recalculateAllScores(this.getTaskArray(), this.globalML.heuristicWeights);
      this.taskMap.clear();
      for (const task of recalculated) {
        this.taskMap.set(task.id, task);
      }
      this.rebuildHeap();
      
      // Log rebalance event
      this.logRebalanceEvent('task_deleted', tasksBefore, this.getTaskArray(), id);
    } else {
      this.taskHeap.remove(id);
    }
    
    await this.save();
    return true;
  }

  /**
   * V2: Recalculate all priority scores
   * Useful after changing heuristic weights or bulk updates
   */
  async recalculateAllPriorities(): Promise<WeightedTask[]> {
    // Snapshot before state for rebalance logging
    const tasksBefore = this.getTaskArray();
    
    const recalculated = recalculateAllScores(this.getTaskArray(), this.globalML.heuristicWeights);
    this.taskMap.clear();
    for (const task of recalculated) {
      this.taskMap.set(task.id, task);
    }
    this.rebuildHeap();
    
    // Log rebalance event
    this.logRebalanceEvent('weights_changed', tasksBefore, this.getTaskArray());
    
    await this.save();
    return this.taskHeap.toSortedArray();
  }

  /**
   * V2: Update heuristic weights and recalculate all scores
   */
  async updateHeuristicWeights(weights: UpdateHeuristicWeightsDTO): Promise<HeuristicWeights> {
    // Snapshot before state for rebalance logging
    const tasksBefore = this.getTaskArray();
    
    this.globalML.heuristicWeights = {
      ...this.globalML.heuristicWeights,
      ...weights,
    };
    
    // Recalculate all task scores with new weights
    const recalculated = recalculateAllScores(this.getTaskArray(), this.globalML.heuristicWeights);
    this.taskMap.clear();
    for (const task of recalculated) {
      this.taskMap.set(task.id, task);
    }
    this.rebuildHeap();
    
    // Log rebalance event (weights changed always logs)
    this.logRebalanceEvent('weights_changed', tasksBefore, this.getTaskArray());

    // CONC-006: atomic dual-write — heuristic weights mutation and the
    // resulting recalculation of task scores must land on disk together.
    await this.saveBoth();

    return this.globalML.heuristicWeights;
  }

  /**
   * V2: Get current heuristic weights
   * V4: Now from global ML data
   */
  async getHeuristicWeights(): Promise<HeuristicWeights> {
    return this.globalML.heuristicWeights;
  }

  // Data Gaps
  async getDataGaps(): Promise<DataGap[]> {
    const priorityOrder: Record<Priority, number> = { P0: 0, P1: 1, P2: 2, P3: 3 };
    return [...this.db.dataGaps].sort((a, b) => priorityOrder[a.priority] - priorityOrder[b.priority]);
  }

  async getDataGap(id: string): Promise<DataGap | null> {
    return this.db.dataGaps.find(d => d.id === id) || null;
  }

  async createDataGap(data: CreateDataGapDTO): Promise<DataGap> {
    const now = new Date().toISOString();
    const gap: DataGap = {
      id: uuidv4(),
      element: data.element,
      coverage: data.coverage,
      priority: data.priority,
      impact: data.impact,
      effort: data.effort,
      createdAt: now,
      updatedAt: now,
    };
    this.db.dataGaps.push(gap);
    await this.save();
    return gap;
  }

  async updateDataGap(id: string, data: UpdateDataGapDTO): Promise<DataGap | null> {
    const idx = this.db.dataGaps.findIndex(d => d.id === id);
    if (idx === -1) return null;
    
    this.db.dataGaps[idx] = {
      ...this.db.dataGaps[idx],
      ...data,
      updatedAt: new Date().toISOString(),
    };
    await this.save();
    return this.db.dataGaps[idx];
  }

  async deleteDataGap(id: string): Promise<boolean> {
    const idx = this.db.dataGaps.findIndex(d => d.id === id);
    if (idx === -1) return false;
    this.db.dataGaps.splice(idx, 1);
    await this.save();
    return true;
  }

  // Decisions
  async getDecisions(): Promise<Decision[]> {
    return [...this.db.decisions].sort((a, b) => 
      new Date(b.date).getTime() - new Date(a.date).getTime()
    );
  }

  async createDecision(data: CreateDecisionDTO): Promise<Decision> {
    // V4.4: best-effort cross-forge link from whichever task this decision is about
    const decisionTaskId = data.relatedTaskId || data.selectedTaskId;
    const decision: Decision = {
      id: uuidv4(),
      date: data.date,
      decision: data.decision,
      rationale: data.rationale,
      createdAt: new Date().toISOString(),
      decisionType: data.decisionType || 'other',
      relatedTaskId: data.relatedTaskId,
      skippedTaskId: data.skippedTaskId,
      selectedTaskId: data.selectedTaskId,
      workspaceId: this.currentWorkspaceId || undefined,
      sharedTaskId: decisionTaskId ? this.taskMap.get(decisionTaskId)?.sharedTaskId : undefined,
    };
    this.db.decisions.push(decision);
    // V4.2: Dual-write to global ML for training data
    this.globalML.decisions.push(decision);
    // CONC-006: atomic dual-write — db.decisions (workspace) and
    // globalML.decisions must stay in sync.
    await this.saveBoth();
    return decision;
  }

  // V3 Prep: Context Switch Tracking
  async logContextSwitch(taskId: string): Promise<void> {
    const count = contextSwitchCounts.get(taskId) || 0;
    contextSwitchCounts.set(taskId, count + 1);
  }

  async completeTask(
    taskId: string,
    outcome: 'completed' | 'cancelled' | 'deferred'
  ): Promise<TaskCompletionRecord | null> {
    const task = this.taskMap.get(taskId);
    if (!task) return null;

    // Snapshot before state for rebalance logging
    const tasksBefore = this.getTaskArray();

    // V4.2: If the task was completed without ever transitioning through
    // in_progress, synthesize that transition now — fire a TaskSelectionEvent
    // (which auto-creates skip decisions for higher-ranked tasks) and use
    // task.createdAt as the effective startedAt. This guarantees ML signal
    // capture even when callers forget the explicit in_progress hop.
    // The synthesizedSelection flag on the record makes this filterable
    // during training (the work-window is artificial — workTime ≈ 0).
    let effectiveStartedAt = task.startedAt;
    let synthesizedSelection = false;
    if (!effectiveStartedAt) {
      effectiveStartedAt = task.createdAt;
      synthesizedSelection = true;
      console.log(`📊 V4.2: Task ${taskId} completed without prior in_progress — synthesizing selection event (startedAt ← createdAt)`);
      // Fire selection BEFORE marking complete so the heap still ranks this task.
      await this.logTaskSelection(taskId);
    }

    const completedAt = new Date().toISOString();
    const createdAt = new Date(task.createdAt).getTime();
    const completedTime = new Date(completedAt).getTime();
    const hoursElapsed = (completedTime - createdAt) / (1000 * 60 * 60);

    // V4.0: Calculate actual work time (from startedAt, not createdAt).
    // V4.2: effectiveStartedAt is always set now (synthesized to createdAt
    // if the task never went through in_progress), so workTime is always
    // computed — for synthesized completions it'll equal hoursElapsed.
    const startedTime = new Date(effectiveStartedAt).getTime();
    const actualWorkTime = Math.round(((completedTime - startedTime) / (1000 * 60 * 60)) * 100) / 100;
    if (synthesizedSelection) {
      console.log(`📊 V4.2: Task ${taskId} synthesized work duration: ${actualWorkTime}h (uses createdAt; treat as queue-time)`);
    } else {
      console.log(`📊 V4.0: Task ${taskId} work duration: ${actualWorkTime}h (started: ${task.startedAt})`);
    }

    // Count priority changes for this task (from global ML data)
    const priorityChangeCount = this.globalML.priorityChangeEvents
      .filter(e => e.taskId === taskId).length;

    const record: TaskCompletionRecord = {
      id: uuidv4(),
      taskId,
      actualCompletionTime: Math.round(hoursElapsed * 100) / 100,
      wasBlocking: !!task.blocking,
      userOverrideCount: priorityChangeCount,
      contextSwitchCount: contextSwitchCounts.get(taskId) || 0,
      outcome,
      completedAt,
      // V3: Capture score at completion for training
      initialPriorityScore: task.priorityScore,
      finalPriorityScore: task.priorityScore,
      // V4.0/V4.2: Actual work duration tracking — startedAt may be synthesized
      startedAt: effectiveStartedAt,
      actualWorkTime,
      // V4: Tag with workspace for ML filtering
      workspaceId: this.currentWorkspaceId || undefined,
      // V4.4: cross-forge continuity across release→reclaim id churn
      sharedTaskId: task.sharedTaskId,
      // V4.2: Mark synthesized completions so trainers can filter
      synthesizedSelection: synthesizedSelection || undefined,
    };

    // V4: Write to global ML file (not workspace-scoped)
    this.globalML.completionRecords.push(record);
    contextSwitchCounts.delete(taskId);

    // Update task status in Map (persist synthesized startedAt for consistency)
    const updatedTask: WeightedTask = {
      ...task,
      status: 'complete',
      updatedAt: new Date().toISOString(),
      startedAt: effectiveStartedAt,
    };
    this.taskMap.set(taskId, updatedTask);

    // Recalculate all tasks - completing a task changes dependency graph
    // Tasks that depended on this one now have lower dependencyDepth
    // Tasks blocked by this one now have different blocking relationships
    const recalculated = recalculateAllScores(this.getTaskArray(), this.globalML.heuristicWeights);
    this.taskMap.clear();
    for (const t of recalculated) {
      this.taskMap.set(t.id, t);
    }
    this.rebuildHeap();
    
    // Log rebalance event
    this.logRebalanceEvent('task_completed', tasksBefore, this.getTaskArray(), taskId);

    // CONC-006: atomic dual-write — completion record (globalML) and task
    // status transition (workspace) must land together.
    await this.saveBoth();
    return record;
  }

  async getCompletionRecords(): Promise<TaskCompletionRecord[]> {
    // V4: Return from global ML data
    return this.globalML.completionRecords;
  }

  // ========== V3: ML Training Data Methods ==========

  /**
   * V3: Log when user selects a task to work on
   * This captures user preference signals for training
   * V4.1: Enhanced with skipped task tracking and pairwise preferences
   */
  async logTaskSelection(selectedTaskId: string): Promise<TaskSelectionEvent | null> {
    const selectedTask = this.taskMap.get(selectedTaskId);
    if (!selectedTask) return null;

    const sorted = this.taskHeap.toSortedArray().filter(t => t.status !== 'complete');
    const topTask = sorted[0];
    const selectedRank = sorted.findIndex(t => t.id === selectedTaskId);

    // V4.1: Collect all skipped task IDs (tasks ranked higher than selected)
    const skippedTaskIds: string[] = [];
    for (let i = 0; i < selectedRank; i++) {
      skippedTaskIds.push(sorted[i].id);
    }

    // V4.1: Generate pairwise preferences (user prefers selected over all skipped)
    // This is the ML gold - each skip is an implicit preference signal
    const implicitPreferences: TaskSelectionEvent['implicitPreferences'] = [];
    for (let i = 0; i < selectedRank; i++) {
      const skippedTask = sorted[i];
      implicitPreferences.push({
        preferredTaskId: selectedTaskId,
        skippedTaskId: skippedTask.id,
        // Positive scoreDiff means heuristics ranked skipped higher but user disagreed
        scoreDiff: selectedTask.priorityScore - skippedTask.priorityScore,
      });
    }

    // V4.1: Capture feature snapshot for offline retraining
    const selectedTaskFeatures: TaskSelectionEvent['selectedTaskFeatures'] = {
      priority: selectedTask.priority,
      priorityScore: selectedTask.priorityScore,
      weights: selectedTask.weights,
      effort: selectedTask.effort,
      hasDeadline: !!selectedTask.deadline,
      hasBlocking: !!selectedTask.blocking,
      hasDependencies: !!(selectedTask.dependencies && selectedTask.dependencies.length > 0),
    };

    const event: TaskSelectionEvent = {
      id: uuidv4(),
      selectedTaskId,
      selectedTaskScore: selectedTask.priorityScore,
      selectedTaskRank: selectedRank,
      topTaskId: topTask?.id || selectedTaskId,
      topTaskScore: topTask?.priorityScore || selectedTask.priorityScore,
      queueSize: sorted.length,
      wasTopSelected: selectedTaskId === topTask?.id,
      timestamp: new Date().toISOString(),
      // V4: Tag with workspace for ML filtering
      workspaceId: this.currentWorkspaceId || undefined,
      // V4.4: cross-forge continuity across release→reclaim id churn
      sharedTaskId: selectedTask.sharedTaskId,
      // V4.1: Enhanced learning signals
      skippedTaskIds: skippedTaskIds.length > 0 ? skippedTaskIds : undefined,
      implicitPreferences: implicitPreferences.length > 0 ? implicitPreferences : undefined,
      selectedTaskFeatures,
    };

    // V4: Write to global ML file
    this.globalML.taskSelectionEvents.push(event);
    
    const skippedCount = skippedTaskIds.length;
    console.log(`📊 V4.1: Logged task selection: ${selectedTaskId} (rank ${selectedRank + 1}/${sorted.length}, skipped ${skippedCount} tasks, was_top: ${event.wasTopSelected})`);
    if (skippedCount > 0) {
      console.log(`  ↳ Generated ${implicitPreferences.length} pairwise preferences from skipped tasks`);
    }
    
    // V4: Save global ML data
    await this.saveGlobalML();
    return event;
  }

  /**
   * V3: Get all priority change events
   * V4: Now from global ML data
   */
  async getPriorityChangeEvents(): Promise<PriorityChangeEvent[]> {
    return this.globalML.priorityChangeEvents;
  }

  /**
   * V3: Get all task selection events
   * V4: Now from global ML data
   */
  async getTaskSelectionEvents(): Promise<TaskSelectionEvent[]> {
    return this.globalML.taskSelectionEvents;
  }

  /**
   * V3: Get all queue rebalance events
   * V4: Now from global ML data
   */
  async getQueueRebalanceEvents(): Promise<QueueRebalanceEvent[]> {
    return this.globalML.queueRebalanceEvents;
  }

  /**
   * V3: Export training data for XGBoost
   * Returns structured data ready for ML training
   * Handles null values with sensible defaults for backward compatibility
   */
  async exportTrainingData(): Promise<{
    completionRecords: TaskCompletionRecord[];
    priorityChangeEvents: PriorityChangeEvent[];
    taskSelectionEvents: TaskSelectionEvent[];
    queueRebalanceEvents: QueueRebalanceEvent[];
    decisions: Decision[];
    tasks: WeightedTask[];
    heuristicWeights: HeuristicWeights;
    summary: {
      totalCompletions: number;
      totalPriorityChanges: number;
      totalSelections: number;
      totalRebalances: number;
      totalDecisions: number;
      totalSkipDecisions: number;
      selectionAccuracy: number;
      dataQuality: {
        completionsWithScores: number;
        completionsWithWorkTime: number;  // V4.0: How many completions have reliable work duration
        tasksWithEffort: number;
        tasksWithDependencies: number;
        rebalancesWithSignificantChanges: number;
        selectionsWithPairwiseData: number;  // V4.1: How many selections have pairwise preferences
        totalSelectionPairs: number;  // V4.1: Total pairwise preferences from selections
        decisionsWithSkipData: number;
        decisionsLinkedToSelections: number;
      };
    };
    // ML-ready format with nulls handled
    mlReady: {
      completions: Array<{
        taskId: string;
        completionTimeHours: number;
        workTimeHours: number;     // V4.0: Actual work time (startedAt → completedAt)
        queueTimeHours: number;    // V4.0: Queue time (createdAt → startedAt)
        hasWorkTimeData: number;   // V4.0: 1 if workTimeHours is reliable, 0 if fallback
        wasBlocking: number;  // 0 or 1
        outcome: string;
        initialScore: number;
        finalScore: number;
        scoreDelta: number;
      }>;
      tasks: Array<{
        id: string;
        priority: number;  // P0=0, P1=1, P2=2, P3=3
        priorityScore: number;
        effort: number;  // low=1, medium=2, high=3, unknown=2
        blockingCount: number;
        crossProjectImpact: number;
        timeSensitivity: number;
        effortValueRatio: number;
        dependencyDepth: number;
        hasDependencies: number;  // 0 or 1
        hasBlocking: number;  // 0 or 1
      }>;
      // V3: Rebalance trajectory data
      rebalances: Array<{
        trigger: string;
        queueSizeBefore: number;
        queueSizeAfter: number;
        significantChangeCount: number;
        topTaskChanged: number;  // 0 or 1
      }>;
      // V4.1: Selection pairwise preferences (combined with drag reorder for training)
      selectionPairs: Array<{
        preferredTaskId: string;
        skippedTaskId: string;
        scoreDiff: number;  // positive = heuristics got it wrong
        timestamp: string;
        queueSize: number;
      }>;
      // V4.2: Decision records for ML
      decisions: Array<{
        id: string;
        decisionType: string;
        relatedTaskId: string | null;
        skippedTaskId: string | null;
        selectedTaskId: string | null;
        rankDelta: number;
        hasLinkedSelectionEvent: number;
        timestamp: string;
      }>;
    };
  }> {
    // V4: Read from global ML data
    const completionRecords = this.globalML.completionRecords;
    const priorityChangeEvents = this.globalML.priorityChangeEvents;
    const taskSelectionEvents = this.globalML.taskSelectionEvents;
    const queueRebalanceEvents = this.globalML.queueRebalanceEvents;
    const tasks = this.getTaskArray();
    
    const topSelections = taskSelectionEvents.filter(e => e.wasTopSelected).length;
    const selectionAccuracy = taskSelectionEvents.length > 0
      ? (topSelections / taskSelectionEvents.length) * 100
      : 0;

    // Data quality metrics
    const completionsWithScores = completionRecords.filter(
      r => r.initialPriorityScore !== undefined
    ).length;
    // V4.0: Track completions with actual work time data
    const completionsWithWorkTime = completionRecords.filter(
      r => r.actualWorkTime !== undefined
    ).length;
    const tasksWithEffort = tasks.filter(t => t.effort).length;
    const tasksWithDependencies = tasks.filter(
      t => t.dependencies && t.dependencies.length > 0
    ).length;

    // Priority mapping
    const priorityMap: Record<string, number> = { P0: 0, P1: 1, P2: 2, P3: 3 };
    const effortMap: Record<string, number> = { low: 1, medium: 2, high: 3 };

    // V4.4: index live tasks by sharedTaskId so completion records whose original
    // local id was archived (the release→reclaim churn) still hydrate via the
    // stable cross-forge key instead of silently dropping out of training.
    const bySharedTaskId = new Map<string, WeightedTask>();
    for (const t of this.taskMap.values()) {
      if (t.sharedTaskId) bySharedTaskId.set(t.sharedTaskId, t);
    }

    // ML-ready completions with defaults for missing values
    const mlCompletions = completionRecords.map(r => {
      const task =
        this.taskMap.get(r.taskId) ??
        (r.sharedTaskId ? bySharedTaskId.get(r.sharedTaskId) : undefined);
      const initialScore = r.initialPriorityScore ?? task?.priorityScore ?? 0;
      const finalScore = r.finalPriorityScore ?? task?.priorityScore ?? 0;
      // V4.0: Include work time and queue time separately
      const hasWorkTime = r.actualWorkTime !== undefined;
      const queueTimeHours = hasWorkTime && r.startedAt
        ? Math.round(((new Date(r.startedAt).getTime() - new Date(task?.createdAt || r.startedAt).getTime()) / (1000 * 60 * 60)) * 100) / 100
        : r.actualCompletionTime;  // Fallback: use total time as queue time
      return {
        taskId: r.taskId,
        completionTimeHours: r.actualCompletionTime,
        workTimeHours: r.actualWorkTime ?? r.actualCompletionTime,  // Fallback: use total time
        queueTimeHours,
        hasWorkTimeData: hasWorkTime ? 1 : 0,  // Flag for ML to know if data is reliable
        wasBlocking: r.wasBlocking ? 1 : 0,
        outcome: r.outcome,
        initialScore,
        finalScore,
        scoreDelta: finalScore - initialScore,
      };
    });

    // ML-ready tasks with defaults for missing values
    const mlTasks = tasks.map(t => ({
      id: t.id,
      priority: priorityMap[t.priority] ?? 1,
      priorityScore: t.priorityScore,
      effort: effortMap[t.effort || 'medium'] ?? 2,  // Default: medium
      blockingCount: t.weights?.blockingCount ?? 0,
      crossProjectImpact: t.weights?.crossProjectImpact ?? 0,
      timeSensitivity: t.weights?.timeSensitivity ?? 0,
      effortValueRatio: t.weights?.effortValueRatio ?? 5,
      dependencyDepth: t.weights?.dependencyDepth ?? 0,
      hasDependencies: (t.dependencies && t.dependencies.length > 0) ? 1 : 0,
      hasBlocking: t.blocking ? 1 : 0,
    }));

    // ML-ready rebalance events
    const mlRebalances = queueRebalanceEvents.map(r => ({
      trigger: r.trigger,
      queueSizeBefore: r.queueSizeBefore,
      queueSizeAfter: r.queueSizeAfter,
      significantChangeCount: r.significantChanges.length,
      topTaskChanged: r.topTasksBefore[0] !== r.topTasksAfter[0] ? 1 : 0,
    }));

    // Rebalances with significant changes
    const rebalancesWithSignificantChanges = queueRebalanceEvents.filter(
      r => r.significantChanges.length > 0
    ).length;

    // V4.1: Count selections with pairwise data and total pairs
    const selectionsWithPairwiseData = taskSelectionEvents.filter(
      e => e.implicitPreferences && e.implicitPreferences.length > 0
    ).length;
    const totalSelectionPairs = taskSelectionEvents.reduce(
      (sum, e) => sum + (e.implicitPreferences?.length || 0), 0
    );

    // V4.1: ML-ready selection pairwise preferences
    const mlSelectionPairs: Array<{
      preferredTaskId: string;
      skippedTaskId: string;
      scoreDiff: number;
      timestamp: string;
      queueSize: number;
    }> = [];
    for (const event of taskSelectionEvents) {
      if (event.implicitPreferences) {
        for (const pref of event.implicitPreferences) {
          mlSelectionPairs.push({
            preferredTaskId: pref.preferredTaskId,
            skippedTaskId: pref.skippedTaskId,
            scoreDiff: pref.scoreDiff,
            timestamp: event.timestamp,
            queueSize: event.queueSize,
          });
        }
      }
    }

    // V4.2: Decision metrics
    const decisions = this.globalML.decisions;
    const skipDecisions = decisions.filter(d => d.decisionType === 'skip');
    const decisionsWithSkipData = decisions.filter(
      d => d.decisionType === 'skip' && d.skippedTaskId && d.selectedTaskId
    ).length;
    const decisionsLinkedToSelections = decisions.filter(d => d.selectionEventId).length;

    const mlDecisions = decisions.map(d => ({
      id: d.id,
      decisionType: d.decisionType || 'other',
      relatedTaskId: d.relatedTaskId || null,
      skippedTaskId: d.skippedTaskId || null,
      selectedTaskId: d.selectedTaskId || null,
      rankDelta: d.rankDelta || 0,
      hasLinkedSelectionEvent: d.selectionEventId ? 1 : 0,
      timestamp: d.createdAt,
    }));

    return {
      completionRecords,
      priorityChangeEvents,
      taskSelectionEvents,
      queueRebalanceEvents,
      decisions,
      tasks,
      heuristicWeights: this.globalML.heuristicWeights,
      summary: {
        totalCompletions: completionRecords.length,
        totalPriorityChanges: priorityChangeEvents.length,
        totalSelections: taskSelectionEvents.length,
        totalRebalances: queueRebalanceEvents.length,
        totalDecisions: decisions.length,
        totalSkipDecisions: skipDecisions.length,
        selectionAccuracy,
        dataQuality: {
          completionsWithScores,
          completionsWithWorkTime,  // V4.0: Tracks how many have actual work duration
          tasksWithEffort,
          tasksWithDependencies,
          rebalancesWithSignificantChanges,
          selectionsWithPairwiseData,  // V4.1: Selections with skipped task data
          totalSelectionPairs,  // V4.1: Total pairwise preferences from selections
          decisionsWithSkipData,
          decisionsLinkedToSelections,
        },
      },
      mlReady: {
        completions: mlCompletions,
        tasks: mlTasks,
        rebalances: mlRebalances,
        selectionPairs: mlSelectionPairs,  // V4.1: Pairwise preferences from task selections
        decisions: mlDecisions,
      },
    };
  }

  // ========== V4.2: Backfill Skip Decisions ==========

  async backfillSkipDecisions(dryRun: boolean = false): Promise<{
    created: number;
    skipped: number;
    details: Array<{ selectionEventId: string; selectedTaskId: string; topTaskId: string; rankDelta: number }>;
  }> {
    const existingLinkedEventIds = new Set(
      this.globalML.decisions
        .filter(d => d.selectionEventId)
        .map(d => d.selectionEventId)
    );

    const toCreate: Decision[] = [];

    for (const event of this.globalML.taskSelectionEvents) {
      if (event.wasTopSelected) continue;
      if (existingLinkedEventIds.has(event.id)) continue;

      const selectedTask = this.taskMap.get(event.selectedTaskId);
      const topTask = this.taskMap.get(event.topTaskId);

      const decision: Decision = {
        id: uuidv4(),
        date: event.timestamp.split('T')[0],
        decision: `[Backfilled] Selected "${selectedTask?.task || event.selectedTaskId}" (rank ${event.selectedTaskRank + 1}) over recommended "${topTask?.task || event.topTaskId}" (rank 1)`,
        rationale: `Reconstructed from TaskSelectionEvent ${event.id}. Rank ${event.selectedTaskRank + 1}/${event.queueSize}, score diff: ${event.selectedTaskScore - event.topTaskScore}.`,
        createdAt: event.timestamp,
        decisionType: 'skip',
        relatedTaskId: event.selectedTaskId,
        skippedTaskId: event.topTaskId,
        selectedTaskId: event.selectedTaskId,
        rankDelta: event.selectedTaskRank,
        selectionEventId: event.id,
        workspaceId: event.workspaceId,
      };
      toCreate.push(decision);
    }

    if (!dryRun && toCreate.length > 0) {
      this.globalML.decisions.push(...toCreate);
      for (const d of toCreate) {
        if (d.workspaceId === this.currentWorkspaceId) {
          this.db.decisions.push(d);
        }
      }
      // CONC-006: atomic dual-write — both workspace decisions and globalML
      // decisions arrays must land together.
      await this.saveBoth();
      console.log(`📊 V4.2: Backfilled ${toCreate.length} skip decisions from historical selection events`);
    }

    const alreadyLinked = this.globalML.taskSelectionEvents.filter(
      e => !e.wasTopSelected && existingLinkedEventIds.has(e.id)
    ).length;

    return {
      created: dryRun ? 0 : toCreate.length,
      skipped: alreadyLinked,
      details: toCreate.map(d => ({
        selectionEventId: d.selectionEventId!,
        selectedTaskId: d.selectedTaskId!,
        topTaskId: d.skippedTaskId!,
        rankDelta: d.rankDelta!,
      })),
    };
  }

  // ========== V3.2: Online Learning Methods ==========

  /**
   * V3.2: Log a drag-and-drop reorder event and optionally update weights
   * This is the core entry point for the online learning system.
   * 
   * @param dto - The drag reorder data from the frontend
   * @returns The created DragReorderEvent with applied weight updates
   */
  async logDragReorder(dto: LogDragReorderDTO): Promise<DragReorderEvent> {
    const { taskId, fromRank, toRank } = dto;
    
    // Get current sorted queue (non-completed tasks only)
    const sortedTasks = this.taskHeap.toSortedArray().filter(t => t.status !== 'complete');
    const draggedTask = sortedTasks.find(t => t.id === taskId);
    
    if (!draggedTask) {
      throw new Error(`Task ${taskId} not found in queue`);
    }
    
    const direction = toRank < fromRank ? 'promoted' : 'demoted';
    
    // Generate implicit pairwise preferences
    // If promoted (moved up), we prefer dragged task over tasks it passed
    // If demoted (moved down), tasks it passed are preferred over it
    const implicitPreferences: DragReorderEvent['implicitPreferences'] = [];
    const tasksPassedIds: string[] = [];
    
    if (direction === 'promoted') {
      // Task moved from rank 5 to rank 2 → prefers over tasks at 2, 3, 4
      for (let i = toRank; i < fromRank; i++) {
        if (i < sortedTasks.length) {
          const passedTask = sortedTasks[i];
          tasksPassedIds.push(passedTask.id);
          implicitPreferences.push({
            preferredTaskId: taskId,
            demotedTaskId: passedTask.id,
            scoreDiff: draggedTask.priorityScore - passedTask.priorityScore,
          });
        }
      }
    } else {
      // Task moved from rank 2 to rank 5 → tasks at 3, 4, 5 preferred over it
      for (let i = fromRank + 1; i <= toRank; i++) {
        if (i < sortedTasks.length) {
          const passedTask = sortedTasks[i];
          tasksPassedIds.push(passedTask.id);
          implicitPreferences.push({
            preferredTaskId: passedTask.id,
            demotedTaskId: taskId,
            scoreDiff: passedTask.priorityScore - draggedTask.priorityScore,
          });
        }
      }
    }
    
    // Compute weight update if online learning is enabled (from global ML state)
    let appliedWeightDelta: Partial<HeuristicWeights> | undefined;
    const learnerState = this.globalML.onlineLearnerState;
    
    if (learnerState.enabled && implicitPreferences.length > 0) {
      appliedWeightDelta = await this.computeOnlineWeightUpdate(
        implicitPreferences,
        sortedTasks,
        learnerState
      );
    }
    
    // Create the drag event
    const event: DragReorderEvent = {
      id: uuidv4(),
      taskId,
      fromRank,
      toRank,
      direction,
      timestamp: new Date().toISOString(),
      implicitPreferences,
      draggedTaskFeatures: {
        priority: draggedTask.priority,
        priorityScore: draggedTask.priorityScore,
        weights: draggedTask.weights,
        effort: draggedTask.effort,
        hasDeadline: !!draggedTask.deadline,
        hasBlocking: !!draggedTask.blocking,
        hasDependencies: !!(draggedTask.dependencies && draggedTask.dependencies.length > 0),
      },
      appliedWeightDelta,
      queueSize: sortedTasks.length,
      tasksPassedIds,
      // V4: Tag with workspace for ML filtering
      workspaceId: this.currentWorkspaceId || undefined,
      // V4.4: cross-forge continuity across release→reclaim id churn
      sharedTaskId: draggedTask.sharedTaskId,
    };

    // V4: Store in global ML file
    this.globalML.dragReorderEvents.push(event);
    
    // Update learner state in global ML
    this.globalML.onlineLearnerState.totalUpdates++;
    this.globalML.onlineLearnerState.lastUpdateTimestamp = new Date().toISOString();
    this.globalML.onlineLearnerState.totalPairs += implicitPreferences.length;
    
    // Count correct predictions (where heuristics agreed with user)
    const correctCount = implicitPreferences.filter(p => p.scoreDiff < 0).length;
    this.globalML.onlineLearnerState.correctPredictions += correctCount;
    
    console.log(`📊 V3.2: Logged drag reorder: ${taskId} ${fromRank} → ${toRank} (${direction})`);
    console.log(`  ↳ Generated ${implicitPreferences.length} pairwise preferences`);
    if (appliedWeightDelta) {
      console.log(`  ↳ Applied weight update:`, appliedWeightDelta);
    }

    // CONC-006: atomic dual-write — drag reorder event (globalML) and
    // resulting weight-change side effects (workspace) must land together.
    await this.saveBoth();
    return event;
  }

  /**
   * V3.2: Compute and apply weight updates using SGD with momentum
   * 
   * The loss function is pairwise ranking loss (hinge loss):
   *   L = Σ max(0, margin - (score_preferred - score_demoted))
   * 
   * Gradient update moves weights to increase score of preferred task
   * relative to demoted task.
   */
  private async computeOnlineWeightUpdate(
    preferences: DragReorderEvent['implicitPreferences'],
    sortedTasks: WeightedTask[],
    learnerState: OnlineLearnerState
  ): Promise<Partial<HeuristicWeights>> {
    const { learningRate, momentum, maxWeightChange, minWeight, maxWeight } = learnerState;
    const taskMap = new Map(sortedTasks.map(t => [t.id, t]));
    
    // Accumulate gradients across all preferences
    const gradient: HeuristicWeights = {
      blocking: 0,
      crossProject: 0,
      timeSensitive: 0,
      effortValue: 0,
      dependency: 0,
    };
    
    const margin = 1.0; // Hinge loss margin
    let totalLoss = 0;
    
    for (const pref of preferences) {
      const preferred = taskMap.get(pref.preferredTaskId);
      const demoted = taskMap.get(pref.demotedTaskId);
      
      if (!preferred || !demoted) continue;
      
      // Current score difference (should be negative for correct prediction)
      const scoreDiff = preferred.priorityScore - demoted.priorityScore;
      
      // Hinge loss: max(0, margin + scoreDiff)
      // We want preferred.score < demoted.score (lower score = higher priority)
      const loss = Math.max(0, margin + scoreDiff);
      totalLoss += loss;
      
      if (loss > 0) {
        // Gradient: ∂L/∂w_i = (feature_preferred - feature_demoted)
        // We want to DECREASE score of preferred and INCREASE score of demoted
        // Since score = base - weighted_sum, we need to INCREASE features of preferred
        const prefWeights = preferred.weights;
        const demWeights = demoted.weights;
        
        gradient.blocking += (prefWeights.blockingCount - demWeights.blockingCount);
        gradient.crossProject += (prefWeights.crossProjectImpact - demWeights.crossProjectImpact);
        gradient.timeSensitive += (prefWeights.timeSensitivity - demWeights.timeSensitivity);
        gradient.effortValue += (prefWeights.effortValueRatio - demWeights.effortValueRatio);
        gradient.dependency += (prefWeights.dependencyDepth - demWeights.dependencyDepth);
      }
    }
    
    // Normalize gradient by number of preferences
    const n = preferences.length || 1;
    gradient.blocking /= n;
    gradient.crossProject /= n;
    gradient.timeSensitive /= n;
    gradient.effortValue /= n;
    gradient.dependency /= n;
    
    // Apply momentum
    const momentumBuffer = learnerState.momentumBuffer;
    momentumBuffer.blocking = momentum * momentumBuffer.blocking + gradient.blocking;
    momentumBuffer.crossProject = momentum * momentumBuffer.crossProject + gradient.crossProject;
    momentumBuffer.timeSensitive = momentum * momentumBuffer.timeSensitive + gradient.timeSensitive;
    momentumBuffer.effortValue = momentum * momentumBuffer.effortValue + gradient.effortValue;
    momentumBuffer.dependency = momentum * momentumBuffer.dependency + gradient.dependency;
    
    // Compute weight delta (clamped)
    const clamp = (val: number) => Math.max(-maxWeightChange, Math.min(maxWeightChange, val));
    const weightDelta: Partial<HeuristicWeights> = {
      blocking: clamp(learningRate * momentumBuffer.blocking),
      crossProject: clamp(learningRate * momentumBuffer.crossProject),
      timeSensitive: clamp(learningRate * momentumBuffer.timeSensitive),
      effortValue: clamp(learningRate * momentumBuffer.effortValue),
      dependency: clamp(learningRate * momentumBuffer.dependency),
    };
    
    // Apply to weights (with min/max bounds)
    const bound = (val: number) => Math.max(minWeight, Math.min(maxWeight, val));
    const newWeights: HeuristicWeights = {
      blocking: bound(this.globalML.heuristicWeights.blocking + (weightDelta.blocking || 0)),
      crossProject: bound(this.globalML.heuristicWeights.crossProject + (weightDelta.crossProject || 0)),
      timeSensitive: bound(this.globalML.heuristicWeights.timeSensitive + (weightDelta.timeSensitive || 0)),
      effortValue: bound(this.globalML.heuristicWeights.effortValue + (weightDelta.effortValue || 0)),
      dependency: bound(this.globalML.heuristicWeights.dependency + (weightDelta.dependency || 0)),
    };
    
    // Only update if there's meaningful change
    const hasChange = Object.values(weightDelta).some(v => v && Math.abs(v) > 0.001);
    if (hasChange) {
      this.globalML.heuristicWeights = newWeights;
      
      // Recalculate all task scores with new weights
      const recalculated = recalculateAllScores(this.getTaskArray(), this.globalML.heuristicWeights);
      this.taskMap.clear();
      for (const task of recalculated) {
        this.taskMap.set(task.id, task);
      }
      this.rebuildHeap();
      
      // Track cumulative loss in global ML state
      this.globalML.onlineLearnerState.cumulativeLoss += totalLoss;
    }
    
    return weightDelta;
  }

  /**
   * V3.2: Get current online learner state
   * V4: Now from global ML data
   */
  async getOnlineLearnerState(): Promise<OnlineLearnerState> {
    return this.globalML.onlineLearnerState;
  }

  /**
   * V3.2: Update online learner configuration
   * V4: Now stored in global ML data
   */
  async updateOnlineLearnerConfig(config: UpdateOnlineLearnerDTO): Promise<OnlineLearnerState> {
    this.globalML.onlineLearnerState = {
      ...this.globalML.onlineLearnerState,
      ...config,
    };
    
    await this.saveGlobalML();
    console.log('📊 V3.2: Updated online learner config:', config);
    return this.globalML.onlineLearnerState;
  }

  /**
   * V3.2: Get all drag reorder events
   * V4: Now from global ML data
   */
  async getDragReorderEvents(): Promise<DragReorderEvent[]> {
    return this.globalML.dragReorderEvents;
  }

  /**
   * V3.2: Get online learning accuracy metrics
   * V4: Now from global ML data
   */
  async getOnlineLearnerMetrics(): Promise<{
    totalUpdates: number;
    totalPairs: number;
    correctPredictions: number;
    accuracy: number;
    cumulativeLoss: number;
    currentWeights: HeuristicWeights;
    learningRate: number;
    enabled: boolean;
  }> {
    const state = this.globalML.onlineLearnerState;
    const accuracy = state.totalPairs > 0 
      ? (state.correctPredictions / state.totalPairs) * 100 
      : 0;
    
    return {
      totalUpdates: state.totalUpdates,
      totalPairs: state.totalPairs,
      correctPredictions: state.correctPredictions,
      accuracy: Math.round(accuracy * 100) / 100,
      cumulativeLoss: Math.round(state.cumulativeLoss * 100) / 100,
      currentWeights: this.globalML.heuristicWeights,
      learningRate: state.learningRate,
      enabled: state.enabled,
    };
  }

  // ========== V4: Workspace Management Methods ==========

  /**
   * V4: Get all workspaces
   */
  async getWorkspaces(): Promise<Workspace[]> {
    const metadata = this.loadWorkspaceMetadata();
    return metadata.workspaces;
  }

  /**
   * V4: Get current workspace ID
   */
  async getCurrentWorkspaceId(): Promise<string | null> {
    return this.currentWorkspaceId;
  }

  /**
   * V4: Get workspace by ID
   */
  async getWorkspace(id: string): Promise<Workspace | null> {
    const metadata = this.loadWorkspaceMetadata();
    return metadata.workspaces.find(w => w.id === id) || null;
  }

  /**
   * Compute freshness / dormancy for a single workspace without switching
   * to it. Reads the workspace's progress.json directly off disk. See
   * `WorkspaceFreshness` in schema.ts for field semantics and rationale.
   *
   * Active iff EITHER:
   *   - any task createdAt/updatedAt within `interactionWindowDays` of now, OR
   *   - any open task with a deadline within `futureDeadlineHorizonDays`.
   */
  async computeWorkspaceFreshness(
    workspaceId: string,
    thresholds: WorkspaceFreshnessThresholds = DEFAULT_WORKSPACE_FRESHNESS_THRESHOLDS,
    now: Date = new Date(),
  ): Promise<WorkspaceFreshness> {
    const { computeFreshness } = await import('../utils/workspaceFreshness');
    const dbFile = this.getWorkspaceDbFile(workspaceId);
    let tasks: WeightedTask[] = [];
    if (fs.existsSync(dbFile)) {
      try {
        const db = JSON.parse(fs.readFileSync(dbFile, 'utf-8')) as { tasks?: WeightedTask[] };
        tasks = db.tasks || [];
      } catch {
        tasks = [];
      }
    }
    return computeFreshness(tasks, thresholds, now);
  }

  /**
   * V4: Create a new workspace
   */
  async createWorkspace(data: CreateWorkspaceDTO): Promise<Workspace> {
    const metadata = this.loadWorkspaceMetadata();
    const now = new Date().toISOString();
    const workspace: Workspace = {
      id: uuidv4(),
      name: data.name,
      description: data.description,
      createdAt: now,
      updatedAt: now,
    };
    
    metadata.workspaces.push(workspace);
    
    // If no current workspace, set this as current
    if (!metadata.currentWorkspaceId) {
      metadata.currentWorkspaceId = workspace.id;
      this.currentWorkspaceId = workspace.id;
    }
    
    this.saveWorkspaceMetadata(metadata);
    
    // Initialize workspace database (empty)
    const dbFile = this.getWorkspaceDbFile(workspace.id);
    if (!fs.existsSync(dbFile)) {
      const empty = getEmptyDatabase();
      fs.writeFileSync(dbFile, JSON.stringify(empty, null, 2));
    }
    
    console.log(`📁 V4: Created workspace "${workspace.name}" (${workspace.id})`);
    return workspace;
  }

  /**
   * V4: Get a seeded database with example data
   */
  private getSeededDatabase(): ProgressDatabase {
    const exampleProjectId = uuidv4();
    return {
      ...getEmptyDatabase(),
      projects: [
        {
          id: exampleProjectId,
          name: 'example-project',
          path: '~/projects/example',
          status: 'active',
          primaryFocus: 'Example project - replace with your own',
          createdAt: new Date().toISOString(),
          updatedAt: new Date().toISOString(),
        },
      ],
      tasks: [
        {
          id: 'EXAMPLE-001',
          priority: 'P1',
          task: 'Replace this with your first task',
          project: 'example-project',
          status: 'not_started',
          notes: 'Delete this example and add your own tasks',
          createdAt: new Date().toISOString(),
          updatedAt: new Date().toISOString(),
          priorityScore: 82,
          weights: {
            blockingCount: 0,
            crossProjectImpact: 0,
            timeSensitivity: 0,
            effortValueRatio: 6,
            dependencyDepth: 0,
          },
        },
      ],
    };
  }

  /**
   * V4: Seed the current workspace with example data (if empty)
   */
  async seedCurrentWorkspace(): Promise<void> {
    if (!this.currentWorkspaceId) {
      throw new Error('No workspace is currently active');
    }

    const dbFile = this.getWorkspaceDbFile(this.currentWorkspaceId);
    
    // Only seed if database is empty
    if (fs.existsSync(dbFile)) {
      const existing = JSON.parse(fs.readFileSync(dbFile, 'utf-8')) as ProgressDatabase;
      if (existing.tasks.length > 0 || existing.projects.length > 0) {
        throw new Error('Workspace already has data. Cannot seed non-empty workspace.');
      }
    }

    const seeded = this.getSeededDatabase();
    fs.writeFileSync(dbFile, JSON.stringify(seeded, null, 2));
    
    // Reload the database
    this.db = this.load();
    this.initializeTaskMap();
    this.taskHeap = new MinHeap(this.getTaskArray());
    
    console.log(`📁 V4: Seeded workspace ${this.currentWorkspaceId} with example data`);
  }

  /**
   * V4: Switch to a different workspace
   */
  async switchWorkspace(workspaceId: string): Promise<void> {
    const metadata = this.loadWorkspaceMetadata();
    const workspace = metadata.workspaces.find(w => w.id === workspaceId);
    
    if (!workspace) {
      throw new Error(`Workspace ${workspaceId} not found`);
    }
    
    // Save current workspace state before switching
    await this.save();
    
    // Switch workspace
    this.currentWorkspaceId = workspaceId;
    metadata.currentWorkspaceId = workspaceId;
    this.saveWorkspaceMetadata(metadata);
    
    // Reload database for new workspace
    this.db = this.load();
    this.initializeTaskMap();
    this.taskHeap = new MinHeap(this.getTaskArray());
    
    // V4.1: Restart file watcher for new workspace's file
    this.startFileWatcher();
    
    console.log(`📁 V4: Switched to workspace "${workspace.name}" (${workspaceId})`);
  }

  /**
   * V4.3: Update workspace metadata (name and/or description).
   *
   * The workspace ID is immutable; tasks, completion records, and all ML
   * signals remain attached via workspaceId references. This method only
   * mutates display-layer fields. Returns null if the workspace doesn't
   * exist; otherwise returns the updated Workspace record.
   */
  async updateWorkspace(
    workspaceId: string,
    updates: Partial<Pick<Workspace, 'name' | 'description'>>,
  ): Promise<Workspace | null> {
    const metadata = this.loadWorkspaceMetadata();
    const idx = metadata.workspaces.findIndex(w => w.id === workspaceId);
    if (idx === -1) {
      return null;
    }
    const existing = metadata.workspaces[idx];
    const updated: Workspace = {
      ...existing,
      ...(updates.name !== undefined && { name: updates.name }),
      ...(updates.description !== undefined && { description: updates.description }),
      updatedAt: new Date().toISOString(),
    };
    metadata.workspaces[idx] = updated;
    this.saveWorkspaceMetadata(metadata);
    console.log(`📁 V4.3: Updated workspace "${updated.name}" (${workspaceId})`);
    return updated;
  }

  /**
   * V4: Delete a workspace
   */
  async deleteWorkspace(workspaceId: string): Promise<boolean> {
    const metadata = this.loadWorkspaceMetadata();
    const idx = metadata.workspaces.findIndex(w => w.id === workspaceId);
    
    if (idx === -1) {
      return false;
    }
    
    // Don't allow deleting the current workspace
    if (metadata.currentWorkspaceId === workspaceId) {
      throw new Error('Cannot delete the current workspace. Switch to another workspace first.');
    }
    
    // Remove workspace from metadata
    metadata.workspaces.splice(idx, 1);
    this.saveWorkspaceMetadata(metadata);
    
    // Delete workspace directory
    const workspaceDir = path.join(WORKSPACES_DIR, workspaceId);
    if (fs.existsSync(workspaceDir)) {
      fs.rmSync(workspaceDir, { recursive: true, force: true });
    }
    
    console.log(`📁 V4: Deleted workspace ${workspaceId}`);
    return true;
  }
}

export const storage = new JsonStorage();
