## nimpak/cache/statistics.nim
## Cache performance statistics and metrics for NimPak
##
## This module provides comprehensive cache analytics including:
## - Hit/miss rate tracking
## - Performance metrics and trends
## - Storage utilization analysis
## - Binary selection effectiveness
## - Cache health monitoring

import std/[times, json, tables, sequtils, strformat, algorithm, math]
import ../cli/core

type
  CacheMetricType* = enum
    MetricHitRate = "hit_rate"
    MetricMissRate = "miss_rate"
    MetricEvictionRate = "eviction_rate"
    MetricStorageUtilization = "storage_utilization"
    MetricAverageAccessTime = "average_access_time"
    MetricCorruptionRate = "corruption_rate"

  TimeWindow* = enum
    WindowHour = "hour"
    WindowDay = "day"
    WindowWeek = "week"
    WindowMonth = "month"

  CacheEvent* = object
    timestamp*: times.DateTime
    eventType*: string         # "hit", "miss", "store", "evict", "corrupt"
    cacheKey*: string
    packageName*: string
    size*: int64
    accessTime*: float         # Time taken for operation
    metadata*: JsonNode

  MetricSnapshot* = object
    timestamp*: times.DateTime
    hitCount*: int64
    missCount*: int64
    evictionCount*: int64
    corruptionCount*: int64
    totalSize*: int64
    totalEntries*: int64
    averageAccessTime*: float
    hitRate*: float
    storageUtilization*: float

  TrendAnalysis* = object
    metric*: CacheMetricType
    window*: TimeWindow
    trend*: string             # "increasing", "decreasing", "stable"
    changeRate*: float         # Rate of change per time unit
    confidence*: float         # Confidence in trend analysis (0.0-1.0)
    dataPoints*: int

  CacheHealthStatus* = enum
    HealthExcellent = "excellent"
    HealthGood = "good"
    HealthFair = "fair"
    HealthPoor = "poor"
    HealthCritical = "critical"

  CacheHealth* = object
    status*: CacheHealthStatus
    score*: float              # Overall health score (0.0-1.0)
    issues*: seq[string]
    recommendations*: seq[string]
    metrics*: Table[CacheMetricType, float]

  CacheStatisticsManager* = object
    events*: seq[CacheEvent]
    snapshots*: seq[MetricSnapshot]
    maxEvents*: int            # Maximum events to keep in memory
    maxSnapshots*: int         # Maximum snapshots to keep
    snapshotInterval*: int     # Seconds between snapshots
    lastSnapshot*: times.DateTime

# =============================================================================
# Statistics Manager Initialization
# =============================================================================

proc newCacheStatisticsManager*(maxEvents: int = 10000, maxSnapshots: int = 1000,
                               snapshotInterval: int = 300): CacheStatisticsManager =
  ## Create a new cache statistics manager
  CacheStatisticsManager(
    events: @[],
    snapshots: @[],
    maxEvents: maxEvents,
    maxSnapshots: maxSnapshots,
    snapshotInterval: snapshotInterval,
    lastSnapshot: default(times.DateTime)
  )

# =============================================================================
# Event Recording
# =============================================================================

proc recordCacheEvent*(manager: var CacheStatisticsManager, eventType: string,
                      cacheKey: string, packageName: string = "", size: int64 = 0,
                      accessTime: float = 0.0, metadata: JsonNode = newJNull()) =
  ## Record a cache event
  let event = CacheEvent(
    timestamp: now(),
    eventType: eventType,
    cacheKey: cacheKey,
    packageName: packageName,
    size: size,
    accessTime: accessTime,
    metadata: metadata
  )

  manager.events.add(event)

  # Trim events if we exceed the limit
  if manager.events.len > manager.maxEvents:
    let excessEvents = manager.events.len - manager.maxEvents
    manager.events = manager.events[excessEvents..^1]

proc recordCacheHit*(manager: var CacheStatisticsManager, cacheKey: string,
                    packageName: string, accessTime: float) =
  ## Record a cache hit
  manager.recordCacheEvent("hit", cacheKey, packageName, 0, accessTime)

proc recordCacheMiss*(manager: var CacheStatisticsManager, cacheKey: string,
                     packageName: string, reason: string = "") =
  ## Record a cache miss
  let metadata = if reason != "": %*{"reason": reason} else: newJNull()
  manager.recordCacheEvent("miss", cacheKey, packageName, 0, 0.0, metadata)

proc recordCacheStore*(manager: var CacheStatisticsManager, cacheKey: string,
                      packageName: string, size: int64, storeTime: float) =
  ## Record a cache store operation
  manager.recordCacheEvent("store", cacheKey, packageName, size, storeTime)

proc recordCacheEviction*(manager: var CacheStatisticsManager, cacheKey: string,
                         packageName: string, size: int64, reason: string = "") =
  ## Record a cache eviction
  let metadata = if reason != "": %*{"reason": reason} else: newJNull()
  manager.recordCacheEvent("evict", cacheKey, packageName, size, 0.0, metadata)

proc recordCacheCorruption*(manager: var CacheStatisticsManager, cacheKey: string,
                           packageName: string, reason: string = "") =
  ## Record a cache corruption
  let metadata = if reason != "": %*{"reason": reason} else: newJNull()
  manager.recordCacheEvent("corrupt", cacheKey, packageName, 0, 0.0, metadata)

# =============================================================================
# Metric Calculation
# =============================================================================

proc calculateMetrics*(manager: CacheStatisticsManager, window: TimeWindow): MetricSnapshot =
  ## Calculate metrics for a specific time window
  let now = times.now()
  let windowStart = case window:
    of WindowHour: now - initDuration(hours = 1)
    of WindowDay: now - initDuration(days = 1)
    of WindowWeek: now - initDuration(weeks = 1)
    of WindowMonth: now - initDuration(days = 30)

  # Filter events within the time window
  let windowEvents = manager.events.filterIt(it.timestamp >= windowStart)

  var hitCount: int64 = 0
  var missCount: int64 = 0
  var evictionCount: int64 = 0
  var corruptionCount: int64 = 0
  var totalSize: int64 = 0
  var totalEntries: int64 = 0
  var totalAccessTime: float = 0.0
  var accessTimeCount: int = 0

  for event in windowEvents:
    case event.eventType:
    of "hit":
      hitCount += 1
      totalAccessTime += event.accessTime
      accessTimeCount += 1
    of "miss":
      missCount += 1
    of "store":
      totalSize += event.size
      totalEntries += 1
    of "evict":
      evictionCount += 1
      totalSize -= event.size
      totalEntries -= 1
    of "corrupt":
      corruptionCount += 1

  let totalRequests = hitCount + missCount
  let hitRate = if totalRequests > 0: hitCount.float / totalRequests.float else: 0.0
  let averageAccessTime = if accessTimeCount > 0: totalAccessTime / accessTimeCount.float else: 0.0

  # Calculate storage utilization (assuming max size from config)
  let maxSize: int64 = 50 * 1024 * 1024 * 1024  # 50GB default
  let storageUtilization = if maxSize > 0: totalSize.float / maxSize.float else: 0.0

  MetricSnapshot(
    timestamp: now,
    hitCount: hitCount,
    missCount: missCount,
    evictionCount: evictionCount,
    corruptionCount: corruptionCount,
    totalSize: totalSize,
    totalEntries: totalEntries,
    averageAccessTime: averageAccessTime,
    hitRate: hitRate,
    storageUtilization: storageUtilization
  )

proc takeSnapshot*(manager: var CacheStatisticsManager): MetricSnapshot =
  ## Take a snapshot of current metrics
  let snapshot = manager.calculateMetrics(WindowHour)
  manager.snapshots.add(snapshot)
  manager.lastSnapshot = now()

  # Trim snapshots if we exceed the limit
  if manager.snapshots.len > manager.maxSnapshots:
    let excessSnapshots = manager.snapshots.len - manager.maxSnapshots
    manager.snapshots = manager.snapshots[excessSnapshots..^1]

  return snapshot

proc shouldTakeSnapshot*(manager: CacheStatisticsManager): bool =
  ## Check if it's time to take a snapshot
  if manager.lastSnapshot == default(times.DateTime):
    return true

  let timeSinceLastSnapshot = (now() - manager.lastSnapshot).inSeconds
  return timeSinceLastSnapshot >= manager.snapshotInterval

# =============================================================================
# Trend Analysis
# =============================================================================

proc analyzeTrend*(manager: CacheStatisticsManager, metric: CacheMetricType,
                  window: TimeWindow): TrendAnalysis =
  ## Analyze trend for a specific metric
  let windowDuration = case window:
    of WindowHour: 3600
    of WindowDay: 86400
    of WindowWeek: 604800
    of WindowMonth: 2592000

  # Get snapshots within the window
  let cutoffTime = now() - initDuration(seconds = windowDuration)
  let windowSnapshots = manager.snapshots.filterIt(it.timestamp >= cutoffTime)

  if windowSnapshots.len < 2:
    return TrendAnalysis(
      metric: metric,
      window: window,
      trend: "insufficient_data",
      changeRate: 0.0,
      confidence: 0.0,
      dataPoints: windowSnapshots.len
    )

  # Extract metric values
  var values: seq[float] = @[]
  for snapshot in windowSnapshots:
    let value = case metric:
      of MetricHitRate: snapshot.hitRate
      of MetricMissRate: 1.0 - snapshot.hitRate
      of MetricEvictionRate: snapshot.evictionCount.float / max(1.0, snapshot.totalEntries.float)
      of MetricStorageUtilization: snapshot.storageUtilization
      of MetricAverageAccessTime: snapshot.averageAccessTime
      of MetricCorruptionRate: snapshot.corruptionCount.float / max(1.0, snapshot.totalEntries.float)
    values.add(value)

  # Calculate linear regression for trend
  let n = values.len.float
  let xValues = toSeq(0..<values.len).mapIt(it.float)
  let xMean = xValues.sum() / n
  let yMean = values.sum() / n

  var numerator = 0.0
  var denominator = 0.0
  for i in 0..<values.len:
    let xDiff = xValues[i] - xMean
    let yDiff = values[i] - yMean
    numerator += xDiff * yDiff
    denominator += xDiff * xDiff

  let slope = if denominator != 0.0: numerator / denominator else: 0.0
  let changeRate = slope * (windowDuration.float / values.len.float)  # Change per time unit

  # Determine trend direction
  let trend = if abs(slope) < 0.001:
    "stable"
  elif slope > 0:
    "increasing"
  else:
    "decreasing"

  # Calculate confidence based on R-squared
  var ssRes = 0.0
  var ssTot = 0.0
  for i in 0..<values.len:
    let predicted = yMean + slope * (xValues[i] - xMean)
    ssRes += pow(values[i] - predicted, 2)
    ssTot += pow(values[i] - yMean, 2)

  let rSquared = if ssTot != 0.0: 1.0 - (ssRes / ssTot) else: 0.0
  let confidence = max(0.0, min(1.0, rSquared))

  TrendAnalysis(
    metric: metric,
    window: window,
    trend: trend,
    changeRate: changeRate,
    confidence: confidence,
    dataPoints: values.len
  )

# =============================================================================
# Cache Health Assessment
# =============================================================================

proc assessCacheHealth*(manager: CacheStatisticsManager): CacheHealth =
  ## Assess overall cache health
  let recentMetrics = manager.calculateMetrics(WindowDay)
  var health = CacheHealth(
    status: HealthGood,
    score: 0.8,
    issues: @[],
    recommendations: @[],
    metrics: initTable[CacheMetricType, float]()
  )

  # Store current metrics
  health.metrics[MetricHitRate] = recentMetrics.hitRate
  health.metrics[MetricStorageUtilization] = recentMetrics.storageUtilization
  health.metrics[MetricAverageAccessTime] = recentMetrics.averageAccessTime
  health.metrics[MetricCorruptionRate] = if recentMetrics.totalEntries > 0:
    recentMetrics.corruptionCount.float / recentMetrics.totalEntries.float
  else:
    0.0

  var score = 1.0

  # Assess hit rate
  if recentMetrics.hitRate < 0.3:
    health.issues.add("Very low cache hit rate")
    health.recommendations.add("Review cache size limits and eviction policies")
    score -= 0.4
  elif recentMetrics.hitRate < 0.5:
    health.issues.add("Low cache hit rate")
    health.recommendations.add("Consider increasing cache size")
    score -= 0.2

  # Assess storage utilization
  if recentMetrics.storageUtilization > 0.95:
    health.issues.add("Cache storage nearly full")
    health.recommendations.add("Increase cache size or adjust eviction policy")
    score -= 0.2
  elif recentMetrics.storageUtilization > 0.85:
    health.issues.add("High storage utilization")
    health.recommendations.add("Monitor cache growth")
    score -= 0.1

  # Assess access time
  if recentMetrics.averageAccessTime > 1.0:
    health.issues.add("Slow cache access times")
    health.recommendations.add("Check disk I/O performance")
    score -= 0.2
  elif recentMetrics.averageAccessTime > 0.5:
    health.issues.add("Moderate cache access times")
    score -= 0.1

  # Assess corruption rate
  let corruptionRate = health.metrics[MetricCorruptionRate]
  if corruptionRate > 0.05:
    health.issues.add("High cache corruption rate")
    health.recommendations.add("Check disk health and enable verification")
    score -= 0.3
  elif corruptionRate > 0.01:
    health.issues.add("Moderate cache corruption")
    health.recommendations.add("Enable cache verification")
    score -= 0.1

  # Assess trends
  let hitRateTrend = manager.analyzeTrend(MetricHitRate, WindowWeek)
  if hitRateTrend.trend == "decreasing" and hitRateTrend.confidence > 0.7:
    health.issues.add("Declining hit rate trend")
    health.recommendations.add("Investigate cause of hit rate decline")
    score -= 0.1

  # Determine overall status
  health.score = max(0.0, score)
  health.status = if health.score >= 0.9:
    HealthExcellent
  elif health.score >= 0.7:
    HealthGood
  elif health.score >= 0.5:
    HealthFair
  elif health.score >= 0.3:
    HealthPoor
  else:
    HealthCritical

  return health

# =============================================================================
# Reporting and Visualization
# =============================================================================

proc generateCacheReport*(manager: CacheStatisticsManager): JsonNode =
  ## Generate comprehensive cache statistics report
  let currentMetrics = manager.calculateMetrics(WindowDay)
  let health = manager.assessCacheHealth()

  var trends = newJObject()
  for metric in [MetricHitRate, MetricStorageUtilization, MetricAverageAccessTime, MetricCorruptionRate]:
    let trend = manager.analyzeTrend(metric, WindowWeek)
    trends[$metric] = %*{
      "trend": trend.trend,
      "change_rate": trend.changeRate,
      "confidence": trend.confidence,
      "data_points": trend.dataPoints
    }

  return %*{
    "report_timestamp": $now(),
    "current_metrics": {
      "hit_rate": currentMetrics.hitRate,
      "miss_rate": 1.0 - currentMetrics.hitRate,
      "hit_count": currentMetrics.hitCount,
      "miss_count": currentMetrics.missCount,
      "eviction_count": currentMetrics.evictionCount,
      "corruption_count": currentMetrics.corruptionCount,
      "total_entries": currentMetrics.totalEntries,
      "total_size": currentMetrics.totalSize,
      "total_size_formatted": formatFileSize(currentMetrics.totalSize),
      "storage_utilization": currentMetrics.storageUtilization,
      "average_access_time": currentMetrics.averageAccessTime
    },
    "health_assessment": {
      "status": $health.status,
      "score": health.score,
      "issues": health.issues,
      "recommendations": health.recommendations
    },
    "trends": trends,
    "statistics": {
      "total_events": manager.events.len,
      "total_snapshots": manager.snapshots.len,
      "oldest_event": if manager.events.len > 0: $manager.events[0].timestamp else: null,
      "newest_event": if manager.events.len > 0: $manager.events[^1].timestamp else: null
    }
  }

proc formatHealthReport*(health: CacheHealth): string =
  ## Format cache health as human-readable report
  var report = fmt"Cache Health: {health.status} (Score: {health.score:.2f})\n"

  if health.issues.len > 0:
    report.add("\nIssues:\n")
    for issue in health.issues:
      report.add(fmt"  ⚠️  {issue}\n")

  if health.recommendations.len > 0:
    report.add("\nRecommendations:\n")
    for rec in health.recommendations:
      report.add(fmt"  💡 {rec}\n")

  report.add("\nKey Metrics:\n")
  for metric, value in health.metrics.pairs:
    let formattedValue = case metric:
      of MetricHitRate, MetricStorageUtilization, MetricCorruptionRate:
        fmt"{value * 100:.1f}%"
      of MetricAverageAccessTime:
        fmt"{value:.3f}s"
      else:
        fmt"{value:.3f}"

    report.add(fmt"  {metric}: {formattedValue}\n")

  return report

# =============================================================================
# Performance Optimization Suggestions
# =============================================================================

proc generateOptimizationSuggestions*(manager: CacheStatisticsManager): seq[string] =
  ## Generate cache optimization suggestions based on statistics
  var suggestions: seq[string] = @[]
  let metrics = manager.calculateMetrics(WindowWeek)
  let health = manager.assessCacheHealth()

  # Hit rate optimizations
  if metrics.hitRate < 0.6:
    suggestions.add("Increase cache size to improve hit rate")
    suggestions.add("Review eviction policy - consider LRU if using size-based eviction")
    suggestions.add("Analyze access patterns to identify frequently used packages")

  # Storage optimizations
  if metrics.storageUtilization > 0.8:
    suggestions.add("Enable compression to reduce storage usage")
    suggestions.add("Implement more aggressive eviction policies")
    suggestions.add("Consider increasing cache storage capacity")

  # Performance optimizations
  if metrics.averageAccessTime > 0.3:
    suggestions.add("Consider using faster storage (SSD) for cache")
    suggestions.add("Optimize cache directory structure")
    suggestions.add("Enable parallel cache operations")

  # Reliability optimizations
  let corruptionRate = health.metrics.getOrDefault(MetricCorruptionRate, 0.0)
  if corruptionRate > 0.01:
    suggestions.add("Enable cache verification on access")
    suggestions.add("Check disk health and file system integrity")
    suggestions.add("Implement redundant cache storage")

  # Trend-based optimizations
  let hitRateTrend = manager.analyzeTrend(MetricHitRate, WindowWeek)
  if hitRateTrend.trend == "decreasing":
    suggestions.add("Investigate cause of declining hit rate")
    suggestions.add("Review recent changes to cache configuration")

  return suggestions

# =============================================================================
# Export main functions
# =============================================================================

export CacheMetricType, TimeWindow, CacheEvent, MetricSnapshot
export TrendAnalysis, CacheHealthStatus, CacheHealth, CacheStatisticsManager
export newCacheStatisticsManager
export recordCacheEvent, recordCacheHit, recordCacheMiss
export recordCacheStore, recordCacheEviction, recordCacheCorruption
export calculateMetrics, takeSnapshot, shouldTakeSnapshot
export analyzeTrend, assessCacheHealth
export generateCacheReport, formatHealthReport, generateOptimizationSuggestions