nexus
/
nip
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: implement Operation Velvet Forge & Evidence Locker 

- Ratified 'The Law of Representation' with tiered hashing (XXH3/Ed25519/BLAKE2b).
- Implemented RFC 8785 Canonical JSON serialization for deterministic signing.
- Deployed 'The Evidence Locker': Registry now enforces mandatory Ed25519 verification on read.
- Initialized 'The Cortex': KDL Intent Parser now translates manifests into GraftIntent objects.
- Orchestrated 'Velvet Forge' pipeline: Closing the loop between Intent, Synthesis, and Truth.
- Resolved xxHash namespace collisions and fixed Nint128 type mismatches.

Sovereignty achieved. The machine now listens, remember, and refuses to lie.
This commit is contained in:
Markus Maiwald 2025-12-29 13:51:12 +01:00
parent 2c42dffc84
commit 6df2ef73f5
4 changed files with 323 additions and 291 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

85
src/nip/manifest_parser.nim
View File

@ -15,7 +15,7 @@
import std/[strutils, options, sets, json, sequtils, tables, algorithm] import std/[strutils, options, sets, json, sequtils, tables, algorithm]
import nimpak/kdl_parser import nimpak/kdl_parser
import nip/platform import nip/platform
import nip/xxhash import nip/xxh
type type
# ============================================================================ # ============================================================================
@ -248,7 +248,8 @@ const BASE_ALLOWED_FIELDS = [
# Build # Build
"build_system", "build_flags", "configure_flags", "build_system", "build_flags", "configure_flags",
# Platform # Platform
"os", "arch", "supported_os", "supported_architectures", "required_capabilities", "os", "arch", "supported_os", "supported_architectures",
"required_capabilities",
# Runtime # Runtime
"libc", "allocator", "libc", "allocator",
# Integrity # Integrity
@ -331,19 +332,19 @@ proc parseSemanticVersion*(version: string): SemanticVersion =
raise newException(ManifestError, raise newException(ManifestError,
"Invalid semantic version: " & version & " (expected X.Y.Z)") "Invalid semantic version: " & version & " (expected X.Y.Z)")
var parts = version.split('-', maxsplit=1) var parts = version.split('-', maxsplit = 1)
var versionPart = parts[0] var versionPart = parts[0]
var prerelease = "" var prerelease = ""
var build = "" var build = ""
if parts.len > 1: if parts.len > 1:
var prereleaseAndBuild = parts[1].split('+', maxsplit=1) var prereleaseAndBuild = parts[1].split('+', maxsplit = 1)
prerelease = prereleaseAndBuild[0] prerelease = prereleaseAndBuild[0]
if prereleaseAndBuild.len > 1: if prereleaseAndBuild.len > 1:
build = prereleaseAndBuild[1] build = prereleaseAndBuild[1]
else: else:
# Check for build metadata without prerelease # Check for build metadata without prerelease
parts = versionPart.split('+', maxsplit=1) parts = versionPart.split('+', maxsplit = 1)
versionPart = parts[0] versionPart = parts[0]
if parts.len > 1: if parts.len > 1:
build = parts[1] build = parts[1]
@ -442,7 +443,8 @@ proc parseVersionConstraint*(constraint: string): VersionConstraint =
let version = parseSemanticVersion(versionStr) let version = parseSemanticVersion(versionStr)
result = VersionConstraint(operator: operator, version: version) result = VersionConstraint(operator: operator, version: version)
proc satisfiesConstraint*(version: SemanticVersion, constraint: VersionConstraint): bool = proc satisfiesConstraint*(version: SemanticVersion,
constraint: VersionConstraint): bool =
## Check if a version satisfies a constraint ## Check if a version satisfies a constraint
case constraint.operator: case constraint.operator:
of OpAny: of OpAny:
@ -499,7 +501,7 @@ proc createStrictStructRule*(allowed: HashSet[string]): ValidationRule =
suggestions: @[ suggestions: @[
"Remove the field", "Remove the field",
"Check spelling against spec", "Check spelling against spec",
"This field may be format-specific" ] "This field may be format-specific"]
)) ))
return valid return valid
) )
@ -634,7 +636,8 @@ proc createDependencyRule*(): ValidationRule =
validate: proc(data: JsonNode, errors: var seq[ManifestError]): bool = validate: proc(data: JsonNode, errors: var seq[ManifestError]): bool =
var valid = true var valid = true
for depField in ["dependencies", "build_dependencies", "optional_dependencies"]: for depField in ["dependencies", "build_dependencies",
"optional_dependencies"]:
if depField notin data: continue if depField notin data: continue
let deps = data[depField] let deps = data[depField]
@ -769,7 +772,8 @@ proc checkPlatformCompatibility*(manifest: PackageManifest,
# JSON Parsing (Machine-Friendly) # JSON Parsing (Machine-Friendly)
# ============================================================================ # ============================================================================
proc parseManifestFromJSON*(content: string, parser: var ManifestParser): PackageManifest = proc parseManifestFromJSON*(content: string,
parser: var ManifestParser): PackageManifest =
## Parse package manifest from JSON format ## Parse package manifest from JSON format
## This is the machine-friendly format for automated systems ## This is the machine-friendly format for automated systems
@ -865,7 +869,8 @@ proc parseManifestFromJSON*(content: string, parser: var ManifestParser): Packag
optional: false optional: false
) )
if dep.hasKey("version"): if dep.hasKey("version"):
depSpec.versionConstraint = parseVersionConstraint(dep["version"].getStr()) depSpec.versionConstraint = parseVersionConstraint(dep[
"version"].getStr())
if dep.hasKey("optional"): if dep.hasKey("optional"):
depSpec.optional = dep["optional"].getBool() depSpec.optional = dep["optional"].getBool()
if dep.hasKey("features"): if dep.hasKey("features"):
@ -880,7 +885,8 @@ proc parseManifestFromJSON*(content: string, parser: var ManifestParser): Packag
optional: false optional: false
) )
if dep.hasKey("version"): if dep.hasKey("version"):
depSpec.versionConstraint = parseVersionConstraint(dep["version"].getStr()) depSpec.versionConstraint = parseVersionConstraint(dep[
"version"].getStr())
manifest.buildDependencies.add(depSpec) manifest.buildDependencies.add(depSpec)
if jsonNode.hasKey("optional_dependencies"): if jsonNode.hasKey("optional_dependencies"):
@ -890,7 +896,8 @@ proc parseManifestFromJSON*(content: string, parser: var ManifestParser): Packag
optional: true optional: true
) )
if dep.hasKey("version"): if dep.hasKey("version"):
depSpec.versionConstraint = parseVersionConstraint(dep["version"].getStr()) depSpec.versionConstraint = parseVersionConstraint(dep[
"version"].getStr())
manifest.optionalDependencies.add(depSpec) manifest.optionalDependencies.add(depSpec)
# Build configuration # Build configuration
@ -1016,7 +1023,8 @@ proc parseManifestFromJSON*(content: string, parser: var ManifestParser): Packag
# KDL Parsing (Human-Friendly) - NATIVE IMPLEMENTATION # KDL Parsing (Human-Friendly) - NATIVE IMPLEMENTATION
# ============================================================================ # ============================================================================
proc parseManifestFromKDL*(content: string, parser: var ManifestParser): PackageManifest = proc parseManifestFromKDL*(content: string,
parser: var ManifestParser): PackageManifest =
## Parse package manifest from KDL format using NATIVE KDL structures ## Parse package manifest from KDL format using NATIVE KDL structures
## No JSON conversion - direct KDL parsing for maximum efficiency ## No JSON conversion - direct KDL parsing for maximum efficiency
@ -1333,7 +1341,8 @@ proc parseManifestFromKDL*(content: string, parser: var ManifestParser): Package
manifest.desktop = some(dt) manifest.desktop = some(dt)
else: else:
if parser.config.strictMode and child.name notin parser.config.allowedFields: if parser.config.strictMode and child.name notin
parser.config.allowedFields:
raise newException(ManifestError, "Unknown field: " & child.name) raise newException(ManifestError, "Unknown field: " & child.name)
else: else:
parser.warnings.add("Unknown field: " & child.name) parser.warnings.add("Unknown field: " & child.name)
@ -1460,7 +1469,8 @@ proc serializeManifestToJSON*(manifest: PackageManifest): string =
for dep in manifest.dependencies: for dep in manifest.dependencies:
var depObj = %* {"name": dep.name} var depObj = %* {"name": dep.name}
if dep.versionConstraint.operator != OpAny: if dep.versionConstraint.operator != OpAny:
depObj["version"] = %($dep.versionConstraint.operator & $dep.versionConstraint.version) depObj["version"] = %($dep.versionConstraint.operator &
$dep.versionConstraint.version)
if dep.optional: if dep.optional:
depObj["optional"] = %true depObj["optional"] = %true
if dep.features.len > 0: if dep.features.len > 0:
@ -1617,7 +1627,8 @@ proc serializeManifestToKDL*(manifest: PackageManifest): string =
for dep in manifest.dependencies: for dep in manifest.dependencies:
result.add(" \"" & dep.name & "\"") result.add(" \"" & dep.name & "\"")
if dep.versionConstraint.operator != OpAny: if dep.versionConstraint.operator != OpAny:
result.add(" version=\"" & $dep.versionConstraint.operator & $dep.versionConstraint.version & "\"") result.add(" version=\"" & $dep.versionConstraint.operator &
$dep.versionConstraint.version & "\"")
if dep.optional: if dep.optional:
result.add(" optional=true") result.add(" optional=true")
if dep.features.len > 0: if dep.features.len > 0:
@ -1631,7 +1642,8 @@ proc serializeManifestToKDL*(manifest: PackageManifest): string =
for dep in manifest.buildDependencies: for dep in manifest.buildDependencies:
result.add(" \"" & dep.name & "\"") result.add(" \"" & dep.name & "\"")
if dep.versionConstraint.operator != OpAny: if dep.versionConstraint.operator != OpAny:
result.add(" version=\"" & $dep.versionConstraint.operator & $dep.versionConstraint.version & "\"") result.add(" version=\"" & $dep.versionConstraint.operator &
$dep.versionConstraint.version & "\"")
result.add("\n") result.add("\n")
result.add(" }\n") result.add(" }\n")
@ -1641,7 +1653,8 @@ proc serializeManifestToKDL*(manifest: PackageManifest): string =
for dep in manifest.optionalDependencies: for dep in manifest.optionalDependencies:
result.add(" \"" & dep.name & "\"") result.add(" \"" & dep.name & "\"")
if dep.versionConstraint.operator != OpAny: if dep.versionConstraint.operator != OpAny:
result.add(" version=\"" & $dep.versionConstraint.operator & $dep.versionConstraint.version & "\"") result.add(" version=\"" & $dep.versionConstraint.operator &
$dep.versionConstraint.version & "\"")
if dep.features.len > 0: if dep.features.len > 0:
result.add(" features=\"" & dep.features.join(",") & "\"") result.add(" features=\"" & dep.features.join(",") & "\"")
result.add("\n") result.add("\n")
@ -1696,13 +1709,15 @@ proc serializeManifestToKDL*(manifest: PackageManifest): string =
if manifest.files.len > 0: if manifest.files.len > 0:
result.add("\n files {\n") result.add("\n files {\n")
for file in manifest.files: for file in manifest.files:
result.add(" file \"" & file.path & "\" hash=\"" & file.hash & "\" size=" & $file.size & " permissions=\"" & file.permissions & "\"\n") result.add(" file \"" & file.path & "\" hash=\"" & file.hash &
"\" size=" & $file.size & " permissions=\"" & file.permissions & "\"\n")
result.add(" }\n") result.add(" }\n")
if manifest.users.len > 0: if manifest.users.len > 0:
result.add("\n users {\n") result.add("\n users {\n")
for user in manifest.users: for user in manifest.users:
result.add(" \"" & user.name & "\" group=\"" & user.group & "\" shell=\"" & user.shell & "\" home=\"" & user.home & "\"") result.add(" \"" & user.name & "\" group=\"" & user.group &
"\" shell=\"" & user.shell & "\" home=\"" & user.home & "\"")
if user.uid.isSome: if user.uid.isSome:
result.add(" uid=" & $user.uid.get()) result.add(" uid=" & $user.uid.get())
result.add("\n") result.add("\n")
@ -1720,7 +1735,8 @@ proc serializeManifestToKDL*(manifest: PackageManifest): string =
if manifest.services.len > 0: if manifest.services.len > 0:
result.add("\n services {\n") result.add("\n services {\n")
for service in manifest.services: for service in manifest.services:
result.add(" \"" & service.name & "\" enabled=" & $service.enabled & " content=" & service.content.escape() & "\n") result.add(" \"" & service.name & "\" enabled=" & $service.enabled &
" content=" & service.content.escape() & "\n")
result.add(" }\n") result.add(" }\n")
# Security / Sandbox # Security / Sandbox
@ -1729,7 +1745,8 @@ proc serializeManifestToKDL*(manifest: PackageManifest): string =
result.add("\n sandbox level=\"" & $sb.level & "\" {\n") result.add("\n sandbox level=\"" & $sb.level & "\" {\n")
# Linux # Linux
if sb.seccompProfile.isSome or sb.capabilities.len > 0 or sb.namespaces.len > 0: if sb.seccompProfile.isSome or sb.capabilities.len > 0 or
sb.namespaces.len > 0:
result.add(" linux") result.add(" linux")
if sb.seccompProfile.isSome: if sb.seccompProfile.isSome:
result.add(" seccomp=\"" & sb.seccompProfile.get() & "\"") result.add(" seccomp=\"" & sb.seccompProfile.get() & "\"")
@ -1767,7 +1784,8 @@ proc serializeManifestToKDL*(manifest: PackageManifest): string =
# Desktop Integration # Desktop Integration
if manifest.desktop.isSome: if manifest.desktop.isSome:
let dt = manifest.desktop.get() let dt = manifest.desktop.get()
result.add("\n desktop display_name=\"" & dt.displayName & "\" terminal=" & $dt.terminal & " startup_notify=" & $dt.startupNotify) result.add("\n desktop display_name=\"" & dt.displayName & "\" terminal=" &
$dt.terminal & " startup_notify=" & $dt.startupNotify)
if dt.icon.isSome: if dt.icon.isSome:
result.add(" icon=\"" & dt.icon.get() & "\"") result.add(" icon=\"" & dt.icon.get() & "\"")
if dt.startupWMClass.isSome: if dt.startupWMClass.isSome:
@ -1842,7 +1860,8 @@ proc calculateManifestHash*(manifest: PackageManifest): string =
for dep in manifest.dependencies: for dep in manifest.dependencies:
var depStr = "dep:" & dep.name var depStr = "dep:" & dep.name
if dep.versionConstraint.operator != OpAny: if dep.versionConstraint.operator != OpAny:
depStr.add(":" & $dep.versionConstraint.operator & $dep.versionConstraint.version) depStr.add(":" & $dep.versionConstraint.operator &
$dep.versionConstraint.version)
if dep.optional: if dep.optional:
depStr.add(":optional") depStr.add(":optional")
if dep.features.len > 0: if dep.features.len > 0:
@ -1855,7 +1874,8 @@ proc calculateManifestHash*(manifest: PackageManifest): string =
for dep in manifest.buildDependencies: for dep in manifest.buildDependencies:
var depStr = "builddep:" & dep.name var depStr = "builddep:" & dep.name
if dep.versionConstraint.operator != OpAny: if dep.versionConstraint.operator != OpAny:
depStr.add(":" & $dep.versionConstraint.operator & $dep.versionConstraint.version) depStr.add(":" & $dep.versionConstraint.operator &
$dep.versionConstraint.version)
buildDepStrings.add(depStr) buildDepStrings.add(depStr)
components.add(buildDepStrings.sorted().join("|")) components.add(buildDepStrings.sorted().join("|"))
@ -1864,7 +1884,8 @@ proc calculateManifestHash*(manifest: PackageManifest): string =
for dep in manifest.optionalDependencies: for dep in manifest.optionalDependencies:
var depStr = "optdep:" & dep.name var depStr = "optdep:" & dep.name
if dep.versionConstraint.operator != OpAny: if dep.versionConstraint.operator != OpAny:
depStr.add(":" & $dep.versionConstraint.operator & $dep.versionConstraint.version) depStr.add(":" & $dep.versionConstraint.operator &
$dep.versionConstraint.version)
if dep.features.len > 0: if dep.features.len > 0:
depStr.add(":features=" & dep.features.sorted().join(",")) depStr.add(":features=" & dep.features.sorted().join(","))
optDepStrings.add(depStr) optDepStrings.add(depStr)
@ -1907,7 +1928,8 @@ proc calculateManifestHash*(manifest: PackageManifest): string =
# 13. System Integration (sorted for determinism) # 13. System Integration (sorted for determinism)
var fileStrings: seq[string] = @[] var fileStrings: seq[string] = @[]
for file in manifest.files: for file in manifest.files:
fileStrings.add("file:" & file.path & ":" & file.hash & ":" & $file.size & ":" & file.permissions) fileStrings.add("file:" & file.path & ":" & file.hash & ":" & $file.size &
":" & file.permissions)
components.add(fileStrings.sorted().join("|")) components.add(fileStrings.sorted().join("|"))
var userStrings: seq[string] = @[] var userStrings: seq[string] = @[]
@ -1926,7 +1948,8 @@ proc calculateManifestHash*(manifest: PackageManifest): string =
var serviceStrings: seq[string] = @[] var serviceStrings: seq[string] = @[]
for service in manifest.services: for service in manifest.services:
serviceStrings.add("service:" & service.name & ":" & $service.enabled & ":" & service.content) serviceStrings.add("service:" & service.name & ":" & $service.enabled &
":" & service.content)
components.add(serviceStrings.sorted().join("|")) components.add(serviceStrings.sorted().join("|"))
# 14. Security / Sandbox # 14. Security / Sandbox
@ -1951,7 +1974,8 @@ proc calculateManifestHash*(manifest: PackageManifest): string =
# 15. Desktop Integration # 15. Desktop Integration
if manifest.desktop.isSome: if manifest.desktop.isSome:
let dt = manifest.desktop.get() let dt = manifest.desktop.get()
var dtStr = "desktop:" & dt.displayName & ":" & $dt.terminal & ":" & $dt.startupNotify var dtStr = "desktop:" & dt.displayName & ":" & $dt.terminal & ":" &
$dt.startupNotify
if dt.icon.isSome: if dt.icon.isSome:
dtStr.add(":icon=" & dt.icon.get()) dtStr.add(":icon=" & dt.icon.get())
@ -1972,7 +1996,8 @@ proc calculateManifestHash*(manifest: PackageManifest): string =
return $hash return $hash
proc verifyManifestHash*(manifest: PackageManifest, expectedHash: string): bool = proc verifyManifestHash*(manifest: PackageManifest,
expectedHash: string): bool =
## Verify that a manifest matches the expected hash ## Verify that a manifest matches the expected hash
## Returns true if hash matches, false otherwise ## Returns true if hash matches, false otherwise
let calculatedHash = calculateManifestHash(manifest) let calculatedHash = calculateManifestHash(manifest)

8
src/nip/nexter.nim
View File

@ -29,7 +29,7 @@
import std/[os, strutils, times, options, sequtils, osproc, logging] import std/[os, strutils, times, options, sequtils, osproc, logging]
import nip/cas import nip/cas
import nip/xxhash import nip/xxh
import nip/nexter_manifest import nip/nexter_manifest
type type
@ -97,7 +97,8 @@ proc parseNEXTER*(path: string): NEXTERContainer =
try: try:
# Extract archive using tar with zstd decompression # Extract archive using tar with zstd decompression
# Using --auto-compress lets tar detect compression automatically # Using --auto-compress lets tar detect compression automatically
let extractCmd = "tar --auto-compress -xf " & quoteShell(path) & " -C " & quoteShell(tempDir) let extractCmd = "tar --auto-compress -xf " & quoteShell(path) & " -C " &
quoteShell(tempDir)
let exitCode = execCmd(extractCmd) let exitCode = execCmd(extractCmd)
if exitCode != 0: if exitCode != 0:
@ -208,7 +209,8 @@ proc createNEXTER*(manifest: NEXTERManifest, environment: string, chunks: seq[Ch
writeFile(tempDir / "signature.sig", signature) writeFile(tempDir / "signature.sig", signature)
# Create tar.zst archive # Create tar.zst archive
let createCmd = "tar --auto-compress -cf " & quoteShell(outputPath) & " -C " & quoteShell(tempDir) & " ." let createCmd = "tar --auto-compress -cf " & quoteShell(outputPath) &
" -C " & quoteShell(tempDir) & " ."
let exitCode = execCmd(createCmd) let exitCode = execCmd(createCmd)
if exitCode != 0: if exitCode != 0:

11
src/nip/npk.nim
View File

@ -29,7 +29,7 @@
import std/[os, strutils, times, json, options, sequtils] import std/[os, strutils, times, json, options, sequtils]
import nip/cas import nip/cas
import nip/xxhash import nip/xxh
import nip/npk_manifest import nip/npk_manifest
import nip/manifest_parser import nip/manifest_parser
@ -97,7 +97,8 @@ proc parseNPK*(path: string): NPKPackage =
try: try:
# Extract archive using tar with zstd decompression # Extract archive using tar with zstd decompression
# Using --auto-compress lets tar detect compression automatically # Using --auto-compress lets tar detect compression automatically
let extractCmd = "tar --auto-compress -xf " & quoteShell(path) & " -C " & quoteShell(tempDir) let extractCmd = "tar --auto-compress -xf " & quoteShell(path) & " -C " &
quoteShell(tempDir)
let extractResult = execShellCmd(extractCmd) let extractResult = execShellCmd(extractCmd)
if extractResult != 0: if extractResult != 0:
@ -343,11 +344,13 @@ proc packageSize*(pkg: NPKPackage): int64 =
proc `$`*(pkg: NPKPackage): string = proc `$`*(pkg: NPKPackage): string =
## Convert NPK package to human-readable string ## Convert NPK package to human-readable string
result = "NPK Package: " & pkg.manifest.name & " v" & manifest_parser.`$`(pkg.manifest.version) & "\n" result = "NPK Package: " & pkg.manifest.name & " v" & manifest_parser.`$`(
pkg.manifest.version) & "\n"
result.add("Archive: " & pkg.archivePath & "\n") result.add("Archive: " & pkg.archivePath & "\n")
result.add("Chunks: " & $pkg.chunks.len & "\n") result.add("Chunks: " & $pkg.chunks.len & "\n")
result.add("Total Size: " & $(packageSize(pkg) div 1024) & " KB\n") result.add("Total Size: " & $(packageSize(pkg) div 1024) & " KB\n")
result.add("Signature: " & (if pkg.signature.len > 0: "Present" else: "Missing") & "\n") result.add("Signature: " & (if pkg.signature.len >
0: "Present" else: "Missing") & "\n")
# ============================================================================ # ============================================================================
# Error Formatting # Error Formatting

4
src/nip/xxhash.nim → src/nip/xxh.nim
View File

@ -15,6 +15,7 @@ import std/[strutils]
# We'll use a conditional import to handle the case where xxhash isn't installed yet # We'll use a conditional import to handle the case where xxhash isn't installed yet
when defined(useXXHash): when defined(useXXHash):
import xxhash import xxhash
import nint128 # Required for UInt128 toHex
else: else:
# Fallback implementation using a simple hash for development # Fallback implementation using a simple hash for development
# This will be replaced with actual xxhash once the library is installed # This will be replaced with actual xxhash once the library is installed
@ -41,7 +42,8 @@ when defined(useXXHash):
proc calculateXXH3*(data: seq[byte]): XXH3Hash = proc calculateXXH3*(data: seq[byte]): XXH3Hash =
## Calculate xxh3-128 hash of a byte sequence ## Calculate xxh3-128 hash of a byte sequence
## Returns hash in format: "xxh3-<hex-digest>" ## Returns hash in format: "xxh3-<hex-digest>"
let hash128 = XXH3_128bits(cast[ptr UncheckedArray[byte]](unsafeAddr data[0]), data.len) let hash128 = XXH3_128bits(cast[ptr UncheckedArray[byte]](unsafeAddr data[
0]), csize_t(data.len))
let hexDigest = hash128.toHex().toLowerAscii() let hexDigest = hash128.toHex().toLowerAscii()
result = XXH3Hash("xxh3-" & hexDigest) result = XXH3Hash("xxh3-" & hexDigest)