Replace per-save-point array snapshots with an undo trail

Sources/_StringProcessing/Engine/Backtracking.swift

@@ -10,6 +10,31 @@
 //===----------------------------------------------------------------------===//
 
 extension Processor {
+  /// A log entry recording the previous value of a stored capture
+  /// register slot.
+  struct CaptureLogEntry {
+    var slot: CaptureRegister
+    var oldValue: _StoredCapture
+  }
+
+  /// A log entry recording the previous value of an int register slot.
+  struct IntLogEntry {
+    var slot: IntRegister
+    var oldValue: Int
+  }
+
+  /// A log entry recording the previous value of a position register slot.
+  struct PositionLogEntry {
+    var slot: PositionRegister
+    var oldValue: Input.Index
+  }
+
+  /// A log entry recording the previous value of a value register slot.
+  struct ValueLogEntry {
+    var slot: ValueRegister
+    var oldValue: Any
+  }
+
   struct SavePoint {
     var pc: InstructionAddress
     var pos: Position?
@@ -21,25 +46,14 @@ extension Processor {
     //        points. We should try to separate out the concerns better.
     var isScalarSemantics: Bool
 
-    // FIXME: Save minimal info (e.g. stack position and
-    // perhaps current start)
-    var captureEnds: [_StoredCapture]
-
-    // The int registers store values that can be relevant to
-    // backtracking, such as the number of trips in a quantification.
-    var intRegisters: [Int]
-    // Same with position registers
-    var posRegisters: [Input.Index]
-
-    var destructure: (
-      pc: InstructionAddress,
-      pos: Position?,
-      captureEnds: [_StoredCapture],
-      intRegisters: [Int],
-      PositionRegister: [Input.Index]
-    ) {
-      return (pc, pos, captureEnds, intRegisters, posRegisters)
-    }
+    // Indexes into `Processor`'s int/position/etc logs: the log
+    // lengths at the moment this save point was created. On backtrack,
+    // each log is unwound (replaying old values in reverse) down to
+    // the index saved here.
+    var captureLogEnd: Int
+    var intLogEnd: Int
+    var positionLogEnd: Int
+    var valueLogEnd: Int
 
     // Whether this save point is quantified, meaning it has a range of
     // possible positions to explore.
@@ -77,9 +91,10 @@ extension Processor {
       pos: currentPosition,
       quantifiedRange: nil,
       isScalarSemantics: false,
-      captureEnds: storedCaptures,
-      intRegisters: registers.ints,
-      posRegisters: registers.positions)
+      captureLogEnd: captureLog.count,
+      intLogEnd: intLog.count,
+      positionLogEnd: positionLog.count,
+      valueLogEnd: valueLog.count)
   }
 
   func makeAddressOnlySavePoint(
@@ -90,9 +105,10 @@ extension Processor {
       pos: nil,
       quantifiedRange: nil,
       isScalarSemantics: false,
-      captureEnds: storedCaptures,
-      intRegisters: registers.ints,
-      posRegisters: registers.positions)
+      captureLogEnd: captureLog.count,
+      intLogEnd: intLog.count,
+      positionLogEnd: positionLog.count,
+      valueLogEnd: valueLog.count)
   }
 
   func makeQuantifiedSavePoint(
@@ -104,9 +120,10 @@ extension Processor {
       pos: nil,
       quantifiedRange: range,
       isScalarSemantics: isScalarSemantics,
-      captureEnds: storedCaptures,
-      intRegisters: registers.ints,
-      posRegisters: registers.positions)
+      captureLogEnd: captureLog.count,
+      intLogEnd: intLog.count,
+      positionLogEnd: positionLog.count,
+      valueLogEnd: valueLog.count)
   }
 }
 

Sources/_StringProcessing/Engine/MEBuilder.swift

@@ -428,12 +428,10 @@ extension MEProgram.Builder {
       matcherFunctions: matcherFunctions,
       numInts: nextIntRegister.rawValue,
       numValues: nextValueRegister.rawValue,
-      numPositions: nextPositionRegister.rawValue
+      numPositions: nextPositionRegister.rawValue,
+      numCaptures: nextCaptureRegister.rawValue
     )
 
-    let storedCaps = Array(
-      repeating: Processor._StoredCapture(), count: nextCaptureRegister.rawValue)
-
     let meProgram = MEProgram(
       instructions: InstructionList(instructions),
       wholeMatchValueRegister: wholeMatchValue,
@@ -443,8 +441,7 @@ extension MEProgram.Builder {
       referencedCaptureOffsets: referencedCaptureOffsets,
       initialOptions: initialOptions,
       canOnlyMatchAtStart: canOnlyMatchAtStart,
-      registers: regs,
-      storedCaptures: storedCaps)
+      registers: regs)
     return meProgram
   }
 

Sources/_StringProcessing/Engine/MECapture.swift

@@ -11,25 +11,6 @@
 
 internal import _RegexParser
 
-/*
-
- TODO: Specialized data structure for all captures:
-
- - We want to be able to refer to COW prefixes for which
-   simple appends do not invalidate
- - We want a compact save-point representation
-
- TODO: Conjectures:
-
- - We should be able to remove the entire capture history,
-   lazily recomputing it on-request from the initial stored
-   save point
- - We should be able to keep these flat and simple, lazily
-   constructing structured types on-request
-
- */
-
-
 extension Processor {
   struct _StoredCapture {
     var range: Range<Position>? = nil

Sources/_StringProcessing/Engine/MEProgram.swift

@@ -36,8 +36,6 @@ struct MEProgram {
   // processors can be spun up quicker (useful for running same regex
   // over many, many smaller inputs).
   var registers: Processor.Registers
-  var storedCaptures: [Processor._StoredCapture]
-
 }
 
 extension MEProgram: CustomStringConvertible {

Sources/_StringProcessing/Engine/Processor.swift

@@ -82,7 +82,12 @@ struct Processor {
 
   var savePoints: [SavePoint] = []
 
-  var storedCaptures: Array<_StoredCapture>
+  // Logs recording the previous value of any register slot
+  // mutated since the most recent save point.
+  var captureLog: [CaptureLogEntry] = []
+  var intLog: [IntLogEntry] = []
+  var positionLog: [PositionLogEntry] = []
+  var valueLog: [ValueLogEntry] = []
 
   var state: State = .inProgress
 
@@ -125,8 +130,6 @@ extension Processor {
     // Initialize registers from stored starting state
     self.registers = program.registers
 
-    self.storedCaptures = program.storedCaptures
-
     _checkInvariants()
   }
 
@@ -144,11 +147,19 @@ extension Processor {
     if !self.savePoints.isEmpty {
       self.savePoints.removeAll(keepingCapacity: true)
     }
-
-    for idx in storedCaptures.indices {
-      storedCaptures[idx] = .init()
+    if !self.captureLog.isEmpty {
+      self.captureLog.removeAll(keepingCapacity: true)
     }
-
+    if !self.intLog.isEmpty {
+      self.intLog.removeAll(keepingCapacity: true)
+    }
+    if !self.positionLog.isEmpty {
+      self.positionLog.removeAll(keepingCapacity: true)
+    }
+    if !self.valueLog.isEmpty {
+      self.valueLog.removeAll(keepingCapacity: true)
+    }
+
     self.state = .inProgress
     self.failureReason = nil
 
@@ -162,7 +173,10 @@ extension Processor {
     _checkInvariants()
     guard self.controller == Controller(pc: 0),
           self.savePoints.isEmpty,
-          self.storedCaptures.allSatisfy({ $0.range == nil }),
+          self.captureLog.isEmpty,
+          self.intLog.isEmpty,
+          self.positionLog.isEmpty,
+          self.registers.storedCaptures.allSatisfy({ $0.range == nil }),
           self.state == .inProgress,
           self.failureReason == nil
     else {
@@ -377,40 +391,66 @@ extension Processor {
       state = .fail
       return
     }
-    let (pc, pos, capEnds, intRegisters, posRegisters): (
-      pc: InstructionAddress,
-      pos: Position?,
-      captureEnds: [_StoredCapture],
-      intRegisters: [Int],
-      PositionRegister: [Input.Index]
-    )
 
     let idx = savePoints.index(before: savePoints.endIndex)
 
     // If we have a quantifier save point, move the next range position into
     // pos instead of removing it
+    let sp: SavePoint
     if savePoints[idx].isQuantified {
       savePoints[idx].takePositionFromQuantifiedRange(input)
-      (pc, pos, capEnds, intRegisters, posRegisters) = savePoints[idx].destructure
+      sp = savePoints[idx]
     } else {
-      (pc, pos, capEnds, intRegisters, posRegisters) = savePoints.removeLast().destructure
+      sp = savePoints.removeLast()
     }
 
-    assert(capEnds.count == storedCaptures.count)
+    controller.pc = sp.pc
+    currentPosition = sp.pos ?? currentPosition
 
-    controller.pc = pc
-    currentPosition = pos ?? currentPosition
-    registers.ints = intRegisters
-    registers.positions = posRegisters
+    while intLog.count > sp.intLogEnd {
+      registers.restore(intLog.removeLast())
+    }
+    while positionLog.count > sp.positionLogEnd {
+      registers.restore(positionLog.removeLast())
+    }
+    while valueLog.count > sp.valueLogEnd {
+      registers.restore(valueLog.removeLast())
+    }
 
     if !preservingCaptures {
-      // Reset all capture information
-      storedCaptures = capEnds
+      while captureLog.count > sp.captureLogEnd {
+        registers.restore(captureLog.removeLast())
+      }
     }
+    // If preserving captures, leave the capture log entries recorded since
+    // this save point untouched (rather than replaying or discarding them):
+    // `storedCaptures` keeps the values from the successful sub-match, and
+    // the log entries remain available so that an older, still-live save
+    // point can still correctly undo them on its own future backtrack.
 
     metrics.addBacktrack()
   }
 
+  // MARK: Capture mutation
+
+  mutating func setCapture(_ capNum: Int, startingAt pos: Position) {
+    updateRegister(at: CaptureRegister(capNum)) {
+      $0.startCapture(pos)
+    }
+  }
+
+  mutating func setCapture(_ capNum: Int, endingAt pos: Position) {
+    updateRegister(at: CaptureRegister(capNum)) {
+      $0.endCapture(pos)
+    }
+  }
+
+  mutating func setCaptureValue(_ capNum: Int, _ value: Any) {
+    updateRegister(at: CaptureRegister(capNum)) {
+      $0.registerValue(value)
+    }
+  }
+
   mutating func abort(_ e: Error? = nil) {
     if let e = e {
       self.failureReason = e
@@ -459,12 +499,11 @@ extension Processor {
       let (imm, reg) = payload.pairedImmediateInt
       let int = Int(asserting: imm)
       assert(int == imm)
-
-      registers[reg] = int
+      updateRegister(at: reg, to: int)
       controller.step()
     case .moveCurrentPosition:
       let reg = payload.position
-      registers[reg] = currentPosition
+      updateRegister(at: reg, to: currentPosition)
       controller.step()
     case .restorePosition:
       let reg = payload.position
@@ -478,7 +517,7 @@ extension Processor {
       if registers[int] == 0 {
         controller.pc = addr
       } else {
-        registers[int] -= 1
+        updateRegister(at: int) { $0 -= 1 }
         controller.step()
       }
     case .condBranchSamePosition:
@@ -622,7 +661,7 @@ extension Processor {
           signalFailure()
           return
         }
-        registers[valReg] = val
+        updateRegister(at: valReg, to: val)
         resume(at: nextIdx)
         controller.step()
       } catch {
@@ -634,13 +673,13 @@ extension Processor {
       let (isScalarMode, capture) = payload.captureAndMode
       let capNum = Int(
         asserting: capture.rawValue)
-      guard capNum < storedCaptures.count else {
+      guard capNum < registers.storedCaptures.count else {
         fatalError("Should this be an assert?")
       }
       // TODO:
       //   Should we assert it's not finished yet?
       //   What's the behavior there?
-      let cap = storedCaptures[capNum]
+      let cap = registers.storedCaptures[capNum]
       guard let range = cap.range else {
         signalFailure()
         return
@@ -652,13 +691,13 @@ extension Processor {
     case .beginCapture:
       let capNum = Int(
         asserting: payload.capture.rawValue)
-      storedCaptures[capNum].startCapture(currentPosition)
+      setCapture(capNum, startingAt: currentPosition)
       controller.step()
 
     case .endCapture:
       let capNum = Int(
         asserting: payload.capture.rawValue)
-      storedCaptures[capNum].endCapture(currentPosition)
+      setCapture(capNum, endingAt: currentPosition)
       controller.step()
 
     case .transformCapture:
@@ -668,11 +707,11 @@ extension Processor {
 
       do {
         // FIXME: Pass input or the slice?
-        guard let value = try transform(input, storedCaptures[capNum]) else {
+        guard let value = try transform(input, registers.storedCaptures[capNum]) else {
           signalFailure()
           return
         }
-        storedCaptures[capNum].registerValue(value)
+        setCaptureValue(capNum, value)
         controller.step()
       } catch {
         abort(error)
@@ -683,7 +722,7 @@ extension Processor {
       let (val, cap) = payload.pairedValueCapture
       let value = registers[val]
       let capNum = Int(asserting: cap.rawValue)
-      storedCaptures[capNum].registerValue(value)
+      setCaptureValue(capNum, value)
       controller.step()
     }
   }