From f924c43e873f143a170c244d541808e7e36efd6b Mon Sep 17 00:00:00 2001 From: park-byeong-gwan Date: Thu, 30 May 2024 19:21:05 +0900 Subject: [PATCH 01/63] prepare typedThrow --- .../Tetra/Combine/ExperimentalMapTask.swift | 26 ++++++++++++------- .../Concurrency/AsyncSequencePublisher.swift | 2 +- 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/Sources/Tetra/Combine/ExperimentalMapTask.swift b/Sources/Tetra/Combine/ExperimentalMapTask.swift index c5c6dcc..0d17601 100644 --- a/Sources/Tetra/Combine/ExperimentalMapTask.swift +++ b/Sources/Tetra/Combine/ExperimentalMapTask.swift @@ -23,7 +23,7 @@ internal struct MultiMapTask: Publisher whe public let maxTasks:Subscribers.Demand public let upstream:Upstream - public let transform:@Sendable (Upstream.Output) async -> Result + public let transform:@Sendable (Upstream.Output) async throws(Failure) -> Output public func receive(subscriber: S) where S : Subscriber, Upstream.Failure == S.Failure, Output == S.Input { let processor = Inner(maxTasks: maxTasks, subscriber: subscriber, transform: transform) @@ -65,11 +65,15 @@ extension MultiMapTask { let valueSource = AsyncStream>.makeStream() let demandSource = AsyncStream.makeStream() let state: some UnfairStateLock> = createUncheckedStateLock(uncheckedState: TaskState()) - let transform:@Sendable (Upstream.Output) async -> Result + let transform:@Sendable (Upstream.Output) async throws(Failure) -> Output let combineIdentifier = CombineIdentifier() - init(maxTasks:Subscribers.Demand, subscriber:S, transform: @escaping @Sendable (Upstream.Output) async -> Result) { + init( + maxTasks:Subscribers.Demand, + subscriber:S, + transform: @escaping @Sendable (Upstream.Output) async throws(Failure) -> Output + ) { self.maxTasks = maxTasks self.transform = transform state.withLockUnchecked{ @@ -95,16 +99,20 @@ extension MultiMapTask { break case .success(let success): let flag = group.addTaskUnlessCancelled(priority: nil) { - switch await transform(success) { - case .failure(let failure): - send(completion: .failure(failure)) - throw CancellationError() - case .success(let value): + var shouldBreak = false + do { + let value = try await transform(success) if let demand = send(value) { subscription.request(demand) } else { - throw CancellationError() + shouldBreak = true } + } catch { + send(completion: .failure(error)) + shouldBreak = true + } + if shouldBreak { + throw CancellationError() } } if !flag { diff --git a/Sources/Tetra/Concurrency/AsyncSequencePublisher.swift b/Sources/Tetra/Concurrency/AsyncSequencePublisher.swift index 1da40b5..94f8b76 100644 --- a/Sources/Tetra/Concurrency/AsyncSequencePublisher.swift +++ b/Sources/Tetra/Concurrency/AsyncSequencePublisher.swift @@ -20,7 +20,7 @@ public extension AsyncSequence { public struct AsyncSequencePublisher: Publisher { public typealias Output = Base.Element - public typealias Failure = Error + public typealias Failure = Base.Failure public var base:Base From bfa0b13fb53db350d5ba8864be1a16e8d954957a Mon Sep 17 00:00:00 2001 From: park-byeong-gwan Date: Sat, 1 Jun 2024 00:15:52 +0900 Subject: [PATCH 02/63] adapt `fullTypedThrow` --- Package.swift | 1 + .../Tetra/Combine/Combine+Concurrency.swift | 43 +--------------- .../Tetra/Combine/CompatAsyncPublisher.swift | 4 +- .../CompatAsyncThrowingPublisher.swift | 5 +- .../Tetra/Combine/ExperimentalMapTask.swift | 10 ++-- .../Concurrency/AsyncSequencePublisher.swift | 12 +---- .../Concurrency/AsyncTypedSequence.swift | 50 +++---------------- .../Notification+AsyncSequence.swift | 30 +++-------- Tests/TetraTests/MultiMapTaskTests.swift | 12 ++--- 9 files changed, 35 insertions(+), 132 deletions(-) diff --git a/Package.swift b/Package.swift index d575614..41e473e 100644 --- a/Package.swift +++ b/Package.swift @@ -32,6 +32,7 @@ let package = Package( dependencies: [], swiftSettings: [ .enableExperimentalFeature("StrictConcurrency=complete"), + .enableUpcomingFeature("FullTypedThrows"), ] ), .testTarget( diff --git a/Sources/Tetra/Combine/Combine+Concurrency.swift b/Sources/Tetra/Combine/Combine+Concurrency.swift index eed27c5..2931b06 100644 --- a/Sources/Tetra/Combine/Combine+Concurrency.swift +++ b/Sources/Tetra/Combine/Combine+Concurrency.swift @@ -16,21 +16,9 @@ public extension Publisher { } -public extension TetraExtension where Base: Publisher, Base.Failure == Never { - - var values: WrappedAsyncSequence { - if #available(iOS 15.0, macOS 12.0, macCatalyst 15.0, tvOS 15.0, watchOS 8.0, *) { - return WrappedAsyncSequence(base: base.values) - } else { - return WrappedAsyncSequence(base: CompatAsyncPublisher(publisher: base)) - } - } - -} - public extension TetraExtension where Base: Publisher { - var values: some AsyncTypedSequence { + var values: some AsyncTypedSequence { if #available(iOS 15.0, macOS 12.0, macCatalyst 15.0, tvOS 15.0, watchOS 8.0, *) { return base.values } else { @@ -54,35 +42,8 @@ public extension Publisher { @_spi(Experimental) @inlinable - func multiMapTask(maxTasks: Subscribers.Demand = .max(1), transform: @escaping @Sendable (Output) async -> Result) -> MultiMapTask where Output: Sendable { + func multiMapTask(maxTasks: Subscribers.Demand = .max(1), transform: @escaping @Sendable (Output) async throws(Self.Failure) -> T) -> MultiMapTask where Output: Sendable { MultiMapTask(maxTasks: maxTasks, upstream: self, transform: transform) } - -} - -public extension Publisher { - - @available(iOS, deprecated: 15.0, renamed: "values", message: "will be removed on Swift 6") - @available(macCatalyst, deprecated: 15.0, renamed: "values", message: "will be removed on Swift 6") - @available(tvOS, deprecated: 15.0, renamed: "values", message: "will be removed on Swift 6") - @available(macOS, deprecated: 12.0, renamed: "values", message: "will be removed on Swift 6") - @available(watchOS, deprecated: 8.0, renamed: "values", message: "will be removed on Swift 6") - var asyncSequence: some AsyncTypedSequence { - return TetraExtension(self).values - } - -} - -public extension Publisher where Failure == Never { - - @available(iOS, deprecated: 15.0, renamed: "values", message: "will be removed on Swift 6") - @available(macCatalyst, deprecated: 15.0, renamed: "values", message: "will be removed on Swift 6") - @available(tvOS, deprecated: 15.0, renamed: "values", message: "will be removed on Swift 6") - @available(macOS, deprecated: 12.0, renamed: "values", message: "will be removed on Swift 6") - @available(watchOS, deprecated: 8.0, renamed: "values", message: "will be removed on Swift 6") - var asyncSequence:WrappedAsyncSequence { - return TetraExtension(self).values - } - } diff --git a/Sources/Tetra/Combine/CompatAsyncPublisher.swift b/Sources/Tetra/Combine/CompatAsyncPublisher.swift index 41d8985..1febfd2 100644 --- a/Sources/Tetra/Combine/CompatAsyncPublisher.swift +++ b/Sources/Tetra/Combine/CompatAsyncPublisher.swift @@ -13,6 +13,7 @@ public struct CompatAsyncPublisher: AsyncSequence where P.Failure = public typealias AsyncIterator = Iterator public typealias Element = P.Output + public typealias Failure = P.Failure public var publisher:P @@ -24,9 +25,10 @@ public struct CompatAsyncPublisher: AsyncSequence where P.Failure = self.publisher = publisher } - public struct Iterator: NonThrowingAsyncIteratorProtocol { + public struct Iterator: AsyncIteratorProtocol { public typealias Element = P.Output + public typealias Failure = P.Failure private let inner = AsyncSubscriber

() private let reference:AnyCancellable diff --git a/Sources/Tetra/Combine/CompatAsyncThrowingPublisher.swift b/Sources/Tetra/Combine/CompatAsyncThrowingPublisher.swift index 367a076..b7cb0a6 100644 --- a/Sources/Tetra/Combine/CompatAsyncThrowingPublisher.swift +++ b/Sources/Tetra/Combine/CompatAsyncThrowingPublisher.swift @@ -12,7 +12,6 @@ import Foundation public struct CompatAsyncThrowingPublisher: AsyncTypedSequence { public typealias AsyncIterator = Iterator - public typealias Element = P.Output public var publisher:P @@ -23,11 +22,11 @@ public struct CompatAsyncThrowingPublisher: AsyncTypedSequence { public struct Iterator: AsyncIteratorProtocol { public typealias Element = P.Output - + public typealias Failure = P.Failure private let inner = AsyncThrowingSubscriber

() private let reference:AnyCancellable - public mutating func next() async throws -> P.Output? { + public mutating func next() async throws(P.Failure) -> P.Output? { let result = await withTaskCancellationHandler(operation: inner.next) { [reference] in reference.cancel() } diff --git a/Sources/Tetra/Combine/ExperimentalMapTask.swift b/Sources/Tetra/Combine/ExperimentalMapTask.swift index 8f356e5..5aca7bf 100644 --- a/Sources/Tetra/Combine/ExperimentalMapTask.swift +++ b/Sources/Tetra/Combine/ExperimentalMapTask.swift @@ -37,7 +37,7 @@ public struct MultiMapTask: Publisher where public init( maxTasks: Subscribers.Demand = .max(1), upstream: Upstream, - transform: @Sendable @escaping (Upstream.Output) async -> Result + transform: @Sendable @escaping (Upstream.Output) async throws(Failure) -> Output ) { precondition(maxTasks != .none, "maxTasks can not be zero") self.maxTasks = maxTasks @@ -225,12 +225,12 @@ extension MultiMapTask { } else { try? await withThrowingTaskGroup(of: Void.self, returning: Void.self) { group in defer { terminateStream() } - var iterator = group.makeAsyncIterator() - let stream = AsyncThrowingStream(unfolding: { try await iterator.next() }) - async let subTask:() = { - for try await _ in stream { + async let subTask:() = { [iter = group.makeAsyncIterator()] in + var iterator = iter + while let _ = try await iterator.next() { } + }() await localTask( subscription: subscription, diff --git a/Sources/Tetra/Concurrency/AsyncSequencePublisher.swift b/Sources/Tetra/Concurrency/AsyncSequencePublisher.swift index 8b7cea5..ac0dbbb 100644 --- a/Sources/Tetra/Concurrency/AsyncSequencePublisher.swift +++ b/Sources/Tetra/Concurrency/AsyncSequencePublisher.swift @@ -25,16 +25,6 @@ public extension TetraExtension where Base: AsyncSequence & Sendable { } -public extension AsyncSequence where Self:Sendable { - - @available(*, deprecated, message: "use explicit extension publisher property instead, will be removed on Swift 6") - @inlinable - var asyncPublisher:AsyncSequencePublisher { - TetraExtension(base: self).publisher - } - -} - public struct AsyncSequencePublisher: Publisher { public typealias Output = Base.Element @@ -146,7 +136,7 @@ extension AsyncSequencePublisher { do { for await var pending in demandSource.stream { while pending > .none { - if let value = try await iterator.next() { + if let value = try await iterator.next(isolation: nil) { pending -= 1 if let newDemand = send(value) { pending += newDemand diff --git a/Sources/Tetra/Concurrency/AsyncTypedSequence.swift b/Sources/Tetra/Concurrency/AsyncTypedSequence.swift index 8b71ca9..72fec7d 100644 --- a/Sources/Tetra/Concurrency/AsyncTypedSequence.swift +++ b/Sources/Tetra/Concurrency/AsyncTypedSequence.swift @@ -7,53 +7,19 @@ import Combine import _Concurrency +import Foundation -@usableFromInline -internal protocol NonThrowingAsyncIteratorProtocol: AsyncIteratorProtocol { - - mutating func next() async -> Element? - -} -@usableFromInline -internal protocol NonThrowingAsyncSequence: AsyncSequence where AsyncIterator: NonThrowingAsyncIteratorProtocol { -} - -public protocol AsyncTypedSequence:AsyncSequence {} +public protocol AsyncTypedSequence:AsyncSequence {} @available(iOS 15.0, macOS 12.0, macCatalyst 15.0, tvOS 15.0, watchOS 8.0, *) extension AsyncThrowingPublisher: AsyncTypedSequence {} +@available(iOS 15.0, tvOS 15.0, macCatalyst 15.0, watchOS 8.0, macOS 12.0, *) +extension AsyncPublisher: AsyncTypedSequence {} + +extension AsyncThrowingStream: AsyncTypedSequence {} +extension AsyncStream: AsyncTypedSequence {} @available(iOS 15.0, tvOS 15.0, macCatalyst 15.0, watchOS 8.0, macOS 12.0, *) -extension AsyncPublisher.Iterator: NonThrowingAsyncIteratorProtocol {} -extension AsyncStream.Iterator: NonThrowingAsyncIteratorProtocol {} -extension AsyncStream: NonThrowingAsyncSequence {} -public struct WrappedAsyncSequence:AsyncSequence { - - public func makeAsyncIterator() -> Iterator { - builder() - } - - public typealias AsyncIterator = Iterator - private let builder: () -> AsyncIterator - - internal init(base:T) where T.Element == Element, T.AsyncIterator: NonThrowingAsyncIteratorProtocol { - builder = { [base] in - Iterator(base: base.makeAsyncIterator()) - } - } - - public struct Iterator: AsyncIteratorProtocol { - - private var iterator:any NonThrowingAsyncIteratorProtocol - - mutating public func next() async -> Element? { - await iterator.next() - } - - internal init(base:T) where T.Element == Element { - iterator = base - } - } -} +extension NotificationCenter.Notifications: AsyncTypedSequence {} diff --git a/Sources/Tetra/Concurrency/Notification+AsyncSequence.swift b/Sources/Tetra/Concurrency/Notification+AsyncSequence.swift index e0cb07a..ff2f4a2 100644 --- a/Sources/Tetra/Concurrency/Notification+AsyncSequence.swift +++ b/Sources/Tetra/Concurrency/Notification+AsyncSequence.swift @@ -13,40 +13,23 @@ extension NotificationCenter: TetraExtended {} extension TetraExtension where Base: NotificationCenter { - func notifications(named: Notification.Name, object: AnyObject? = nil) -> WrappedAsyncSequence { + func notifications(named: Notification.Name, object: AnyObject? = nil) -> some AsyncTypedSequence { if #available(iOS 15.0, tvOS 15.0, macCatalyst 15.0, watchOS 8.0, macOS 12.0, *) { - return WrappedAsyncSequence(base: base.notifications(named: named, object: object)) + return base.notifications(named: named, object: object) } else { - return WrappedAsyncSequence(base: NotificationSequence(center: base, named: named, object: object)) + return NotificationSequence(center: base, named: named, object: object) } } } -@available(iOS 13.0, tvOS 13.0, macCatalyst 13.0, watchOS 6.0, macOS 10.15, *) -public extension NotificationCenter { - - - @available(iOS, introduced: 13.0, deprecated: 15.0, renamed: "notifications", message: "use explicit extension method, will be removed on Swift 6") - @available(tvOS, introduced: 13.0, deprecated: 15.0, renamed: "notifications", message: "use explicit extension method, will be removed on Swift 6") - @available(macCatalyst, introduced: 13.0, deprecated: 15.0, renamed: "notifications", message: "use explicit extension method, will be removed on Swift 6") - @available(watchOS, introduced: 6.0, deprecated: 8.0, renamed: "notifications", message: "use explicit extension method, will be removed on Swift 6") - @available(macOS, introduced: 10.15, deprecated: 12.0, renamed: "notifications", message: "use explicit extension method, will be removed on Swift 6") - func sequence(named:Notification.Name, object:AnyObject? = nil) -> WrappedAsyncSequence { - tetra.notifications(named: named, object: object) - } - -} - -@available(iOS 15.0, tvOS 15.0, macCatalyst 15.0, watchOS 8.0, macOS 12.0, *) -extension NotificationCenter.Notifications.AsyncIterator: NonThrowingAsyncIteratorProtocol {} -public final class NotificationSequence: AsyncSequence, Sendable { +public final class NotificationSequence: AsyncTypedSequence, Sendable { - public typealias Element = Notification public typealias AsyncIterator = Iterator + public func makeAsyncIterator() -> Iterator { Iterator(parent: self) } @@ -54,8 +37,9 @@ public final class NotificationSequence: AsyncSequence, Sendable { let center: NotificationCenter private let lock:some UnfairStateLock = createUncheckedStateLock(uncheckedState: NotficationState()) - public struct Iterator: NonThrowingAsyncIteratorProtocol { + public struct Iterator: AsyncIteratorProtocol { public typealias Element = Notification + public typealias Failure = Never let parent:NotificationSequence diff --git a/Tests/TetraTests/MultiMapTaskTests.swift b/Tests/TetraTests/MultiMapTaskTests.swift index bcf1c81..9bafbdd 100644 --- a/Tests/TetraTests/MultiMapTaskTests.swift +++ b/Tests/TetraTests/MultiMapTaskTests.swift @@ -22,7 +22,7 @@ final class MultiMapTaskTests: XCTestCase { ) let cancellable = MultiMapTask(maxTasks: .max(1), upstream: publisher) { await Task.yield() - return .success($0) + return $0 }.sink { _ in expect.fulfill() } receiveValue: { @@ -50,7 +50,7 @@ final class MultiMapTaskTests: XCTestCase { } XCTAssertTrue(Task.isCancelled) } - return .success(value) + return value }.handleEvents( receiveCancel: { expect.fulfill() } ) @@ -71,12 +71,12 @@ final class MultiMapTaskTests: XCTestCase { let target = try XCTUnwrap(sequence.randomElement()) let upstream = sequence.publisher.setFailureType(to: CancellationError.self) let expect = expectation(description: "task failure") - let pub = MultiMapTask(maxTasks: .max(1), upstream: upstream, transform: { value in + let pub = MultiMapTask(maxTasks: .max(1), upstream: upstream) { value in if value == target { - return .failure(CancellationError()) as Result + throw CancellationError() } - return .success(value) as Result - }) + return value + } var bag = Set() pub.sink { completion in switch completion { From cfd1e5d3b4b2714aa3b5fd87058b3d958aa2a753 Mon Sep 17 00:00:00 2001 From: park-byeong-gwan Date: Sat, 1 Jun 2024 02:31:09 +0900 Subject: [PATCH 03/63] implement custom isolation check --- Sources/Tetra/Concurrency/DispatchSerialExecutor.swift | 4 ++++ Sources/Tetra/Concurrency/RunLoopExecutor.swift | 5 ++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/Sources/Tetra/Concurrency/DispatchSerialExecutor.swift b/Sources/Tetra/Concurrency/DispatchSerialExecutor.swift index 00b156e..88d413b 100644 --- a/Sources/Tetra/Concurrency/DispatchSerialExecutor.swift +++ b/Sources/Tetra/Concurrency/DispatchSerialExecutor.swift @@ -99,6 +99,10 @@ public final class DispatchQueueExecutor: SerialExecutor { return result } + public func checkIsolated() { + dispatchPrecondition(condition: .onQueue(queue)) + } + } diff --git a/Sources/Tetra/Concurrency/RunLoopExecutor.swift b/Sources/Tetra/Concurrency/RunLoopExecutor.swift index 2a03a01..55a1319 100644 --- a/Sources/Tetra/Concurrency/RunLoopExecutor.swift +++ b/Sources/Tetra/Concurrency/RunLoopExecutor.swift @@ -71,7 +71,7 @@ public final class RunLoopExecutor: SerialExecutor { return runner.thread == other.runner.thread } - public func checkIsolation() { + public func checkIsolated() { precondition(runner.thread == Thread.current, "Expected \(runner.thread) but found \(Thread.current)") } @@ -133,6 +133,9 @@ public final class LegacyRunLoopExecutor: SerialExecutor { } #endif + public func checkIsolated() { + precondition(runner.thread == Thread.current, "Expected \(runner.thread) but found \(Thread.current)") + } } From 8131b43f8451524f43982733922c4840b8a91c6b Mon Sep 17 00:00:00 2001 From: park-byeong-gwan Date: Sat, 1 Jun 2024 10:45:32 +0900 Subject: [PATCH 04/63] make properties private --- Sources/Tetra/Combine/ExperimentalMapTask.swift | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Sources/Tetra/Combine/ExperimentalMapTask.swift b/Sources/Tetra/Combine/ExperimentalMapTask.swift index ef41a9f..b22d15d 100644 --- a/Sources/Tetra/Combine/ExperimentalMapTask.swift +++ b/Sources/Tetra/Combine/ExperimentalMapTask.swift @@ -60,11 +60,11 @@ extension MultiMapTask { struct Inner: CustomCombineIdentifierConvertible where S.Failure == Failure, S.Input == Output { - let maxTasks:Subscribers.Demand - let valueSource = AsyncStream>.makeStream() - let demandSource = AsyncStream.makeStream() - let state: some UnfairStateLock> = createUncheckedStateLock(uncheckedState: TaskState()) - let transform:@Sendable (Upstream.Output) async throws(Failure) -> Output + private let maxTasks:Subscribers.Demand + private let valueSource = AsyncStream>.makeStream() + private let demandSource = AsyncStream.makeStream() + private let state: some UnfairStateLock> = createUncheckedStateLock(uncheckedState: TaskState()) + private let transform:@Sendable (Upstream.Output) async throws(Failure) -> Output let combineIdentifier = CombineIdentifier() From 01e39ccd9a3a8c11c016cfba297c34655d5563ab Mon Sep 17 00:00:00 2001 From: park-byeong-gwan Date: Tue, 4 Jun 2024 17:13:21 +0900 Subject: [PATCH 05/63] fix typedThrow compile issue --- Sources/Tetra/Combine/AsyncSubscriber.swift | 4 +- .../CompatAsyncThrowingPublisher.swift | 7 +- .../Concurrency/AsyncSequencePublisher.swift | 29 +++--- .../CoreDataStack+Concurrency.swift | 91 ++++++++----------- Sources/Tetra/Foundation/Mics.swift | 25 +++++ 5 files changed, 84 insertions(+), 72 deletions(-) diff --git a/Sources/Tetra/Combine/AsyncSubscriber.swift b/Sources/Tetra/Combine/AsyncSubscriber.swift index 37905e5..5c096a1 100644 --- a/Sources/Tetra/Combine/AsyncSubscriber.swift +++ b/Sources/Tetra/Combine/AsyncSubscriber.swift @@ -6,10 +6,10 @@ // import Foundation -import Combine +@preconcurrency import Combine @usableFromInline -internal struct AsyncSubscriber: Subscriber, Cancellable { +internal struct AsyncSubscriber: Sendable, Subscriber, Cancellable { public typealias Input = P.Output diff --git a/Sources/Tetra/Combine/CompatAsyncThrowingPublisher.swift b/Sources/Tetra/Combine/CompatAsyncThrowingPublisher.swift index 5f2e371..a0c8435 100644 --- a/Sources/Tetra/Combine/CompatAsyncThrowingPublisher.swift +++ b/Sources/Tetra/Combine/CompatAsyncThrowingPublisher.swift @@ -23,13 +23,14 @@ public struct CompatAsyncThrowingPublisher: AsyncTypedSequence { public struct Iterator: AsyncIteratorProtocol { public typealias Element = P.Output + public typealias Failure = P.Failure @usableFromInline internal let inner = AsyncSubscriber

() @usableFromInline internal let reference:AnyCancellable @inlinable - public mutating func next() async throws -> P.Output? { + public mutating func next(isolation actor: isolated (any Actor)?) async throws(P.Failure) -> P.Output? { let result = await withTaskCancellationHandler(operation: inner.next) { [reference] in reference.cancel() } @@ -43,6 +44,10 @@ public struct CompatAsyncThrowingPublisher: AsyncTypedSequence { } } + public mutating func next() async throws(Failure) -> P.Output? { + try await next(isolation: nil) + } + @usableFromInline internal init(source: P) { self.reference = AnyCancellable(inner) diff --git a/Sources/Tetra/Concurrency/AsyncSequencePublisher.swift b/Sources/Tetra/Concurrency/AsyncSequencePublisher.swift index 89e0bb7..e938e68 100644 --- a/Sources/Tetra/Concurrency/AsyncSequencePublisher.swift +++ b/Sources/Tetra/Concurrency/AsyncSequencePublisher.swift @@ -133,26 +133,27 @@ extension AsyncSequencePublisher { }?.receive(subscription: self) var iterator = base.makeAsyncIterator() await withTaskCancellationHandler { - do { - for await var pending in demandSource.stream { - while pending > .none { - if let value = try await iterator.next(isolation: nil) { - pending -= 1 - if let newDemand = send(value) { - pending += newDemand - } else { - return - } + for await var pending in demandSource.stream { + while pending > .none { + pending -= 1 + guard let result = await wrapToResult(&iterator) else { + send(completion: .finished) + return + } + switch result { + case .failure(let error): + send(completion: .failure(error)) + return + case .success(let value): + if let newDemand = send(value) { + pending += newDemand } else { - send(completion: .finished) return } } } - send(completion: .finished) - } catch { - send(completion: .failure(error)) } + send(completion: .finished) } onCancel: { demandSource.continuation.finish() send(completion: nil) diff --git a/Sources/Tetra/Concurrency/CoreDataStack+Concurrency.swift b/Sources/Tetra/Concurrency/CoreDataStack+Concurrency.swift index a851be8..4115056 100644 --- a/Sources/Tetra/Concurrency/CoreDataStack+Concurrency.swift +++ b/Sources/Tetra/Concurrency/CoreDataStack+Concurrency.swift @@ -18,26 +18,15 @@ extension NSManagedObjectContext: TetraExtended {} extension TetraExtension where Base: NSPersistentStoreCoordinator { @usableFromInline - internal func _perform(_ body: () throws -> T) async rethrows -> T { - let result:Result - do { - let value = try await withoutActuallyEscaping(body) { escapingClosure in - try await withUnsafeThrowingContinuation { continuation in - base.perform { - continuation.resume(with: Result{ try escapingClosure() }) - } + internal func _perform(_ body: () throws(Failure) -> T) async throws(Failure) -> T { + let value:Result = await withoutActuallyEscaping(body) { escapingClosure in + return await withUnsafeContinuation { continuation in + base.perform { + continuation.resume(returning: wrapToResult(escapingClosure)) } } - result = .success(value) - } catch { - result = .failure(error) - } - switch result { - case .success(let success): - return success - case .failure: - try result._rethrowOrFail() } + return try value.get() } @inlinable @@ -50,20 +39,15 @@ extension TetraExtension where Base: NSPersistentStoreCoordinator { } @usableFromInline - internal func _performAndWait(_ body: () throws -> T) rethrows -> T { - var result:Result? = nil + internal func _performAndWait(_ body: () throws(Failure) -> T) throws(Failure) -> T { + var result:Result? = nil base.performAndWait { - result = Result { try body() } + result = wrapToResult(body) } guard let result else { preconditionFailure("performAndWait didn't run") } - switch result { - case .success(let success): - return success - case .failure: - try result._rethrowOrFail() - } + return try result.get() } @inlinable @@ -94,15 +78,15 @@ extension TetraExtension where Base: NSManagedObjectContext { /// /// This method supports reentrancy — meaning it’s safe to call the method again, from within the closure, before the previous invocation completes. @usableFromInline - internal func _performImmediate( - _ body: () throws -> T - ) rethrows -> Result? { + internal func _performImmediate( + _ body: () throws(Failure) -> T + ) throws(Failure) -> Result? { // Suppress deprecation let lock:any ObjcLocking = base // NSManagedObjectContext has Reentrant Locking guard lock.tryLock() else { return nil } defer { lock.unlock() } - let value = try performAndWait(body) + let value = try _performAndWait(body) return .success(value) } @@ -155,20 +139,15 @@ extension TetraExtension where Base: NSManagedObjectContext { } @usableFromInline - internal func _performAndWait(_ body: () throws -> T) rethrows -> T { - var result:Result? = nil + internal func _performAndWait(_ body: () throws(Failure) -> T) throws(Failure) -> T { + var result:Result? = nil base.performAndWait { - result = Result { try body() } + result = wrapToResult(body) } guard let result else { preconditionFailure("performAndWait didn't run") } - switch result { - case .success(let success): - return success - case .failure: - try result._rethrowOrFail() - } + return try result.get() } @inlinable @@ -194,26 +173,27 @@ extension TetraExtension where Base: NSPersistentContainer { } @usableFromInline - internal func _performBackground(_ body: (NSManagedObjectContext) throws -> T) async rethrows -> T { - let result:Result + internal func _convertToResult(_ context:NSManagedObjectContext, _ body: (NSManagedObjectContext) throws(Failure) -> T) -> Result { do { - let value = try await withoutActuallyEscaping(body) { escapingClosure in - try await withUnsafeThrowingContinuation { continuation in - base.performBackgroundTask { newContext in - continuation.resume(with: Result{ try escapingClosure(newContext) }) - } - } - } - result = .success(value) + let value = try body(context) + return .success(value) } catch { - result = .failure(error) + return .failure(error) } - switch result { - case .success(let success): - return success - case .failure: - try result._rethrowOrFail() + } + + + @usableFromInline + internal func _performBackground(_ body: (NSManagedObjectContext) throws(Failure) -> T) async throws(Failure) -> T { + let result = await withoutActuallyEscaping(body) { escapingClosure in + await withUnsafeContinuation { continuation in + base.performBackgroundTask { newContext in + let result = _convertToResult(newContext, escapingClosure) + continuation.resume(returning: result) + } + } } + return try result.get() } } @@ -238,3 +218,4 @@ internal enum CoreDataScheduledTaskType: Sendable, Hashable { } #endif + diff --git a/Sources/Tetra/Foundation/Mics.swift b/Sources/Tetra/Foundation/Mics.swift index d0d94ed..7591675 100644 --- a/Sources/Tetra/Foundation/Mics.swift +++ b/Sources/Tetra/Foundation/Mics.swift @@ -76,3 +76,28 @@ internal extension NSNumber { } + +@usableFromInline +internal +func wrapToResult(_ block: () throws(Failure) -> T) -> Result { + do { + return .success(try block()) + } catch { + return .failure(error) + } +} + +@usableFromInline +internal +func wrapToResult(_ iterator: inout Base) async -> Result? { + do { + let value = try await iterator.next(isolation: nil) + if let value { + return .success(value) + } else { + return nil + } + } catch { + return .failure(error) + } +} From e7e8987530671a13319bfd7ea04d8c18032c485b Mon Sep 17 00:00:00 2001 From: park-byeong-gwan Date: Tue, 4 Jun 2024 20:26:08 +0900 Subject: [PATCH 06/63] inline error conversion --- .../Concurrency/AsyncSequencePublisher.swift | 2 +- .../CoreDataStack+Concurrency.swift | 28 ++++++------------- Sources/Tetra/Foundation/Mics.swift | 2 ++ 3 files changed, 12 insertions(+), 20 deletions(-) diff --git a/Sources/Tetra/Concurrency/AsyncSequencePublisher.swift b/Sources/Tetra/Concurrency/AsyncSequencePublisher.swift index e938e68..e8778e0 100644 --- a/Sources/Tetra/Concurrency/AsyncSequencePublisher.swift +++ b/Sources/Tetra/Concurrency/AsyncSequencePublisher.swift @@ -119,7 +119,7 @@ extension AsyncSequencePublisher { }?.run() } - func run(_ base:consuming Base) async { + func run(_ base: Base) async { let token:Void? = try? await waitForCondition() defer { demandSource.continuation.finish() diff --git a/Sources/Tetra/Concurrency/CoreDataStack+Concurrency.swift b/Sources/Tetra/Concurrency/CoreDataStack+Concurrency.swift index 4115056..e53dbad 100644 --- a/Sources/Tetra/Concurrency/CoreDataStack+Concurrency.swift +++ b/Sources/Tetra/Concurrency/CoreDataStack+Concurrency.swift @@ -91,28 +91,17 @@ extension TetraExtension where Base: NSManagedObjectContext { } @usableFromInline - internal func _performEnqueue( - _ body: () throws -> T - ) async rethrows -> T { - let result:Result - do { - let value = try await withoutActuallyEscaping(body) { escapingClosure in - try await withUnsafeThrowingContinuation { continuation in - base.perform{ - continuation.resume(with: Result { try escapingClosure() }) - } + internal func _performEnqueue( + _ body: () throws(Failure) -> T + ) async throws(Failure) -> T { + let result:Result = await withoutActuallyEscaping(body) { escapingClosure in + await withUnsafeContinuation { continuation in + base.perform{ + continuation.resume(returning: wrapToResult(escapingClosure)) } } - result = .success(value) - } catch { - result = .failure(error) - } - switch result { - case .success(let success): - return success - case .failure: - try result._rethrowOrFail() } + return try result.get() } /// Asynchronously performs the specified closure on the context’s queue. @@ -172,6 +161,7 @@ extension TetraExtension where Base: NSPersistentContainer { } } + @inline(__always) @usableFromInline internal func _convertToResult(_ context:NSManagedObjectContext, _ body: (NSManagedObjectContext) throws(Failure) -> T) -> Result { do { diff --git a/Sources/Tetra/Foundation/Mics.swift b/Sources/Tetra/Foundation/Mics.swift index 7591675..4736edd 100644 --- a/Sources/Tetra/Foundation/Mics.swift +++ b/Sources/Tetra/Foundation/Mics.swift @@ -77,6 +77,7 @@ internal extension NSNumber { } +@inline(__always) @usableFromInline internal func wrapToResult(_ block: () throws(Failure) -> T) -> Result { @@ -87,6 +88,7 @@ func wrapToResult(_ block: () throws(Failure) -> T) -> Result(_ iterator: inout Base) async -> Result? { From 6e644cd8fe0c46c0caeb640074022fc32a25604c Mon Sep 17 00:00:00 2001 From: park-byeong-gwan Date: Thu, 6 Jun 2024 10:07:54 +0900 Subject: [PATCH 07/63] fix errors for swift 6 --- .../Tetra/Combine/Combine+Concurrency.swift | 1 + .../Tetra/Combine/ExperimentalMapTask.swift | 32 +++++++++-------- .../Tetra/Combine/Future+Concurrency.swift | 35 ++----------------- Sources/Tetra/Foundation/Mics.swift | 26 ++++++++++++++ 4 files changed, 48 insertions(+), 46 deletions(-) diff --git a/Sources/Tetra/Combine/Combine+Concurrency.swift b/Sources/Tetra/Combine/Combine+Concurrency.swift index 0f349f0..9767046 100644 --- a/Sources/Tetra/Combine/Combine+Concurrency.swift +++ b/Sources/Tetra/Combine/Combine+Concurrency.swift @@ -20,6 +20,7 @@ public extension TetraExtension where Base: Publisher { @inlinable var values: some AsyncTypedSequence { + if #available(iOS 15.0, tvOS 15.0, watchOS 8.0, macCatalyst 15.0, macOS 12.0, *) { return base.values } else { return CompatAsyncThrowingPublisher(publisher: base) diff --git a/Sources/Tetra/Combine/ExperimentalMapTask.swift b/Sources/Tetra/Combine/ExperimentalMapTask.swift index a7e4d60..57ebfee 100644 --- a/Sources/Tetra/Combine/ExperimentalMapTask.swift +++ b/Sources/Tetra/Combine/ExperimentalMapTask.swift @@ -100,22 +100,19 @@ extension MultiMapTask { break case .success(let success): let flag = group.addTaskUnlessCancelled(priority: nil) { - var shouldBreak = false - do { - let value = try await transform(success) - if let demand = send(value) { + let result = await wrapToResult(success, transform) + switch result { + case .failure(let error): + send(completion: .failure(error)) + throw CancellationError() + case .success(let success): + if let demand = send(success) { if demand > .none { subscription.request(demand) } } else { - shouldBreak = true + throw CancellationError() } - } catch { - send(completion: .failure(error)) - shouldBreak = true - } - if shouldBreak { - throw CancellationError() } } if !flag { @@ -229,12 +226,19 @@ extension MultiMapTask { } else { try? await withThrowingTaskGroup(of: Void.self, returning: Void.self) { group in defer { terminateStream() } - async let subTask:() = { [iter = group.makeAsyncIterator()] in - var iterator = iter + /* + this is very unsafe operation, and there is no way to prove race problem to compiler for now. + + But at least version before `DiscardingTaskGroup` exist, this implementation is safe from race problem. + + Because polling add queueing taskGroup is implemented in Busy waiting atomic alogrithnm. + */ + let unsafe = SuppressSendable(wrapped: group.makeAsyncIterator()) + async let subTask:() = { + var iterator = unsafe.wrapped while let _ = try await iterator.next() { } - }() await localTask( subscription: subscription, diff --git a/Sources/Tetra/Combine/Future+Concurrency.swift b/Sources/Tetra/Combine/Future+Concurrency.swift index 88394ea..945366d 100644 --- a/Sources/Tetra/Combine/Future+Concurrency.swift +++ b/Sources/Tetra/Combine/Future+Concurrency.swift @@ -8,35 +8,6 @@ import Foundation import Combine -public extension Combine.Future where Failure == Never { - - @available(iOS, deprecated: 15.0, renamed: "value") - @available(iOS, deprecated: 15.0, renamed: "value") - @available(iOS, deprecated: 15.0, renamed: "value") - @available(watchOS, deprecated: 8, renamed: "value") - @available(macOS, deprecated: 12.0, renamed: "value") - @inlinable - final var compatValue: Output { - get async { - if #available(iOS 15.0, tvOS 15.0, macCatalyst 15.0, watchOS 8.0, macOS 12.0, *) { - return await value - } else { - return await withCheckedContinuation{ continuation in - self.subscribe(AnySubscriber( - receiveSubscription: { - $0.request(.max(1)) - }, - receiveValue: { - continuation.resume(returning: $0) - return .none - } - )) - } - } - } - } - -} public extension Combine.Future { @@ -48,7 +19,7 @@ public extension Combine.Future { @available(macOS, deprecated: 12.0, renamed: "value") @inlinable final var compatValue: Output { - get async throws { + get async throws(Failure) { if #available(iOS 15.0, tvOS 15.0, macCatalyst 15.0, watchOS 8.0, macOS 12.0, *) { return try await value } else { @@ -57,8 +28,8 @@ public extension Combine.Future { receiveSubscription: { $0.request(.max(1)) }, - receiveValue: { - continuation.resume(returning: .success($0)) + receiveValue: { (value: sending Output) in + continuation.resume(returning: .success(value)) return .none }, receiveCompletion: { diff --git a/Sources/Tetra/Foundation/Mics.swift b/Sources/Tetra/Foundation/Mics.swift index 4736edd..1869c13 100644 --- a/Sources/Tetra/Foundation/Mics.swift +++ b/Sources/Tetra/Foundation/Mics.swift @@ -88,6 +88,7 @@ func wrapToResult(_ block: () throws(Failure) -> T) -> Result(_ iterator: inout Base) async -> return .failure(error) } } + + +@inline(__always) +@usableFromInline +internal func wrapToResult(_ value:T, _ transform: (T) async throws(Failure) -> U) async -> Result { + do { + let success = try await transform(value) + return .success(success) + } catch { + return .failure(error) + } +} + +/// use only when there is no way to prove no data race is occur to the compiler +@usableFromInline +struct SuppressSendable: @unchecked Sendable { + + @usableFromInline + var wrapped:T + + @usableFromInline + init(wrapped: T) { + self.wrapped = wrapped + } +} From d9878ada535812977b579da31db5c44441728a39 Mon Sep 17 00:00:00 2001 From: park-byeong-gwan Date: Fri, 7 Jun 2024 02:38:10 +0900 Subject: [PATCH 08/63] Combine&Concurrency: reimplement subscription model to fix internal behavior change in swift 6 which cause test fail --- .../Combine/AsyncSubscriptionState.swift | 143 ++++++++++++++++++ .../Tetra/Combine/ExperimentalMapTask.swift | 85 +++++------ .../Tetra/Combine/Publishers+MapTask.swift | 86 ++++++----- .../Tetra/Combine/Publishers+TryMapTask.swift | 68 +++++---- Tests/TetraTests/MapTaskTests.swift | 2 +- Tests/TetraTests/MultiMapTaskTests.swift | 14 +- 6 files changed, 275 insertions(+), 123 deletions(-) create mode 100644 Sources/Tetra/Combine/AsyncSubscriptionState.swift diff --git a/Sources/Tetra/Combine/AsyncSubscriptionState.swift b/Sources/Tetra/Combine/AsyncSubscriptionState.swift new file mode 100644 index 0000000..deae9fc --- /dev/null +++ b/Sources/Tetra/Combine/AsyncSubscriptionState.swift @@ -0,0 +1,143 @@ +// +// AsyncSubscriptionState.swift +// +// +// Created by 박병관 on 6/7/24. +// + +import Foundation +import Combine + +enum AsyncSubscriptionState { + + + case waiting + case suspending(UnsafeContinuation) + case cached(any Subscription) + case cancelled + case finished + + var subscription:(any Subscription)? { + guard case let .cached(subscription) = self else { + return nil + } + return subscription + } + + + enum Event { + + case suspend(UnsafeContinuation) + case resume(any Subscription) + case cancel + case finish + + } + + enum Effect { + + case resume(UnsafeContinuation) + case raise(UnsafeContinuation) + case cancel(any Subscription) + + + consuming func run() { + switch self { + case .resume(let unsafeContinuation): + unsafeContinuation.resume() + case .raise(let unsafeContinuation): + unsafeContinuation.resume(throwing: CancellationError()) + case .cancel(let subscription): + subscription.cancel() + } + } + + } + + + mutating func transition(_ event:Event) -> sending Effect? { + switch event { + case .suspend(let unsafeContinuation): + return suspend(unsafeContinuation) + case .resume(let subscription): + return resume(subscription) + case .cancel: + return onCancel() + case .finish: + return finish() + } + } + + + + private mutating func onCancel() -> Effect?{ + switch self { + case .waiting, .cancelled: + self = .cancelled + return nil + case .suspending(let unsafeContinuation): + self = .cancelled + return .raise(unsafeContinuation) + case .cached(let subscription): + self = .cancelled + return .cancel(subscription) + case .finished: + return nil + } + } + + private mutating func resume(_ subscription:any Subscription) -> Effect? { + switch self { + case .waiting: + self = .cached(subscription) + return nil + case .suspending(let unsafeContinuation): + self = .cached(subscription) + return .resume(unsafeContinuation) + case .cached(let old): + self = .cached(subscription) + assertionFailure("Received Subscription more than Once") + return .cancel(old) + case .cancelled: + fallthrough + case .finished: + return .cancel(subscription) + } + } + + private mutating func suspend(_ continuation: UnsafeContinuation) -> sending Effect? { + switch self { + case .waiting: + self = .suspending(continuation) + return nil + case .suspending(let unsafeContinuation): + self = .suspending(continuation) + assertionFailure("Received Continuation more than Once") + return .raise(unsafeContinuation) + case .cancelled: + return .raise(continuation) + case .finished: + fallthrough + case .cached: + return .resume(continuation) + } + } + + private mutating func finish() -> sending Effect? { + switch self { + case .suspending(let unsafeContinuation): + self = .finished + return .resume(unsafeContinuation) + case .cached(let subscription): + fallthrough + case .waiting: + self = .finished + fallthrough + case .finished: + fallthrough + case .cancelled: + return nil + } + } + +} diff --git a/Sources/Tetra/Combine/ExperimentalMapTask.swift b/Sources/Tetra/Combine/ExperimentalMapTask.swift index 57ebfee..12818ab 100644 --- a/Sources/Tetra/Combine/ExperimentalMapTask.swift +++ b/Sources/Tetra/Combine/ExperimentalMapTask.swift @@ -54,7 +54,7 @@ extension MultiMapTask { struct TaskState where S.Failure == Failure, S.Input == Output { var demand = PendingDemandState() var subscriber:S? = nil - var upstreamSubscription = SubscriptionContinuation.waiting + var upstreamSubscription = AsyncSubscriptionState.waiting var condition = TaskValueContinuation.waiting } @@ -62,7 +62,6 @@ extension MultiMapTask { private let maxTasks:Subscribers.Demand private let valueSource = AsyncStream>.makeStream() - private let demandSource = AsyncStream.makeStream() private let state: some UnfairStateLock> = createUncheckedStateLock(uncheckedState: TaskState()) private let transform:@Sendable (Upstream.Output) async throws(Failure) -> Output @@ -81,17 +80,8 @@ extension MultiMapTask { } private func localTask( - subscription: any Subscription, group: inout some CompatThrowingDiscardingTaskGroup ) async { - group.addTask(priority: nil) { - for await demand in demandSource.stream { - let nextDemand = receive(demand: demand) - if nextDemand > .none { - subscription.request(nextDemand) - } - } - } var iterator = valueSource.stream.makeAsyncIterator() while let upstreamValue = await iterator.next() { switch upstreamValue { @@ -106,13 +96,7 @@ extension MultiMapTask { send(completion: .failure(error)) throw CancellationError() case .success(let success): - if let demand = send(success) { - if demand > .none { - subscription.request(demand) - } - } else { - throw CancellationError() - } + try send(success) } } if !flag { @@ -123,47 +107,50 @@ extension MultiMapTask { } private func terminateStream() { - demandSource.continuation.finish() valueSource.continuation.finish() } private func send(completion: Subscribers.Completion?) { - let subscriber = state.withLockUnchecked{ + let (subscriber, effect) = state.withLockUnchecked{ let old = $0.subscriber + let effect = if completion != nil { + $0.upstreamSubscription.transition(.finish) + } else { + $0.upstreamSubscription.transition(.cancel) + } $0.subscriber = nil - return old + return (old, effect) } + effect?.run() if let completion { subscriber?.receive(completion: completion) } } - private func send(_ value: S.Input) -> Subscribers.Demand? { - let newDemand = state.withLockUnchecked{ - $0.subscriber - }?.receive(value) - guard let newDemand else { return nil } - - if maxTasks == .unlimited { - return newDemand + private func send(_ value: S.Input) throws { + let (subscriber, subscription) = state.withLockUnchecked{ + + return ($0.subscriber, $0.upstreamSubscription.subscription) } - return state.withLock{ - $0.demand.transistion(maxTasks: maxTasks, newDemand, reduce: true) + guard let subscriber, let subscription else { + throw CancellationError() } - } - - private func receive(demand:Subscribers.Demand) -> Subscribers.Demand { - if maxTasks == .unlimited { - return demand + var demand = subscriber.receive(value) + guard maxTasks != .unlimited else { + subscription.request(demand) + return + } + demand = state.withLockUnchecked{ + $0.demand.transistion(maxTasks: maxTasks, demand, reduce: true) } - return state.withLock{ - $0.demand.transistion(maxTasks: maxTasks, demand, reduce: false) + if demand > .none { + subscription.request(demand) } } - private func waitForUpStream() async -> (any Subscription)? { - await withTaskCancellationHandler { - await withUnsafeContinuation { coninuation in + private func waitForUpStream() async throws { + try await withTaskCancellationHandler { + try await withUnsafeThrowingContinuation { coninuation in state.withLockUnchecked{ $0.upstreamSubscription.transition(.suspend(coninuation)) }?.run() @@ -206,11 +193,11 @@ extension MultiMapTask { defer { clearCondition() } - let subscription = await waitForUpStream() + let success:Void? = try? await waitForUpStream() state.withLockUnchecked{ $0.subscriber }?.receive(subscription: self) - guard let subscription else { + guard success != nil else { terminateStream() return } @@ -219,7 +206,6 @@ extension MultiMapTask { try? await withThrowingDiscardingTaskGroup(returning: Void.self) { group in defer { terminateStream() } await localTask( - subscription: subscription, group: &group ) } @@ -241,7 +227,6 @@ extension MultiMapTask { } }() await localTask( - subscription: subscription, group: &group ) try await subTask @@ -249,7 +234,6 @@ extension MultiMapTask { } send(completion: .finished) } onCancel: { - subscription.cancel() send(completion: nil) } } @@ -291,7 +275,14 @@ extension MultiMapTask.Inner: Subscriber { extension MultiMapTask.Inner: Subscription { func request(_ demand: Subscribers.Demand) { - demandSource.continuation.yield(demand) + let (subscription, nextDemand) = state.withLock{ + let subscription = $0.upstreamSubscription.subscription + let demand = $0.demand.transistion(maxTasks: maxTasks, demand, reduce: false) + return (subscription, demand) + } + if let subscription, nextDemand > .none { + subscription.request(nextDemand) + } } func cancel() { diff --git a/Sources/Tetra/Combine/Publishers+MapTask.swift b/Sources/Tetra/Combine/Publishers+MapTask.swift index ded201a..b3cbbae 100644 --- a/Sources/Tetra/Combine/Publishers+MapTask.swift +++ b/Sources/Tetra/Combine/Publishers+MapTask.swift @@ -70,14 +70,15 @@ extension MapTask { struct TaskState where S.Failure == Failure, S.Input == Output { var subscriber:S? = nil - var upstreamSubscription = SubscriptionContinuation.waiting + var upstreamSubscription = AsyncSubscriptionState.waiting var condition = TaskValueContinuation.waiting + var isSleeping = true + var pending = Subscribers.Demand.none } struct Inner: CustomCombineIdentifierConvertible, Sendable where S.Failure == Failure, S.Input == Output { private let valueSource = AsyncStream>.makeStream(bufferingPolicy: .bufferingNewest(2)) - private let demandSource = AsyncStream.makeStream() private let state: some UnfairStateLock> = createUncheckedStateLock(uncheckedState: .init()) private let transform:@Sendable (Upstream.Output) async -> Result let combineIdentifier = CombineIdentifier() @@ -91,31 +92,42 @@ extension MapTask { } private func send(completion: Subscribers.Completion?) { - let subscriber = state.withLockUnchecked{ + let (subscriber, effect) = state.withLockUnchecked{ let old = $0.subscriber $0.subscriber = nil - return old + let effect = if completion == nil { + $0.upstreamSubscription.transition(.cancel) + } else { + $0.upstreamSubscription.transition(.finish) + } + return (old, effect) } + effect?.run() if let completion { subscriber?.receive(completion: completion) } } - private func send(_ value:Output) -> Subscribers.Demand? { - state.withLockUnchecked{ - $0.subscriber - }?.receive(value) + private func send(_ value:Output) throws { + let subscriber = state.withLockUnchecked{ + $0.isSleeping = true + return $0.subscriber + } + guard let subscriber else { + throw CancellationError() + } + let demand = subscriber.receive(value) + request(demand) } private func terminateStream() { - demandSource.continuation.finish() valueSource.continuation.finish() } - private func waitForUpStream() async -> (any Subscription)? { - await withTaskCancellationHandler { - await withUnsafeContinuation { coninuation in - state.withLock{ + private func waitForUpStream() async throws { + try await withTaskCancellationHandler { + try await withUnsafeThrowingContinuation { coninuation in + state.withLockUnchecked{ $0.upstreamSubscription.transition(.suspend(coninuation)) }?.run() } @@ -157,12 +169,12 @@ extension MapTask { defer { clearCondition() } - let subscription = await waitForUpStream() + let success: Void? = try? await waitForUpStream() defer { terminateStream() } state.withLockUnchecked{ $0.subscriber }?.receive(subscription: self) - guard let subscription else { + guard success != nil else { return } let stream = valueSource.stream.map{ [transform] in @@ -173,31 +185,18 @@ extension MapTask { return .failure(failure) } } - await withTaskCancellationHandler { - var iterator = stream.makeAsyncIterator() - for await var demand in demandSource.stream { - while demand > .none { - demand -= 1 - subscription.request(.max(1)) - switch (await iterator.next()) { - case .success(let value): - if let newDemand = send(value) { - demand += newDemand - } else { - return - } - case .failure(let error): - send(completion: .failure(error)) - return - case .none: - send(completion: .finished) - return - } + try? await withTaskCancellationHandler { + for await result in stream { + switch result { + case .success(let value): + try send(value) + case .failure(let error): + send(completion: .failure(error)) + throw CancellationError() } - } + send(completion: .finished) } onCancel: { - subscription.cancel() send(completion: nil) } @@ -252,7 +251,18 @@ extension MapTask.Inner: Subscription { } func request(_ demand: Subscribers.Demand) { - demandSource.continuation.yield(demand) + let subscription = state.withLockUnchecked { + $0.pending += demand + if $0.isSleeping && $0.pending > .none { + $0.isSleeping = false + $0.pending -= 1 + return $0.upstreamSubscription.subscription + } else { + + return nil + } + } + subscription?.request(.max(1)) } diff --git a/Sources/Tetra/Combine/Publishers+TryMapTask.swift b/Sources/Tetra/Combine/Publishers+TryMapTask.swift index f2d8562..d4e6bba 100644 --- a/Sources/Tetra/Combine/Publishers+TryMapTask.swift +++ b/Sources/Tetra/Combine/Publishers+TryMapTask.swift @@ -60,14 +60,15 @@ extension TryMapTask { internal struct TaskState where S.Failure == Failure, S.Input == Output { var subscriber:S? = nil - var upstreamSubscription = SubscriptionContinuation.waiting + var upstreamSubscription = AsyncSubscriptionState.waiting var condition = TaskValueContinuation.waiting + var isSleeping = true + var pending = Subscribers.Demand.none } internal struct Inner: CustomCombineIdentifierConvertible where S.Failure == Failure, S.Input == Output { private let valueSource = AsyncThrowingStream.makeStream(bufferingPolicy: .bufferingNewest(2)) - private let demandSource = AsyncStream.makeStream() private let state: some UnfairStateLock> = createUncheckedStateLock(uncheckedState: .init()) private let transform:@Sendable (Upstream.Output) async throws -> Output let combineIdentifier = CombineIdentifier() @@ -92,20 +93,25 @@ extension TryMapTask { } } - private func send(_ value:Output) -> Subscribers.Demand? { - state.withLockUnchecked{ - $0.subscriber - }?.receive(value) + private func send(_ value:Output) throws { + let subscriber = state.withLockUnchecked{ + $0.isSleeping = true + return $0.subscriber + } + guard let subscriber else { + throw CancellationError() + } + let demand = subscriber.receive(value) + request(demand) } private func terminateStream() { - demandSource.continuation.finish() valueSource.continuation.finish() } - private func waitForUpStream() async -> (any Subscription)? { - await withTaskCancellationHandler { - await withUnsafeContinuation { coninuation in + private func waitForUpStream() async throws { + try await withTaskCancellationHandler { + try await withUnsafeThrowingContinuation { coninuation in state.withLockUnchecked { $0.upstreamSubscription.transition(.suspend(coninuation)) }?.run() @@ -148,39 +154,28 @@ extension TryMapTask { defer { clearCondition() } - let subscription = await waitForUpStream() + let subscription: Void? = try? await waitForUpStream() state.withLockUnchecked{ $0.subscriber }?.receive(subscription: self) defer { terminateStream() } - guard let subscription else { + guard subscription != nil else { return } let stream = valueSource.stream.map(transform) await withTaskCancellationHandler { - var iterator = stream.makeAsyncIterator() - for await var demand in demandSource.stream { - while demand > .none { - demand -= 1 - subscription.request(.max(1)) - do { - guard let value = try await iterator.next() else { - send(completion: .finished) - return - } - guard let newDemand = send(value) else { - return - } - demand += newDemand - } catch { - send(completion: .failure(error)) + do { + for try await value in stream { + guard let _ = try? send(value) else { return } } - + send(completion: .finished) + } catch { + send(completion: .failure(error)) + return } } onCancel: { - subscription.cancel() send(completion: nil) } @@ -200,7 +195,18 @@ extension TryMapTask.Inner: Subscription { } func request(_ demand: Subscribers.Demand) { - demandSource.continuation.yield(demand) + let subscription = state.withLockUnchecked { + $0.pending += demand + if $0.isSleeping && $0.pending > .none { + $0.isSleeping = false + $0.pending -= 1 + return $0.upstreamSubscription.subscription + } else { + + return nil + } + } + subscription?.request(.max(1)) } } diff --git a/Tests/TetraTests/MapTaskTests.swift b/Tests/TetraTests/MapTaskTests.swift index f4f9996..9963977 100644 --- a/Tests/TetraTests/MapTaskTests.swift +++ b/Tests/TetraTests/MapTaskTests.swift @@ -118,8 +118,8 @@ final class MapTaskTests: XCTestCase { } .handleEvents( receiveSubscription: { subscription in - warmup.fulfill() XCTAssertEqual("\(subscription)", "MapTask") + warmup.fulfill() }, receiveOutput: { value in outputHandle(value) diff --git a/Tests/TetraTests/MultiMapTaskTests.swift b/Tests/TetraTests/MultiMapTaskTests.swift index 6eaad6a..8443123 100644 --- a/Tests/TetraTests/MultiMapTaskTests.swift +++ b/Tests/TetraTests/MultiMapTaskTests.swift @@ -71,12 +71,14 @@ final class MultiMapTaskTests: XCTestCase { let target = try XCTUnwrap(sequence.randomElement()) let upstream = sequence.publisher.setFailureType(to: CancellationError.self) let expect = expectation(description: "task failure") - let pub = MultiMapTask(maxTasks: .max(1), upstream: upstream) { value in - if value == target { - throw CancellationError() + let block:@Sendable (Int) async throws(CancellationError) -> Int = { + if $0 == target { + try Result.failure(CancellationError()).get() } - return value + await Task.yield() + return $0 } + let pub = MultiMapTask(maxTasks: .max(1), upstream: upstream, transform: block) var bag = Set() pub.sink { completion in switch completion { @@ -121,7 +123,7 @@ final class MultiMapTaskTests: XCTestCase { ) .multiMapTask(maxTasks: .max(3)) { await Task.yield() - return .success($0) + return $0 }.subscribe(subscriber) wait(for: [warmup]) let subscription = try XCTUnwrap(_subscription) @@ -167,7 +169,7 @@ final class MultiMapTaskTests: XCTestCase { ) .multiMapTask(maxTasks: .unlimited) { try? await Task.sleep(nanoseconds: 1_000) - return .success($0) + return $0 } .handleEvents( receiveSubscription: { _ in From ee8468caf1c8db3eeaa30af6bc3adbfba958519a Mon Sep 17 00:00:00 2001 From: park-byeong-gwan Date: Fri, 7 Jun 2024 11:13:01 +0900 Subject: [PATCH 09/63] Combine & Concurrency: introduce asyncflatmap --- Package.swift | 5 +- .../Combine/AsyncFlatMapDemandState.swift | 105 ++++++ .../Tetra/Combine/Combine+Concurrency.swift | 8 + .../Combine/Publishers+AsyncFlatMap.swift | 342 ++++++++++++++++++ Tests/TetraTests/AsyncFlatMapTests.swift | 273 ++++++++++++++ 5 files changed, 732 insertions(+), 1 deletion(-) create mode 100644 Sources/Tetra/Combine/AsyncFlatMapDemandState.swift create mode 100644 Sources/Tetra/Combine/Publishers+AsyncFlatMap.swift create mode 100644 Tests/TetraTests/AsyncFlatMapTests.swift diff --git a/Package.swift b/Package.swift index b0623fd..4db774d 100644 --- a/Package.swift +++ b/Package.swift @@ -23,13 +23,16 @@ let package = Package( dependencies: [ // Dependencies declare other packages that this package depends on. // .package(url: /* package url */, from: "1.0.0"), + .package(url: "https://github.com/apple/swift-collections.git", .upToNextMajor(from: "1.1.0")), ], targets: [ // Targets are the basic building blocks of a package. A target can define a module or a test suite. // Targets can depend on other targets in this package, and on products in packages this package depends on. .target( name: "Tetra", - dependencies: [], + dependencies: [ + .product(name: "DequeModule", package: "swift-collections") + ], swiftSettings: [ .enableExperimentalFeature("StrictConcurrency=complete"), .enableUpcomingFeature("FullTypedThrows"), diff --git a/Sources/Tetra/Combine/AsyncFlatMapDemandState.swift b/Sources/Tetra/Combine/AsyncFlatMapDemandState.swift new file mode 100644 index 0000000..7560add --- /dev/null +++ b/Sources/Tetra/Combine/AsyncFlatMapDemandState.swift @@ -0,0 +1,105 @@ +// +// AsyncFlatMapDemandState.swift +// +// +// Created by 박병관 on 6/7/24. +// + +import Foundation +import Combine +import DequeModule + +struct AsyncFlatMapDemandState: Sendable { + + private var suspended:Deque = [] + private var pending = Subscribers.Demand.none + private var isInterrupted = false + typealias Element = UnsafeContinuation + + enum Event { + + case interrupt + case resume(Subscribers.Demand) + case suspend(Element) + } + + enum Effect { + + case resume(Deque, Bool) + case raise(Deque) + + func run() { + switch self { + case .resume(let array, let demand): + array.forEach{ + $0.resume(returning: demand) + } + case .raise(let array): + array.forEach{ + $0.resume(throwing: CancellationError()) + } + } + } + } + + mutating func transition(_ event:Event) -> Effect? { + switch event { + case .interrupt: + return interrupt() + case .resume(let demand): + return resume(demand) + case .suspend(let unsafeContinuation): + return suspend(unsafeContinuation) + } + } + + private mutating func resume(_ demand:Subscribers.Demand) -> Effect? { + if isInterrupted { + return nil + } + pending += demand + return populateResume() + } + + private mutating func suspend(_ continuation: Element) -> Effect? { + if isInterrupted { + return .raise([continuation]) + } + suspended.append(continuation) + + return populateResume() + } + + + private mutating func populateResume() -> Effect? { + // unlimited + guard let max = pending.max else { + let jobs = suspended + // zero capacity storage creation + suspended = [] + return .resume(jobs, true) + } + if max == 0 || suspended.count == 0 { + return nil + } + var buffer:Deque = [] + var count = min(suspended.count, max) + pending -= count + buffer.reserveCapacity(count) + while let token = suspended.popFirst(), count > 0 { + count -= 1 + buffer.append(token) + } + return .resume(buffer, false) + + } + + private mutating func interrupt() -> Effect? { + pending = .none + isInterrupted = true + let jobs = suspended + suspended = [] + return .raise(jobs) + } + +} diff --git a/Sources/Tetra/Combine/Combine+Concurrency.swift b/Sources/Tetra/Combine/Combine+Concurrency.swift index 9767046..af40c9b 100644 --- a/Sources/Tetra/Combine/Combine+Concurrency.swift +++ b/Sources/Tetra/Combine/Combine+Concurrency.swift @@ -47,4 +47,12 @@ public extension Publisher { MultiMapTask(maxTasks: maxTasks, upstream: self, transform: transform) } + internal + func asyncFlatMap( + maxTasks: Subscribers.Demand = .unlimited, + transform: @escaping @Sendable (Output) async throws -> Segment + ) -> AsyncFlatMap where Failure == any Error { + return .init(maxTasks: maxTasks, upstream: self, transform: transform) + } + } diff --git a/Sources/Tetra/Combine/Publishers+AsyncFlatMap.swift b/Sources/Tetra/Combine/Publishers+AsyncFlatMap.swift new file mode 100644 index 0000000..4fa3c6a --- /dev/null +++ b/Sources/Tetra/Combine/Publishers+AsyncFlatMap.swift @@ -0,0 +1,342 @@ +// +// File.swift +// +// +// Created by 박병관 on 6/7/24. +// + +import Foundation +@preconcurrency import Combine + +struct AsyncFlatMap: Publisher where Upstream.Output:Sendable, Upstream.Failure == any Error { + + typealias Output = Segment.Element + typealias Failure = any Error + typealias Transform = @Sendable (Upstream.Output) async throws(Failure) -> Segment + let maxTasks:Subscribers.Demand + let upstream:Upstream + let transform:Transform + + func receive(subscriber: S) where S : Subscriber, Failure == S.Failure, Segment.Element == S.Input { + let processor = Inner(maxTasks: maxTasks, subscriber: subscriber, transform: transform) + let task = Task(operation: processor.run) + processor.resumeCondition(task) + upstream.subscribe(processor) + } + + init( + maxTasks: Subscribers.Demand, + upstream: Upstream, + transform: @escaping @isolated(any) Transform + ) { + self.maxTasks = maxTasks + self.upstream = upstream + self.transform = transform + } + +} + +extension AsyncFlatMap { + + struct Inner: Subscriber, Sendable, Subscription, CustomStringConvertible, CustomPlaygroundDisplayConvertible + where Segment.Element == Down.Input, Down.Failure == Failure { + + typealias Transformer = @Sendable (Upstream.Output) async throws(Failure) -> Segment + typealias Input = Upstream.Output + typealias Failure = any Error + + + let lock:some UnfairStateLock = createUncheckedStateLock(uncheckedState: TaskState()) + + struct TaskState { + var demandState = AsyncFlatMapDemandState() + var subscriber:Down? = nil + var upstreamSubscription = AsyncSubscriptionState.waiting + var taskCondition = TaskValueContinuation.waiting + } + + let maxTasks:Subscribers.Demand + let transform:Transformer + let valueSource = AsyncStream>.makeStream() + let combineIdentifier = CombineIdentifier() + + + init(maxTasks:Subscribers.Demand ,subscriber: Down,transform: @escaping Transformer) { + self.transform = transform + self.maxTasks = maxTasks + lock.withLockUnchecked{ + $0.subscriber = subscriber + } + } + + @Sendable + func run() async { + let token:Void? = try? await waitForCondition() + if token == nil { + withUnsafeCurrentTask{ + $0?.cancel() + } + } + defer { + clearCondition() + } + let success:Void? = try? await waitForUpStream() + lock.withLockUnchecked{ + $0.subscriber + }?.receive(subscription: self) + guard success != nil else { + terminateStream() + return + } + await withTaskCancellationHandler { + let isCancelled:Bool + if #available(iOS 17.0, tvOS 17.0, macCatalyst 17.0, macOS 14.0, watchOS 10.0, visionOS 1.0, *) { + let void:Void? = try? await withThrowingDiscardingTaskGroup { group in + defer { terminateStream() } + try await localTask(group: &group) + } + + isCancelled = void == nil + } else { + let void:Void? = try? await withThrowingTaskGroup(of: Void.self) { group in + defer { terminateStream() } + let unsafe = SuppressSendable(wrapped: group.makeAsyncIterator()) + async let subTask:() = { + var iterator = unsafe.wrapped + while let _ = try await iterator.next() { } + }() + try await localTask(group: &group) + try await subTask + } + isCancelled = void == nil + } + if !isCancelled { + send(completion: .finished) + } + } onCancel: { + send(completion: nil) + } + } + + var playgroundDescription: Any { description } + + var description: String { "AsyncFlatMap" } + + + func receive(_ input: sending Input) -> Subscribers.Demand { + valueSource.continuation.yield(.success(input)) + return .none + } + + func receive(completion: Subscribers.Completion) { + switch completion { + case .finished: + break + case .failure(let failure): + valueSource.continuation.yield(.failure(failure)) + } + valueSource.continuation.finish() + } + + func receive(subscription: any Subscription) { + let (effect, requestValue) = lock.withLockUnchecked{ + let effect = $0.upstreamSubscription.transition(.resume(subscription)) + let requestValue = subscription.combineIdentifier == $0.upstreamSubscription.subscription?.combineIdentifier + return (effect, requestValue) + } + effect?.run() + if requestValue && maxTasks > .none { + subscription.request(maxTasks) + } + } + + func request(_ demand: Subscribers.Demand) { + lock.withLockUnchecked{ + $0.demandState.transition(.resume(demand)) + }?.run() + } + + func cancel() { + lock.withLockUnchecked{ + $0.taskCondition.transition(.cancel) + }?.run() + } + + private func send(completion: Subscribers.Completion?) { + let (subscriber, effect, interruption) = lock.withLockUnchecked{ + let old = $0.subscriber + $0.subscriber = nil + let effect = if completion == nil { + $0.upstreamSubscription.transition(.cancel) + } else { + $0.upstreamSubscription.transition(.finish) + } + let interruption = $0.demandState.transition(.interrupt) + return (old, effect, interruption) + } + if let completion { + subscriber?.receive(completion: completion) + } + effect?.run() + interruption?.run() + } + + + private func send(_ value:Down.Input) throws(CancellationError) { + let subscriber = lock.withLockUnchecked { + $0.subscriber + } + guard let newDemand = subscriber?.receive(value) else { + throw CancellationError() + } + lock.withLockUnchecked{ + $0.demandState.transition(.resume(newDemand)) + }?.run() + } + + private func waitForUpStream() async throws { + try await withTaskCancellationHandler { + try await withUnsafeThrowingContinuation { coninuation in + lock.withLockUnchecked{ + $0.upstreamSubscription.transition(.suspend(coninuation)) + }?.run() + } + } onCancel: { + lock.withLockUnchecked{ + $0.upstreamSubscription.transition(.cancel) + }?.run() + } + } + + func resumeCondition(_ task:Task) { + lock.withLock{ + $0.taskCondition.transition(.resume(task)) + }?.run() + } + + private func waitForCondition() async throws { + try await withUnsafeThrowingContinuation{ continuation in + lock.withLock{ + $0.taskCondition.transition(.suspend(continuation)) + }?.run() + } + } + + private func clearCondition() { + lock.withLock{ + $0.taskCondition.transition(.finish) + }?.run() + } + + private func terminateStream() { + valueSource.continuation.finish() + } + + private func makeSegment(_ input:Upstream.Output) async throws(CancellationError) -> Segment { + let result:Result + do { + let seg = try await transform(input) + result = .success(seg) + } catch { + result = .failure(error ) + } + switch result { + case .success(let success): + return success + case .failure(let failure): + send(completion: .failure(failure)) + throw CancellationError() + } + } + + /// whether demand is unlimited + /// - Returns: `true` if demand is unlimited, `false` if demand is just `1`. + /// - throws: `CancellationError` if internal state reached cancellation + private func nextDemand() async throws -> Bool { + try await withUnsafeThrowingContinuation { continuation in + lock.withLockUnchecked{ + $0.demandState.transition(.suspend(continuation)) + }?.run() + } + } + + + /// process next segment and send event to downstream + /// - Returns: `false` if iterator reached termination otherwise `true` + /// - throws: `CancellationError` if internal state reached cancellation + private func processNextSegment( + iterator: inout Segment.AsyncIterator + ) async throws(CancellationError) -> Bool { + let nextResult:Result? + do { + let value = try await iterator.next() + if let value { + nextResult = .success(value) + } else { + nextResult = nil + } + } catch { + nextResult = .failure(error) + } + switch nextResult { + case .none: + //finished + // check and request more transformer + // request one more from upstream subscription + if maxTasks != .unlimited { + let subscription = lock.withLockUnchecked{ + $0.upstreamSubscription.subscription + } + if let subscription { + subscription.request(.max(1)) + } else { + throw CancellationError() + } + } + return false + case .failure(let error): + send(completion: .failure(error)) + throw CancellationError() + case .success(let value): + try send(value) + } + return true + } + + private func localTask( + group: inout some CompatThrowingDiscardingTaskGroup + ) async throws { + for await result in valueSource.stream { + switch result { + case .failure(let failure): + send(completion: .failure(failure)) + throw CancellationError() + case .success(let value): + let isSuccess = group.addTaskUnlessCancelled(priority: nil) { + let segment = try await makeSegment(value) + var iterator = segment.makeAsyncIterator() + while true { + let isUnlimited = try await nextDemand() + if isUnlimited { + while try await processNextSegment(iterator: &iterator) { + } + return + } else { + let hasNext = try await processNextSegment(iterator: &iterator) + if !hasNext { + return + } + } + } + } + if !isSuccess { + throw CancellationError() + } + + } + } + } + + } + +} diff --git a/Tests/TetraTests/AsyncFlatMapTests.swift b/Tests/TetraTests/AsyncFlatMapTests.swift new file mode 100644 index 0000000..cd44ad1 --- /dev/null +++ b/Tests/TetraTests/AsyncFlatMapTests.swift @@ -0,0 +1,273 @@ +// +// AsyncFlatMapTests.swift +// +// +// Created by 박병관 on 6/7/24. +// + +import XCTest +import Combine +import AsyncAlgorithms +@testable import Tetra + +final class AsyncFlatMapTests: XCTestCase { + + + func testOrderd() throws { + let completion = expectation(description: "complete") + var array = [Int]() + let sample = Array((0..<5)) + let bag = [0,1].publisher + .handleEvents( + receiveRequest: { + XCTAssertEqual($0, .max(1)) + } + ) + .setFailureType(to: Error.self) + .asyncFlatMap(maxTasks: .max(1)) { value in + AsyncStream{ continuation in + sample.forEach{ + continuation.yield($0) + } + continuation.finish() + }.map{ + await Task.yield() + return $0 + } + }.handleEvents( + receiveSubscription: { + XCTAssertEqual("\($0)", "AsyncFlatMap") + } + ) + .sink { _ in + completion.fulfill() + } receiveValue: { + array.append($0) + } + wait(for: [completion]) + bag.cancel() + XCTAssertEqual(array, sample + sample) + } + + func testUnOrderd() throws { + let completion = expectation(description: "complete") + var array = [Int]() + let sample = Array((0..<5)) + let lock = NSRecursiveLock() + let bag = [0,1].publisher + .handleEvents( + receiveRequest: { + XCTAssertEqual($0, .max(2)) + } + ) + .setFailureType(to: Error.self) + .asyncFlatMap(maxTasks: .max(2)) { value in + return AsyncStream{ continuation in + sample.forEach{ + continuation.yield($0 + value * 10) + } + continuation.finish() + }.map{ + await Task.yield() + return $0 + } + }.handleEvents( + receiveSubscription: { + XCTAssertEqual("\($0)", "AsyncFlatMap") + } + ) + .sink { _ in + completion.fulfill() + } receiveValue: { value in + lock.withLock{ + array.append(value) + } + + } + wait(for: [completion]) + bag.cancel() + XCTAssertEqual(Set(array), [0,1,2,3,4,10,11,12,13,14]) + } + + func testCancelInTranform() throws { + let holder = UnsafeCancellableHolder() + let completion = expectation(description: "cancellation") + let lock = NSRecursiveLock() + lock.withLock { + (0..<5).publisher + .setFailureType(to: Error.self) + .asyncFlatMap(maxTasks: .unlimited) { value in + lock.withLock{ + holder.bag = [] + } + return AsyncStream{ + $0.yield(value) + $0.finish() + } + }.handleEvents( + receiveCancel: { + completion.fulfill() + } + ).sink { _ in + XCTFail("should not reach here") + } receiveValue: { _ in + XCTFail("should not reach here") + }.store(in: &holder.bag) + } + wait(for: [completion], timeout: 0.2) + } + + func testCancelInSegment() throws { + let holder = UnsafeCancellableHolder() + let completion = expectation(description: "cancellation") + let lock = NSRecursiveLock() + lock.withLock { + (0..<5).publisher + .setFailureType(to: Error.self) + .asyncFlatMap(maxTasks: .unlimited) { value in + return AsyncStream{ + $0.yield(value) + $0.finish() + }.map{ + lock.withLock{ + holder.bag = [] + } + return $0 + } + }.handleEvents( + receiveCancel: { + completion.fulfill() + } + ).sink { _ in + XCTFail("should not reach here") + } receiveValue: { _ in + XCTFail("should not reach here") + }.store(in: &holder.bag) + } + wait(for: [completion], timeout: 0.2) + } + + func testCancelInSubscriber() throws { + let holder = UnsafeCancellableHolder() + let completion = expectation(description: "cancellation") + let lock = NSRecursiveLock() + lock.withLock { + (0..<5).publisher + .setFailureType(to: Error.self) + .asyncFlatMap(maxTasks: .unlimited) { value in + return AsyncStream{ + $0.yield(value) + $0.finish() + } + }.handleEvents( + receiveCancel: { + completion.fulfill() + } + ).sink { _ in + XCTFail("should not reach here") + } receiveValue: { _ in + lock.withLock{ + holder.bag = [] + } + }.store(in: &holder.bag) + } + wait(for: [completion], timeout: 0.2) + } + + + func testThrowInTransformer() throws { + let holder = UnsafeCancellableHolder() + let completion = expectation(description: "cancellation") + (0..<5).publisher + .setFailureType(to: Error.self) + .asyncFlatMap(maxTasks: .unlimited) { value in + if value == 3 { + throw CancellationError() + } + return AsyncStream{ + $0.yield(value) + $0.finish() + } + }.sink { + switch $0 { + case .finished: + break + case .failure(let error): + completion.fulfill() + XCTAssertTrue(error is CancellationError) + } + } receiveValue: { + XCTAssertLessThan($0, 3) + }.store(in: &holder.bag) + wait(for: [completion], timeout: 0.2) + } + + func testThrowInSegment() throws { + let holder = UnsafeCancellableHolder() + let completion = expectation(description: "cancellation") + (0..<5).publisher + .setFailureType(to: Error.self) + .asyncFlatMap(maxTasks: .unlimited) { value in + + return AsyncStream{ + $0.yield(value) + $0.finish() + }.map{ + if $0 == 3 { + throw CancellationError() + } + return $0 + } + }.sink { + switch $0 { + case .finished: + break + case .failure(let error): + completion.fulfill() + XCTAssertTrue(error is CancellationError) + } + } receiveValue: { + XCTAssertLessThan($0, 3) + }.store(in: &holder.bag) + wait(for: [completion], timeout: 0.2) + } + + func testTransformUnsafeCancel() throws { + // If Task cancellation occur in transformer, the returned Segment is responsible to handle the cancellation, + // created Segment and transformer runs in the same task + // UnsafeCurrentTask cancellation has no effect on root task + let holder = UnsafeCancellableHolder() + let completion = expectation(description: "complete") + var buffer = [Int]() + (0..<5).publisher + .setFailureType(to: Error.self) + .asyncFlatMap(maxTasks: .max(1)) { value in + if value == 0 { + withUnsafeCurrentTask{ + $0?.cancel() + } + } + let transformTask = withUnsafeCurrentTask{ $0 }?.hashValue + + return AsyncStream{ + await Task.yield() + withUnsafeCurrentTask { + XCTAssertEqual(transformTask, $0?.hashValue) + } + if Task.isCancelled { + return nil + } + withUnsafeCurrentTask{$0?.cancel()} + return value + } + }.sink { _ in + completion.fulfill() + } receiveValue: { + buffer.append($0) + }.store(in: &holder.bag) + wait(for: [completion], timeout: 0.2) + XCTAssertEqual(buffer, [1,2,3,4]) + } + + +} From 686a8923172ec253e3ee975a74cc79395b016547 Mon Sep 17 00:00:00 2001 From: park-byeong-gwan Date: Sat, 8 Jun 2024 00:31:03 +0900 Subject: [PATCH 10/63] fix test fail --- Tests/TetraTests/AsyncFlatMapTests.swift | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Tests/TetraTests/AsyncFlatMapTests.swift b/Tests/TetraTests/AsyncFlatMapTests.swift index cd44ad1..442e333 100644 --- a/Tests/TetraTests/AsyncFlatMapTests.swift +++ b/Tests/TetraTests/AsyncFlatMapTests.swift @@ -180,7 +180,7 @@ final class AsyncFlatMapTests: XCTestCase { let completion = expectation(description: "cancellation") (0..<5).publisher .setFailureType(to: Error.self) - .asyncFlatMap(maxTasks: .unlimited) { value in + .asyncFlatMap(maxTasks: .max(1)) { value in if value == 3 { throw CancellationError() } @@ -207,7 +207,7 @@ final class AsyncFlatMapTests: XCTestCase { let completion = expectation(description: "cancellation") (0..<5).publisher .setFailureType(to: Error.self) - .asyncFlatMap(maxTasks: .unlimited) { value in + .asyncFlatMap(maxTasks: .max(1)) { value in return AsyncStream{ $0.yield(value) From 7fbe6ff95de4c6381324c70e22281468df15fb5f Mon Sep 17 00:00:00 2001 From: park-byeong-gwan Date: Sat, 8 Jun 2024 00:32:35 +0900 Subject: [PATCH 11/63] change implementation to fix rare crash in test --- .../Tetra/Combine/ExperimentalMapTask.swift | 4 +- .../URLSessionDownloadTask+Concurrency.swift | 74 +++++++++---------- 2 files changed, 40 insertions(+), 38 deletions(-) diff --git a/Sources/Tetra/Combine/ExperimentalMapTask.swift b/Sources/Tetra/Combine/ExperimentalMapTask.swift index 12818ab..d2e33f7 100644 --- a/Sources/Tetra/Combine/ExperimentalMapTask.swift +++ b/Sources/Tetra/Combine/ExperimentalMapTask.swift @@ -137,7 +137,9 @@ extension MultiMapTask { } var demand = subscriber.receive(value) guard maxTasks != .unlimited else { - subscription.request(demand) + if demand > .none { + subscription.request(demand) + } return } demand = state.withLockUnchecked{ diff --git a/Sources/Tetra/Concurrency/URLSessionDownloadTask+Concurrency.swift b/Sources/Tetra/Concurrency/URLSessionDownloadTask+Concurrency.swift index 5e9b8ae..006c7b7 100644 --- a/Sources/Tetra/Concurrency/URLSessionDownloadTask+Concurrency.swift +++ b/Sources/Tetra/Concurrency/URLSessionDownloadTask+Concurrency.swift @@ -45,8 +45,8 @@ internal func download_transformer( /// - transform: success completion handler가 리턴직전에 호출되는 callback (at most 1) /// - creator: completion handler를 인자로 받아서 URLSessionTask를 생성하는 함수(exactly 1) @usableFromInline -internal func urltask_transformer( - transform: @Sendable (Value) -> Result, +internal func urltask_transformer( + transform: @escaping @Sendable (Value) -> Result, creator:( @escaping @Sendable (Value?, URLResponse?, Error?) -> Void ) -> T @@ -54,46 +54,46 @@ internal func urltask_transformer( let stateLock = createCheckedStateLock(checkedState: URLSessionTaskAsyncState.waiting) return try await withTaskCancellationHandler { - try await withoutActuallyEscaping(transform) { escapingTransform in - try await withUnsafeThrowingContinuation { continuation in - let sessionTask = creator() { data, response, error in - do { - guard let data, let response else { - throw error ?? URLError(.unknown, userInfo: [ - NSLocalizedDescriptionKey: NSLocalizedString("Err-998", bundle: .init(for: URLSession.self), comment: "unknown error") - ]) - } - let value = try escapingTransform(data).get() - continuation.resume(returning: (value, response)) - } catch { - continuation.resume(throwing: error) + + try await withUnsafeThrowingContinuation { continuation in + let sessionTask = creator() { data, response, error in + do { + guard let data, let response else { + throw error ?? URLError(.unknown, userInfo: [ + NSLocalizedDescriptionKey: NSLocalizedString("Err-998", bundle: .init(for: URLSession.self), comment: "unknown error") + ]) } + let value = try transform(data).get() + continuation.resume(returning: (value, response)) + } catch { + continuation.resume(throwing: error) } - sessionTask.resume() - - let snapShot = stateLock.withLock{ - let oldValue = $0 - switch oldValue { - case .cancelled: - break - case .task: - assertionFailure("unexpected state") - fallthrough - case .waiting: - $0 = .task(sessionTask) - } - return oldValue - - } - switch snapShot { - case .waiting: - break - case .task(let uRLSessionTask): - uRLSessionTask.cancel() + } + sessionTask.resume() + + let snapShot = stateLock.withLock{ + let oldValue = $0 + switch oldValue { case .cancelled: - sessionTask.cancel() + break + case .task: + assertionFailure("unexpected state") + fallthrough + case .waiting: + $0 = .task(sessionTask) } + return oldValue + + } + switch snapShot { + case .waiting: + break + case .task(let uRLSessionTask): + uRLSessionTask.cancel() + case .cancelled: + sessionTask.cancel() } + } } onCancel: { stateLock.withLock{ From d15e723791cf90bb9523e1b410b153a8d4d5fc95 Mon Sep 17 00:00:00 2001 From: park-byeong-gwan Date: Sat, 8 Jun 2024 02:16:07 +0900 Subject: [PATCH 12/63] CoreData: interit Exeuctor attribute to make scheduledTaskType valid --- .../CoreDataStack+Concurrency.swift | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/Sources/Tetra/Concurrency/CoreDataStack+Concurrency.swift b/Sources/Tetra/Concurrency/CoreDataStack+Concurrency.swift index aa8765a..0d42fdd 100644 --- a/Sources/Tetra/Concurrency/CoreDataStack+Concurrency.swift +++ b/Sources/Tetra/Concurrency/CoreDataStack+Concurrency.swift @@ -125,17 +125,20 @@ extension TetraExtension where Base: NSManagedObjectContext { /// Asynchronously performs the specified closure on the context’s queue. @inlinable + @_unsafeInheritExecutor public func perform( + schedule:CoreDataScheduledTaskType = .immediate, _ body: () throws -> T ) async rethrows -> T { /* + Since this method and NSManagedObjectContext peform has no actor preference and isolation restriction. These two are always called on global nonisolated context (Actor switching happen). Which means that `immediate` execution option is totally no-op. - `_performImmediate:` is never called when using `NSManagedObjectContext.perform(schedule: .immediate)` in iOS 15 ~ iOS 17 - + - @_unsafeInheritExecutor do fix the above problem */ return if #available(iOS 15.0, tvOS 15.0, macCatalyst 15.0, watchOS 8.0, macOS 12.0, *) { try await withoutActuallyEscaping(body) { @@ -143,12 +146,18 @@ extension TetraExtension where Base: NSManagedObjectContext { defer { withExtendedLifetime(block, {}) } - return try await base.perform(schedule: .enqueued) { [unowned block] in + return try await base.perform(schedule: schedule.platformValue) { [unowned block] in return try block() } } - } else { + } else if schedule == .enqueued { try await _performEnqueue(body) + } else { + if let result = try _performImmediate(body) { + result.get() + } else { + try await _performEnqueue(body) + } } } @@ -223,8 +232,8 @@ extension TetraExtension where Base: NSPersistentContainer { } -@usableFromInline -internal enum CoreDataScheduledTaskType: Sendable, Hashable { + +public enum CoreDataScheduledTaskType: Sendable, Hashable { case immediate From 1841e58730ee177bf2883b20a01879ca7e7ca2df Mon Sep 17 00:00:00 2001 From: park-byeong-gwan Date: Sat, 8 Jun 2024 08:27:48 +0900 Subject: [PATCH 13/63] remove unused module import declaration --- Tests/TetraTests/AsyncFlatMapTests.swift | 1 - 1 file changed, 1 deletion(-) diff --git a/Tests/TetraTests/AsyncFlatMapTests.swift b/Tests/TetraTests/AsyncFlatMapTests.swift index 442e333..98b4536 100644 --- a/Tests/TetraTests/AsyncFlatMapTests.swift +++ b/Tests/TetraTests/AsyncFlatMapTests.swift @@ -7,7 +7,6 @@ import XCTest import Combine -import AsyncAlgorithms @testable import Tetra final class AsyncFlatMapTests: XCTestCase { From ab175577fcf8af82744d0c0b77408cf880a2d3e5 Mon Sep 17 00:00:00 2001 From: park-byeong-gwan Date: Sat, 8 Jun 2024 16:36:30 +0900 Subject: [PATCH 14/63] suppress sendable warning in dedicated case --- Sources/Tetra/Combine/ExperimentalMapTask.swift | 5 +++-- Sources/Tetra/Combine/Publishers+AsyncFlatMap.swift | 9 ++++++--- .../Concurrency/CoreDataStack+Concurrency.swift | 9 ++++++--- Sources/Tetra/Foundation/Mics.swift | 13 ------------- 4 files changed, 15 insertions(+), 21 deletions(-) diff --git a/Sources/Tetra/Combine/ExperimentalMapTask.swift b/Sources/Tetra/Combine/ExperimentalMapTask.swift index d2e33f7..80e669f 100644 --- a/Sources/Tetra/Combine/ExperimentalMapTask.swift +++ b/Sources/Tetra/Combine/ExperimentalMapTask.swift @@ -221,9 +221,10 @@ extension MultiMapTask { Because polling add queueing taskGroup is implemented in Busy waiting atomic alogrithnm. */ - let unsafe = SuppressSendable(wrapped: group.makeAsyncIterator()) + nonisolated(unsafe) + let unsafe = group.makeAsyncIterator() async let subTask:() = { - var iterator = unsafe.wrapped + var iterator = unsafe while let _ = try await iterator.next() { } diff --git a/Sources/Tetra/Combine/Publishers+AsyncFlatMap.swift b/Sources/Tetra/Combine/Publishers+AsyncFlatMap.swift index 4fa3c6a..db3469a 100644 --- a/Sources/Tetra/Combine/Publishers+AsyncFlatMap.swift +++ b/Sources/Tetra/Combine/Publishers+AsyncFlatMap.swift @@ -100,10 +100,13 @@ extension AsyncFlatMap { } else { let void:Void? = try? await withThrowingTaskGroup(of: Void.self) { group in defer { terminateStream() } - let unsafe = SuppressSendable(wrapped: group.makeAsyncIterator()) + nonisolated(unsafe) + let unsafe = group.makeAsyncIterator() async let subTask:() = { - var iterator = unsafe.wrapped - while let _ = try await iterator.next() { } + var iterator = unsafe + while let _ = try await iterator.next() { + + } }() try await localTask(group: &group) try await subTask diff --git a/Sources/Tetra/Concurrency/CoreDataStack+Concurrency.swift b/Sources/Tetra/Concurrency/CoreDataStack+Concurrency.swift index 0d42fdd..46e22de 100644 --- a/Sources/Tetra/Concurrency/CoreDataStack+Concurrency.swift +++ b/Sources/Tetra/Concurrency/CoreDataStack+Concurrency.swift @@ -26,8 +26,9 @@ extension TetraExtension where Base: NSPersistentStoreCoordinator { } return await withUnsafeContinuation { continuation in base.perform { [unowned holder, continuation] in + nonisolated(unsafe) let result = wrapToResult(holder.closure) - continuation.resume(returning: consume result) + continuation.resume(returning: result) } } } @@ -115,8 +116,9 @@ extension TetraExtension where Base: NSManagedObjectContext { return await withUnsafeContinuation { continuation in base.perform{ [unowned holder, continuation] in + nonisolated(unsafe) let result = wrapToResult(holder.closure) - continuation.resume(returning: consume result) + continuation.resume(returning: result) } } } @@ -222,8 +224,9 @@ extension TetraExtension where Base: NSPersistentContainer { } return await withUnsafeContinuation { continuation in base.performBackgroundTask { [unowned holder, continuation] newContext in + nonisolated(unsafe) let result = _convertToResult(newContext, holder.closure) - continuation.resume(returning: consume result) + continuation.resume(returning: result) } } } diff --git a/Sources/Tetra/Foundation/Mics.swift b/Sources/Tetra/Foundation/Mics.swift index 1869c13..58549e6 100644 --- a/Sources/Tetra/Foundation/Mics.swift +++ b/Sources/Tetra/Foundation/Mics.swift @@ -116,16 +116,3 @@ internal func wrapToResult(_ value:T, _ transform: (T) async return .failure(error) } } - -/// use only when there is no way to prove no data race is occur to the compiler -@usableFromInline -struct SuppressSendable: @unchecked Sendable { - - @usableFromInline - var wrapped:T - - @usableFromInline - init(wrapped: T) { - self.wrapped = wrapped - } -} From 6977f0828c7021ece1b5b6523a58c59bb683cbc4 Mon Sep 17 00:00:00 2001 From: park-byeong-gwan Date: Sat, 8 Jun 2024 17:10:10 +0900 Subject: [PATCH 15/63] suppress sendable warning on dedicated case --- Sources/Tetra/Combine/AsyncSubscriberState.swift | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/Sources/Tetra/Combine/AsyncSubscriberState.swift b/Sources/Tetra/Combine/AsyncSubscriberState.swift index a185976..9ae8261 100644 --- a/Sources/Tetra/Combine/AsyncSubscriberState.swift +++ b/Sources/Tetra/Combine/AsyncSubscriberState.swift @@ -64,7 +64,14 @@ struct AsyncSubscriberState { consuming func run() { switch consume self { case .resumeValue(let continuation, let input): - continuation.resume(returning: .success(input)) + nonisolated(unsafe) + let value = Result.success(consume input) + // just to suppress sendable warning + @inline(__always) + func send(_ value:sending Result) { + continuation.resume(returning: value) + } + send(value) case .request(let subscription, let demand): subscription.request(demand) case .cancel(let array, let subscription): From 006f2845fb37583b70890a18175eb083ceaff94f Mon Sep 17 00:00:00 2001 From: park-byeong-gwan Date: Sat, 8 Jun 2024 17:26:10 +0900 Subject: [PATCH 16/63] fix async iterator isolation --- Sources/Tetra/Concurrency/AsyncSequencePublisher.swift | 2 +- Sources/Tetra/Concurrency/Notification+AsyncSequence.swift | 6 +++++- Sources/Tetra/Foundation/Mics.swift | 7 +++++-- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/Sources/Tetra/Concurrency/AsyncSequencePublisher.swift b/Sources/Tetra/Concurrency/AsyncSequencePublisher.swift index 6261946..33e5259 100644 --- a/Sources/Tetra/Concurrency/AsyncSequencePublisher.swift +++ b/Sources/Tetra/Concurrency/AsyncSequencePublisher.swift @@ -137,7 +137,7 @@ extension AsyncSequencePublisher { for await var pending in demandSource.stream { while pending > .none { pending -= 1 - guard let result = await wrapToResult(&iterator) else { + guard let result = await wrapToResult(nil, &iterator) else { send(completion: .finished) return } diff --git a/Sources/Tetra/Concurrency/Notification+AsyncSequence.swift b/Sources/Tetra/Concurrency/Notification+AsyncSequence.swift index ff2f4a2..0f54ea4 100644 --- a/Sources/Tetra/Concurrency/Notification+AsyncSequence.swift +++ b/Sources/Tetra/Concurrency/Notification+AsyncSequence.swift @@ -44,7 +44,11 @@ public final class NotificationSequence: AsyncTypedSequence, Sendable { let parent:NotificationSequence public func next() async -> Notification? { -// next를 호출한 동안에 task cancellation이 발생하면 observer Token이 무효화되는 것이 확인되므로 아래와 같이 canellation을 추가한다. + await next(isolation: nil) + } + + public func next(isolation actor: isolated (any Actor)?) async throws(Never) -> Notification? { + // next를 호출한 동안에 task cancellation이 발생하면 observer Token이 무효화되는 것이 확인되므로 아래와 같이 canellation을 추가한다. await withTaskCancellationHandler( operation: parent.next, onCancel: parent.cancel diff --git a/Sources/Tetra/Foundation/Mics.swift b/Sources/Tetra/Foundation/Mics.swift index 58549e6..78376fa 100644 --- a/Sources/Tetra/Foundation/Mics.swift +++ b/Sources/Tetra/Foundation/Mics.swift @@ -92,9 +92,12 @@ func wrapToResult(_ block: () throws(Failure) -> T) -> Result(_ iterator: inout Base) async -> Result? { +func wrapToResult( + _ actor: isolated (any Actor)?, + _ iterator: inout Base +) async -> Result? { do { - let value = try await iterator.next(isolation: nil) + let value = try await iterator.next(isolation: actor) if let value { return .success(value) } else { From 5b46411e9eb0e5ade15134917b11fb526bfaaa97 Mon Sep 17 00:00:00 2001 From: park-byeong-gwan Date: Sun, 9 Jun 2024 15:15:35 +0900 Subject: [PATCH 17/63] Concurrency: suppress sendable warning on dedicated case --- .../Concurrency/Notification+AsyncSequence.swift | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/Sources/Tetra/Concurrency/Notification+AsyncSequence.swift b/Sources/Tetra/Concurrency/Notification+AsyncSequence.swift index 0f54ea4..347f1bc 100644 --- a/Sources/Tetra/Concurrency/Notification+AsyncSequence.swift +++ b/Sources/Tetra/Concurrency/Notification+AsyncSequence.swift @@ -72,16 +72,25 @@ public final class NotificationSequence: AsyncTypedSequence, Sendable { self.center = center let observer = center.addObserver(forName: name, object: object, queue: nil) { [lock] notification in - lock.withLockUnchecked { state in + nonisolated(unsafe) + let noti = consume notification + let continuation = lock.withLockUnchecked { state in let captured = state.pending.first if state.pending.isEmpty { - state.buffer.append(notification) + state.buffer.append(noti) } else { state.pending.removeFirst() } return captured - }?.resume(returning: notification) + } + // just to suppress sendable warning + // it is unsafe to do this thing, since NotificationCenter broadcast and share Notifiaction among listeners, but for now there is no way to handle this clearly. + @inline(__always) + func resume(_ noti: sending Notification) { + continuation?.resume(returning: noti) + } + resume(noti) } lock.withLockUnchecked{ $0.observer = observer From 54c37622f5ed8549987826bec49ed51193a34f11 Mon Sep 17 00:00:00 2001 From: park-byeong-gwan Date: Mon, 10 Jun 2024 17:13:54 +0900 Subject: [PATCH 18/63] AsyncFlatMap: fix test fail on precise demand, Notification: fix compile crash --- .../Combine/AsyncFlatMapDemandState.swift | 2 +- Sources/Tetra/Combine/AsyncFlatMapError.swift | 97 +++++++++++++++++++ .../Tetra/Combine/Combine+Concurrency.swift | 16 ++- .../Combine/Publishers+AsyncFlatMap.swift | 50 +++++----- .../Notification+AsyncSequence.swift | 6 +- Tests/TetraTests/AnyEncodableTests.swift | 10 +- Tests/TetraTests/AsyncFlatMapTests.swift | 43 ++++---- 7 files changed, 165 insertions(+), 59 deletions(-) create mode 100644 Sources/Tetra/Combine/AsyncFlatMapError.swift diff --git a/Sources/Tetra/Combine/AsyncFlatMapDemandState.swift b/Sources/Tetra/Combine/AsyncFlatMapDemandState.swift index 7560add..815367d 100644 --- a/Sources/Tetra/Combine/AsyncFlatMapDemandState.swift +++ b/Sources/Tetra/Combine/AsyncFlatMapDemandState.swift @@ -12,7 +12,7 @@ import DequeModule struct AsyncFlatMapDemandState: Sendable { private var suspended:Deque = [] - private var pending = Subscribers.Demand.none + private(set) var pending = Subscribers.Demand.none private var isInterrupted = false typealias Element = UnsafeContinuation diff --git a/Sources/Tetra/Combine/AsyncFlatMapError.swift b/Sources/Tetra/Combine/AsyncFlatMapError.swift new file mode 100644 index 0000000..566682c --- /dev/null +++ b/Sources/Tetra/Combine/AsyncFlatMapError.swift @@ -0,0 +1,97 @@ +// +// AsyncFlatMapError.swift +// +// +// Created by 박병관 on 6/9/24. +// + +import Foundation + + +@usableFromInline +enum AsyncFlatMapError: Error { + + case upstream(First) + case transform(Second) + case segment(Third) + +} + + +extension AsyncFlatMapError{ + + @inlinable + func unwrap() -> Never where First == Second, Second == Third, Third == Never { + fatalError() + } + + @inlinable + func unwrap() -> First where First == Second, Second == Third { + switch self { + case .upstream(let error): + fallthrough + case .segment(let error): + fallthrough + case .transform(let error): + return (error) + } + } + + @inlinable + func unwrap() -> First where First == Second, Third == Never { + switch self { + case .upstream(let error): + fallthrough + case .transform(let error): + return error + + } + } + + @inlinable + func unwrap() -> Second where Second == Third, First == Never { + switch self { + case .transform(let error): + fallthrough + case .segment(let error): + return error + } + } + + @inlinable + func unwrap() -> Third where First == Third, Second == Never { + switch self { + case .upstream(let error): + fallthrough + case .segment(let error): + return error + } + } + + @inlinable + func unwrap() -> First where Second == Third, Third == Never { + switch self { + case .upstream(let error): + return error + } + } + + @inlinable + func unwrap() -> Second where First == Third, First == Never { + switch self { + case .transform(let error): + return error + } + } + + @inlinable + func unwrap() -> Third where First == Second, Second == Never { + switch self { + case .segment(let error): + return error + } + } + + +} + diff --git a/Sources/Tetra/Combine/Combine+Concurrency.swift b/Sources/Tetra/Combine/Combine+Concurrency.swift index af40c9b..560bbe8 100644 --- a/Sources/Tetra/Combine/Combine+Concurrency.swift +++ b/Sources/Tetra/Combine/Combine+Concurrency.swift @@ -47,12 +47,18 @@ public extension Publisher { MultiMapTask(maxTasks: maxTasks, upstream: self, transform: transform) } - internal - func asyncFlatMap( + + +} + +@available(macOS 9999, *) +internal extension Publisher { + + func asyncFlatMap( maxTasks: Subscribers.Demand = .unlimited, - transform: @escaping @Sendable (Output) async throws -> Segment - ) -> AsyncFlatMap where Failure == any Error { - return .init(maxTasks: maxTasks, upstream: self, transform: transform) + transform: @escaping @Sendable (Output) async throws(Err) -> Segment + ) -> AsyncFlatMap where Output:Sendable { + return AsyncFlatMap(maxTasks: maxTasks, upstream: self, transform: transform) } } diff --git a/Sources/Tetra/Combine/Publishers+AsyncFlatMap.swift b/Sources/Tetra/Combine/Publishers+AsyncFlatMap.swift index db3469a..49801a4 100644 --- a/Sources/Tetra/Combine/Publishers+AsyncFlatMap.swift +++ b/Sources/Tetra/Combine/Publishers+AsyncFlatMap.swift @@ -8,11 +8,11 @@ import Foundation @preconcurrency import Combine -struct AsyncFlatMap: Publisher where Upstream.Output:Sendable, Upstream.Failure == any Error { +struct AsyncFlatMap: Publisher where Upstream.Output:Sendable { typealias Output = Segment.Element - typealias Failure = any Error - typealias Transform = @Sendable (Upstream.Output) async throws(Failure) -> Segment + typealias Failure = AsyncFlatMapError + typealias Transform = @Sendable (Upstream.Output) async throws(TransformFail) -> Segment let maxTasks:Subscribers.Demand let upstream:Upstream let transform:Transform @@ -39,11 +39,11 @@ struct AsyncFlatMap: Publisher where U extension AsyncFlatMap { struct Inner: Subscriber, Sendable, Subscription, CustomStringConvertible, CustomPlaygroundDisplayConvertible - where Segment.Element == Down.Input, Down.Failure == Failure { + where Segment.Element == Down.Input, Down.Failure == AsyncFlatMap.Failure { - typealias Transformer = @Sendable (Upstream.Output) async throws(Failure) -> Segment + typealias Transformer = @Sendable (Upstream.Output) async throws(TransformFail) -> Segment typealias Input = Upstream.Output - typealias Failure = any Error + typealias Failure = Upstream.Failure let lock:some UnfairStateLock = createUncheckedStateLock(uncheckedState: TaskState()) @@ -57,7 +57,7 @@ extension AsyncFlatMap { let maxTasks:Subscribers.Demand let transform:Transformer - let valueSource = AsyncStream>.makeStream() + let valueSource = AsyncStream>.makeStream() let combineIdentifier = CombineIdentifier() @@ -131,7 +131,7 @@ extension AsyncFlatMap { return .none } - func receive(completion: Subscribers.Completion) { + func receive(completion: Subscribers.Completion) { switch completion { case .finished: break @@ -165,7 +165,7 @@ extension AsyncFlatMap { }?.run() } - private func send(completion: Subscribers.Completion?) { + private func send(completion: Subscribers.Completion?) { let (subscriber, effect, interruption) = lock.withLockUnchecked{ let old = $0.subscriber $0.subscriber = nil @@ -236,7 +236,7 @@ extension AsyncFlatMap { } private func makeSegment(_ input:Upstream.Output) async throws(CancellationError) -> Segment { - let result:Result + let result:Result do { let seg = try await transform(input) result = .success(seg) @@ -247,7 +247,7 @@ extension AsyncFlatMap { case .success(let success): return success case .failure(let failure): - send(completion: .failure(failure)) + send(completion: .failure(.transform(failure))) throw CancellationError() } } @@ -270,35 +270,33 @@ extension AsyncFlatMap { private func processNextSegment( iterator: inout Segment.AsyncIterator ) async throws(CancellationError) -> Bool { - let nextResult:Result? - do { - let value = try await iterator.next() - if let value { - nextResult = .success(value) - } else { - nextResult = nil - } - } catch { - nextResult = .failure(error) - } + let nextResult = await wrapToResult(nil, &iterator) switch nextResult { case .none: //finished // check and request more transformer // request one more from upstream subscription if maxTasks != .unlimited { - let subscription = lock.withLockUnchecked{ - $0.upstreamSubscription.subscription + let (subscription, effect) = lock.withLockUnchecked{ + let effect = if $0.demandState.pending == .unlimited { + $0.demandState.transition(.resume(.none)) + } else { + // reclaim discarded demand + $0.demandState.transition(.resume(.max(1))) + } + let subscription = $0.upstreamSubscription.subscription + return (subscription, effect) } if let subscription { subscription.request(.max(1)) + effect?.run() } else { throw CancellationError() } } return false case .failure(let error): - send(completion: .failure(error)) + send(completion: .failure(.segment(error))) throw CancellationError() case .success(let value): try send(value) @@ -312,7 +310,7 @@ extension AsyncFlatMap { for await result in valueSource.stream { switch result { case .failure(let failure): - send(completion: .failure(failure)) + send(completion: .failure(.upstream(failure))) throw CancellationError() case .success(let value): let isSuccess = group.addTaskUnlessCancelled(priority: nil) { diff --git a/Sources/Tetra/Concurrency/Notification+AsyncSequence.swift b/Sources/Tetra/Concurrency/Notification+AsyncSequence.swift index 347f1bc..b60ed12 100644 --- a/Sources/Tetra/Concurrency/Notification+AsyncSequence.swift +++ b/Sources/Tetra/Concurrency/Notification+AsyncSequence.swift @@ -73,12 +73,12 @@ public final class NotificationSequence: AsyncTypedSequence, Sendable { self.center = center let observer = center.addObserver(forName: name, object: object, queue: nil) { [lock] notification in nonisolated(unsafe) - let noti = consume notification + let noti2 = notification let continuation = lock.withLockUnchecked { state in let captured = state.pending.first if state.pending.isEmpty { - state.buffer.append(noti) + state.buffer.append(noti2) } else { state.pending.removeFirst() } @@ -90,7 +90,7 @@ public final class NotificationSequence: AsyncTypedSequence, Sendable { func resume(_ noti: sending Notification) { continuation?.resume(returning: noti) } - resume(noti) + resume(noti2) } lock.withLockUnchecked{ $0.observer = observer diff --git a/Tests/TetraTests/AnyEncodableTests.swift b/Tests/TetraTests/AnyEncodableTests.swift index 917f682..ecf1eb8 100644 --- a/Tests/TetraTests/AnyEncodableTests.swift +++ b/Tests/TetraTests/AnyEncodableTests.swift @@ -19,12 +19,10 @@ final class AnyEncodableTests: XCTestCase { try JSONEncoder().encode(targetURL) ) - try XCTExpectFailure { - XCTAssertEqual( - try JSONEncoder().encode(AnyErasedEncodable(value: targetURL)), - try JSONEncoder().encode(targetURL) - ) - } + XCTAssertNotEqual( + try JSONEncoder().encode(AnyErasedEncodable(value: targetURL)), + try JSONEncoder().encode(targetURL) + ) } diff --git a/Tests/TetraTests/AsyncFlatMapTests.swift b/Tests/TetraTests/AsyncFlatMapTests.swift index 98b4536..36925b3 100644 --- a/Tests/TetraTests/AsyncFlatMapTests.swift +++ b/Tests/TetraTests/AsyncFlatMapTests.swift @@ -22,7 +22,6 @@ final class AsyncFlatMapTests: XCTestCase { XCTAssertEqual($0, .max(1)) } ) - .setFailureType(to: Error.self) .asyncFlatMap(maxTasks: .max(1)) { value in AsyncStream{ continuation in sample.forEach{ @@ -37,13 +36,16 @@ final class AsyncFlatMapTests: XCTestCase { receiveSubscription: { XCTAssertEqual("\($0)", "AsyncFlatMap") } - ) + ).mapError{ $0.unwrap() } + // ensure downstream do not request unlimited + .buffer(size: 1, prefetch: .keepFull, whenFull: .customError{ fatalError() }) + .prefix(10) .sink { _ in completion.fulfill() } receiveValue: { array.append($0) } - wait(for: [completion]) + wait(for: [completion], timeout: 0.1) bag.cancel() XCTAssertEqual(array, sample + sample) } @@ -59,7 +61,6 @@ final class AsyncFlatMapTests: XCTestCase { XCTAssertEqual($0, .max(2)) } ) - .setFailureType(to: Error.self) .asyncFlatMap(maxTasks: .max(2)) { value in return AsyncStream{ continuation in sample.forEach{ @@ -74,7 +75,7 @@ final class AsyncFlatMapTests: XCTestCase { receiveSubscription: { XCTAssertEqual("\($0)", "AsyncFlatMap") } - ) + ).mapError{ $0.unwrap() } .sink { _ in completion.fulfill() } receiveValue: { value in @@ -94,7 +95,6 @@ final class AsyncFlatMapTests: XCTestCase { let lock = NSRecursiveLock() lock.withLock { (0..<5).publisher - .setFailureType(to: Error.self) .asyncFlatMap(maxTasks: .unlimited) { value in lock.withLock{ holder.bag = [] @@ -103,7 +103,8 @@ final class AsyncFlatMapTests: XCTestCase { $0.yield(value) $0.finish() } - }.handleEvents( + }.mapError{ $0.unwrap() } + .handleEvents( receiveCancel: { completion.fulfill() } @@ -122,7 +123,6 @@ final class AsyncFlatMapTests: XCTestCase { let lock = NSRecursiveLock() lock.withLock { (0..<5).publisher - .setFailureType(to: Error.self) .asyncFlatMap(maxTasks: .unlimited) { value in return AsyncStream{ $0.yield(value) @@ -133,7 +133,8 @@ final class AsyncFlatMapTests: XCTestCase { } return $0 } - }.handleEvents( + }.mapError{ $0.unwrap() } + .handleEvents( receiveCancel: { completion.fulfill() } @@ -152,14 +153,15 @@ final class AsyncFlatMapTests: XCTestCase { let lock = NSRecursiveLock() lock.withLock { (0..<5).publisher - .setFailureType(to: Error.self) - .asyncFlatMap(maxTasks: .unlimited) { value in - return AsyncStream{ + .asyncFlatMap(maxTasks: .unlimited) { @Sendable value in + return AsyncStream{ @Sendable in $0.yield(value) $0.finish() } + }.mapError{ + $0.unwrap() }.handleEvents( - receiveCancel: { + receiveCancel: { @Sendable in completion.fulfill() } ).sink { _ in @@ -174,11 +176,11 @@ final class AsyncFlatMapTests: XCTestCase { } + @available(macOS 9999, *) func testThrowInTransformer() throws { let holder = UnsafeCancellableHolder() let completion = expectation(description: "cancellation") (0..<5).publisher - .setFailureType(to: Error.self) .asyncFlatMap(maxTasks: .max(1)) { value in if value == 3 { throw CancellationError() @@ -187,6 +189,8 @@ final class AsyncFlatMapTests: XCTestCase { $0.yield(value) $0.finish() } + }.mapError{ + $0.unwrap() }.sink { switch $0 { case .finished: @@ -205,9 +209,7 @@ final class AsyncFlatMapTests: XCTestCase { let holder = UnsafeCancellableHolder() let completion = expectation(description: "cancellation") (0..<5).publisher - .setFailureType(to: Error.self) .asyncFlatMap(maxTasks: .max(1)) { value in - return AsyncStream{ $0.yield(value) $0.finish() @@ -217,7 +219,11 @@ final class AsyncFlatMapTests: XCTestCase { } return $0 } - }.sink { + } + .mapError{ + $0.unwrap() + } + .sink { switch $0 { case .finished: break @@ -239,7 +245,6 @@ final class AsyncFlatMapTests: XCTestCase { let completion = expectation(description: "complete") var buffer = [Int]() (0..<5).publisher - .setFailureType(to: Error.self) .asyncFlatMap(maxTasks: .max(1)) { value in if value == 0 { withUnsafeCurrentTask{ @@ -259,6 +264,8 @@ final class AsyncFlatMapTests: XCTestCase { withUnsafeCurrentTask{$0?.cancel()} return value } + }.mapError{ + $0.unwrap() }.sink { _ in completion.fulfill() } receiveValue: { From 24d54b9c941de91ca348985a9156b42dccf8397e Mon Sep 17 00:00:00 2001 From: park-byeong-gwan Date: Mon, 10 Jun 2024 17:18:14 +0900 Subject: [PATCH 19/63] remove unwanted schema --- .../xcode/xcshareddata/xcschemes/Foo.xcscheme | 78 -------------- .../xcschemes/Tetra-Package.xcscheme | 102 ------------------ 2 files changed, 180 deletions(-) delete mode 100644 .swiftpm/xcode/xcshareddata/xcschemes/Foo.xcscheme delete mode 100644 .swiftpm/xcode/xcshareddata/xcschemes/Tetra-Package.xcscheme diff --git a/.swiftpm/xcode/xcshareddata/xcschemes/Foo.xcscheme b/.swiftpm/xcode/xcshareddata/xcschemes/Foo.xcscheme deleted file mode 100644 index e510279..0000000 --- a/.swiftpm/xcode/xcshareddata/xcschemes/Foo.xcscheme +++ /dev/null @@ -1,78 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/.swiftpm/xcode/xcshareddata/xcschemes/Tetra-Package.xcscheme b/.swiftpm/xcode/xcshareddata/xcschemes/Tetra-Package.xcscheme deleted file mode 100644 index 69b062a..0000000 --- a/.swiftpm/xcode/xcshareddata/xcschemes/Tetra-Package.xcscheme +++ /dev/null @@ -1,102 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - From 77960fa821d7938468e499bc66dbe9d47cc5ca4c Mon Sep 17 00:00:00 2001 From: park-byeong-gwan Date: Mon, 10 Jun 2024 18:26:28 +0900 Subject: [PATCH 20/63] AsyncFlatMap: fix task leak and propagate cancellation to upstream properly --- Sources/Tetra/Combine/AsyncFlatMapDemandState.swift | 8 +++++++- Sources/Tetra/Combine/Publishers+AsyncFlatMap.swift | 10 +++++++++- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/Sources/Tetra/Combine/AsyncFlatMapDemandState.swift b/Sources/Tetra/Combine/AsyncFlatMapDemandState.swift index 815367d..8955a2a 100644 --- a/Sources/Tetra/Combine/AsyncFlatMapDemandState.swift +++ b/Sources/Tetra/Combine/AsyncFlatMapDemandState.swift @@ -55,7 +55,13 @@ struct AsyncFlatMapDemandState: Sendable { private mutating func resume(_ demand:Subscribers.Demand) -> Effect? { if isInterrupted { - return nil + if suspended.isEmpty { + return nil + } else { + let jobs = suspended + suspended = [] + return .raise(jobs) + } } pending += demand return populateResume() diff --git a/Sources/Tetra/Combine/Publishers+AsyncFlatMap.swift b/Sources/Tetra/Combine/Publishers+AsyncFlatMap.swift index 49801a4..b2f62f2 100644 --- a/Sources/Tetra/Combine/Publishers+AsyncFlatMap.swift +++ b/Sources/Tetra/Combine/Publishers+AsyncFlatMap.swift @@ -166,10 +166,18 @@ extension AsyncFlatMap { } private func send(completion: Subscribers.Completion?) { + valueSource.continuation.finish() + let shouldCancel:Bool + switch completion { + case .none, .failure(.segment(_)), .failure(.transform(_)): + shouldCancel = true + case .failure(.upstream(_)), .finished: + shouldCancel = false + } let (subscriber, effect, interruption) = lock.withLockUnchecked{ let old = $0.subscriber $0.subscriber = nil - let effect = if completion == nil { + let effect = if shouldCancel { $0.upstreamSubscription.transition(.cancel) } else { $0.upstreamSubscription.transition(.finish) From 8075e9d6d4242d8d9dd2b8a159bfb829ce51c801 Mon Sep 17 00:00:00 2001 From: park-byeong-gwan Date: Mon, 10 Jun 2024 21:02:18 +0900 Subject: [PATCH 21/63] Concurrency: mark support for isolation --- Sources/Tetra/Combine/AsyncSubscriber.swift | 6 ++++-- .../Tetra/Combine/Combine+Concurrency.swift | 8 ++++---- .../Tetra/Combine/CompatAsyncPublisher.swift | 1 + .../CompatAsyncThrowingPublisher.swift | 5 ++++- .../Tetra/Combine/ExperimentalMapTask.swift | 2 +- .../Combine/Publishers+AsyncFlatMap.swift | 10 ++++++++-- .../Tetra/Combine/Publishers+MapTask.swift | 20 +++++++++---------- .../Tetra/Combine/Publishers+TryMapTask.swift | 7 +++++-- .../Notification+AsyncSequence.swift | 8 +++++--- 9 files changed, 41 insertions(+), 26 deletions(-) diff --git a/Sources/Tetra/Combine/AsyncSubscriber.swift b/Sources/Tetra/Combine/AsyncSubscriber.swift index 5c096a1..5610f4e 100644 --- a/Sources/Tetra/Combine/AsyncSubscriber.swift +++ b/Sources/Tetra/Combine/AsyncSubscriber.swift @@ -52,8 +52,10 @@ internal struct AsyncSubscriber: Sendable, Subscriber, Cancellable } @usableFromInline - func next() async -> Result? { - return await withUnsafeContinuation { continuation in + func next( + isolation: isolated (any Actor)? + ) async -> Result? { + return await withUnsafeContinuation(isolation: isolation) { continuation in lock.withLockUnchecked{ $0.transition(.suspend(continuation)) }?.run() diff --git a/Sources/Tetra/Combine/Combine+Concurrency.swift b/Sources/Tetra/Combine/Combine+Concurrency.swift index 560bbe8..1d3fc2a 100644 --- a/Sources/Tetra/Combine/Combine+Concurrency.swift +++ b/Sources/Tetra/Combine/Combine+Concurrency.swift @@ -32,18 +32,18 @@ public extension TetraExtension where Base: Publisher { public extension Publisher { @inlinable - func mapTask(transform: @escaping @Sendable (Output) async -> T) -> MapTask where Output:Sendable { + func mapTask(transform: @escaping @isolated(any) @Sendable (Output) async -> T) -> MapTask where Output:Sendable { MapTask(upstream: self, transform: transform) } @inlinable - func tryMapTask(transform: @escaping @Sendable (Output) async throws -> T) -> TryMapTask where Output:Sendable { + func tryMapTask(transform: @escaping @isolated(any) @Sendable (Output) async throws -> T) -> TryMapTask where Output:Sendable { TryMapTask(upstream: self, transform: transform) } @_spi(Experimental) @inlinable - func multiMapTask(maxTasks: Subscribers.Demand = .max(1), transform: @escaping @Sendable (Output) async throws(Self.Failure) -> T) -> MultiMapTask where Output: Sendable { + func multiMapTask(maxTasks: Subscribers.Demand = .max(1), transform: @escaping @Sendable @isolated(any) (Output) async throws(Self.Failure) -> T) -> MultiMapTask where Output: Sendable { MultiMapTask(maxTasks: maxTasks, upstream: self, transform: transform) } @@ -56,7 +56,7 @@ internal extension Publisher { func asyncFlatMap( maxTasks: Subscribers.Demand = .unlimited, - transform: @escaping @Sendable (Output) async throws(Err) -> Segment + transform: @escaping @Sendable @isolated(any) (Output) async throws(Err) -> Segment ) -> AsyncFlatMap where Output:Sendable { return AsyncFlatMap(maxTasks: maxTasks, upstream: self, transform: transform) } diff --git a/Sources/Tetra/Combine/CompatAsyncPublisher.swift b/Sources/Tetra/Combine/CompatAsyncPublisher.swift index ff814ac..3273b04 100644 --- a/Sources/Tetra/Combine/CompatAsyncPublisher.swift +++ b/Sources/Tetra/Combine/CompatAsyncPublisher.swift @@ -42,6 +42,7 @@ public struct CompatAsyncPublisher: AsyncSequence where P.Failure = let result = await withTaskCancellationHandler(operation: inner.next) { [reference] in reference.cancel() } + switch result { case .none: return nil diff --git a/Sources/Tetra/Combine/CompatAsyncThrowingPublisher.swift b/Sources/Tetra/Combine/CompatAsyncThrowingPublisher.swift index a0c8435..c728475 100644 --- a/Sources/Tetra/Combine/CompatAsyncThrowingPublisher.swift +++ b/Sources/Tetra/Combine/CompatAsyncThrowingPublisher.swift @@ -31,9 +31,12 @@ public struct CompatAsyncThrowingPublisher: AsyncTypedSequence { @inlinable public mutating func next(isolation actor: isolated (any Actor)?) async throws(P.Failure) -> P.Output? { - let result = await withTaskCancellationHandler(operation: inner.next) { [reference] in + let result = await withTaskCancellationHandler { [inner] in + await inner.next(isolation: actor) + } onCancel: { [reference] in reference.cancel() } + switch result { case .failure(let failure): throw failure diff --git a/Sources/Tetra/Combine/ExperimentalMapTask.swift b/Sources/Tetra/Combine/ExperimentalMapTask.swift index e68ac9c..e7d2352 100644 --- a/Sources/Tetra/Combine/ExperimentalMapTask.swift +++ b/Sources/Tetra/Combine/ExperimentalMapTask.swift @@ -37,7 +37,7 @@ public struct MultiMapTask: Publisher where public init( maxTasks: Subscribers.Demand = .max(1), upstream: Upstream, - transform: @Sendable @escaping (Upstream.Output) async throws(Failure) -> Output + transform: @Sendable @escaping @isolated(any) (Upstream.Output) async throws(Failure) -> Output ) { precondition(maxTasks != .none, "maxTasks can not be zero") self.maxTasks = maxTasks diff --git a/Sources/Tetra/Combine/Publishers+AsyncFlatMap.swift b/Sources/Tetra/Combine/Publishers+AsyncFlatMap.swift index b2f62f2..b7b9ff7 100644 --- a/Sources/Tetra/Combine/Publishers+AsyncFlatMap.swift +++ b/Sources/Tetra/Combine/Publishers+AsyncFlatMap.swift @@ -12,7 +12,7 @@ struct AsyncFlatMap - typealias Transform = @Sendable (Upstream.Output) async throws(TransformFail) -> Segment + typealias Transform = @Sendable @isolated(any) (Upstream.Output) async throws(TransformFail) -> Segment let maxTasks:Subscribers.Demand let upstream:Upstream let transform:Transform @@ -143,8 +143,14 @@ extension AsyncFlatMap { func receive(subscription: any Subscription) { let (effect, requestValue) = lock.withLockUnchecked{ + let requestValue:Bool + switch $0.upstreamSubscription { + case .waiting, .suspending: + requestValue = true + default: + requestValue = false + } let effect = $0.upstreamSubscription.transition(.resume(subscription)) - let requestValue = subscription.combineIdentifier == $0.upstreamSubscription.subscription?.combineIdentifier return (effect, requestValue) } effect?.run() diff --git a/Sources/Tetra/Combine/Publishers+MapTask.swift b/Sources/Tetra/Combine/Publishers+MapTask.swift index 5e24bc7..fda519e 100644 --- a/Sources/Tetra/Combine/Publishers+MapTask.swift +++ b/Sources/Tetra/Combine/Publishers+MapTask.swift @@ -39,16 +39,22 @@ public struct MapTask: Publisher where Upst public typealias Failure = Upstream.Failure public let upstream:Upstream - public var transform:@Sendable (Upstream.Output) async -> Result + public var transform:@Sendable @isolated(any) (Upstream.Output) async -> Result - public init(upstream: Upstream, transform: @escaping @Sendable (Upstream.Output) async -> Output) { + public init( + upstream: Upstream, + transform: @escaping @Sendable @isolated(any) (Upstream.Output) async -> Output + ) { self.upstream = upstream self.transform = { Result.success(await transform($0)) } } - public init(upstream: Upstream, handler: @escaping @Sendable (Upstream.Output) async -> Result) { + public init( + upstream: Upstream, + handler: @escaping @Sendable @isolated(any) (Upstream.Output) async -> Result + ) { self.upstream = upstream self.transform = handler } @@ -177,14 +183,6 @@ extension MapTask { guard success != nil else { return } - let stream = valueSource.stream.map{ [transform] in - switch $0 { - case .success(let value): - return await transform(value) - case .failure(let failure): - return .failure(failure) - } - } try? await withTaskCancellationHandler { for await upstreamResult in valueSource.stream { let upValue: Upstream.Output diff --git a/Sources/Tetra/Combine/Publishers+TryMapTask.swift b/Sources/Tetra/Combine/Publishers+TryMapTask.swift index fe80a44..7980457 100644 --- a/Sources/Tetra/Combine/Publishers+TryMapTask.swift +++ b/Sources/Tetra/Combine/Publishers+TryMapTask.swift @@ -36,9 +36,12 @@ public struct TryMapTask: Publisher where U public typealias Failure = any Error public let upstream:Upstream - public var transform:@Sendable (Upstream.Output) async throws -> Output + public var transform: @isolated(any) @Sendable (Upstream.Output) async throws -> Output - public init(upstream: Upstream, transform: @escaping @Sendable (Upstream.Output) async throws -> Output) { + public init( + upstream: Upstream, + transform: @escaping @isolated(any) @Sendable (Upstream.Output) async throws -> Output + ) { self.upstream = upstream self.transform = transform } diff --git a/Sources/Tetra/Concurrency/Notification+AsyncSequence.swift b/Sources/Tetra/Concurrency/Notification+AsyncSequence.swift index b60ed12..b9f88b0 100644 --- a/Sources/Tetra/Concurrency/Notification+AsyncSequence.swift +++ b/Sources/Tetra/Concurrency/Notification+AsyncSequence.swift @@ -50,7 +50,9 @@ public final class NotificationSequence: AsyncTypedSequence, Sendable { public func next(isolation actor: isolated (any Actor)?) async throws(Never) -> Notification? { // next를 호출한 동안에 task cancellation이 발생하면 observer Token이 무효화되는 것이 확인되므로 아래와 같이 canellation을 추가한다. await withTaskCancellationHandler( - operation: parent.next, + operation: { [parent] in + await parent.next(isolation: actor) + }, onCancel: parent.cancel ) } @@ -117,8 +119,8 @@ public final class NotificationSequence: AsyncTypedSequence, Sendable { snapShot.pending.forEach{ $0.resume(returning: nil) } } - func next() async -> Notification? { - await withUnsafeContinuation { continuation in + func next(isolation: isolated (any Actor)?) async -> Notification? { + await withUnsafeContinuation(isolation: isolation) { continuation in let (notification, isCancelled) = lock.withLockUnchecked { state in if !state.buffer.isEmpty { return (state.buffer.removeFirst() as Notification?, false) From aee8c0909d2887991c897bd01fabe1927ef20022 Mon Sep 17 00:00:00 2001 From: park-byeong-gwan Date: Mon, 10 Jun 2024 21:07:46 +0900 Subject: [PATCH 22/63] CompatAsyncPublisher: fix compiler crash by assertion --- Sources/Tetra/Combine/CompatAsyncPublisher.swift | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/Sources/Tetra/Combine/CompatAsyncPublisher.swift b/Sources/Tetra/Combine/CompatAsyncPublisher.swift index 3273b04..4a7a3f0 100644 --- a/Sources/Tetra/Combine/CompatAsyncPublisher.swift +++ b/Sources/Tetra/Combine/CompatAsyncPublisher.swift @@ -39,7 +39,14 @@ public struct CompatAsyncPublisher: AsyncSequence where P.Failure = @inlinable public mutating func next() async -> P.Output? { - let result = await withTaskCancellationHandler(operation: inner.next) { [reference] in + return await next(isolation: nil) + } + + @inlinable + public func next(isolation actor: isolated (any Actor)?) async -> P.Output? { + let result: Result? = await withTaskCancellationHandler { [inner] in + await inner.next(isolation: actor) + } onCancel: { [reference] in reference.cancel() } From 7f584091cb3c208b3f9323feb7495a9516f98a44 Mon Sep 17 00:00:00 2001 From: park-byeong-gwan Date: Mon, 10 Jun 2024 22:20:22 +0900 Subject: [PATCH 23/63] mapTask, TryMapTask: enqueue to separate task to prevent transformer cancelling root task using `UnsafeCurrentTask` --- .../Tetra/Combine/Publishers+MapTask.swift | 4 +++- .../Tetra/Combine/Publishers+TryMapTask.swift | 4 +++- Tests/TetraTests/MapTaskTests.swift | 22 +++++++++++++++++++ Tests/TetraTests/TryMapTaskTests.swift | 22 +++++++++++++++++++ 4 files changed, 50 insertions(+), 2 deletions(-) diff --git a/Sources/Tetra/Combine/Publishers+MapTask.swift b/Sources/Tetra/Combine/Publishers+MapTask.swift index fda519e..22d1527 100644 --- a/Sources/Tetra/Combine/Publishers+MapTask.swift +++ b/Sources/Tetra/Combine/Publishers+MapTask.swift @@ -193,7 +193,9 @@ extension MapTask { case .success(let value): upValue = value } - switch (await transform(upValue)) { + // enqueue to separate task to prevent transformer cancelling root task using `UnsafeCurrentTask` + async let job = transform(upValue) + switch (await job) { case .failure(let error): send(completion: .failure(error), cancel: true) throw CancellationError() diff --git a/Sources/Tetra/Combine/Publishers+TryMapTask.swift b/Sources/Tetra/Combine/Publishers+TryMapTask.swift index 7980457..3ebea49 100644 --- a/Sources/Tetra/Combine/Publishers+TryMapTask.swift +++ b/Sources/Tetra/Combine/Publishers+TryMapTask.swift @@ -187,7 +187,9 @@ extension TryMapTask { return } do { - let value = try await transform(upValue) + // enqueue to separate task to prevent transformer cancelling root task using `UnsafeCurrentTask` + async let job = transform(upValue) + let value = try await job guard let _ = try? send(value) else { return } diff --git a/Tests/TetraTests/MapTaskTests.swift b/Tests/TetraTests/MapTaskTests.swift index 9963977..5e0fbf6 100644 --- a/Tests/TetraTests/MapTaskTests.swift +++ b/Tests/TetraTests/MapTaskTests.swift @@ -158,5 +158,27 @@ final class MapTaskTests: XCTestCase { } } + + func testUnsafeCancel() throws { + let completion = expectation(description: "completion") + var array = [Int]() + let token = (0..<100) + .publisher + .mapTask { value in + await Task.yield() + withUnsafeCurrentTask { + $0?.cancel() + } + return value + }.buffer(size: 1, prefetch: .keepFull, whenFull: .customError{ fatalError() }) + .sink { _ in + completion.fulfill() + } receiveValue: { + array.append($0) + } + wait(for: [completion], timeout: 0.2) + XCTAssertEqual(array, Array(0..<100)) + token.cancel() + } } diff --git a/Tests/TetraTests/TryMapTaskTests.swift b/Tests/TetraTests/TryMapTaskTests.swift index 13b18a1..1ac5d33 100644 --- a/Tests/TetraTests/TryMapTaskTests.swift +++ b/Tests/TetraTests/TryMapTaskTests.swift @@ -93,5 +93,27 @@ final class TryMapTaskTests: XCTestCase { wait(for: [expect], timeout: 0.5) bag.removeAll() } + + func testUnsafeCancel() throws { + let completion = expectation(description: "completion") + var array = [Int]() + let token = (0..<100) + .publisher + .tryMapTask { value in + await Task.yield() + withUnsafeCurrentTask { + $0?.cancel() + } + return value + }.buffer(size: 1, prefetch: .keepFull, whenFull: .customError{ fatalError() }) + .sink { _ in + completion.fulfill() + } receiveValue: { + array.append($0) + } + wait(for: [completion], timeout: 0.2) + XCTAssertEqual(array, Array(0..<100)) + token.cancel() + } } From 18fbd5ce9dc00d8bec1bea3f616d6e9966ea75ed Mon Sep 17 00:00:00 2001 From: park-byeong-gwan Date: Tue, 11 Jun 2024 10:59:54 +0900 Subject: [PATCH 24/63] swift 6: fix compile fail --- Package.swift | 6 +- Sources/Tetra/Combine/AsyncSubscriber.swift | 2 +- .../Tetra/Combine/AsyncSubscriberState.swift | 7 +-- .../Tetra/Combine/Combine+Concurrency.swift | 11 +--- .../CompatAsyncThrowingPublisher.swift | 2 +- .../Tetra/Combine/ExperimentalMapTask.swift | 4 +- .../Tetra/Combine/Future+Concurrency.swift | 44 +++++++------- .../Combine/Publishers+AsyncFlatMap.swift | 50 +++++++++++++--- .../Concurrency/AsyncSequencePublisher.swift | 36 +++++++++--- .../Concurrency/AsyncTypedSequence.swift | 1 + .../Notification+AsyncSequence.swift | 22 +++---- Sources/Tetra/Foundation/Mics.swift | 2 +- Sources/Tetra/Foundation/Suppress.swift | 19 ++++++ .../Tetra/SwiftUI/Binding+Collection.swift | 10 ++-- .../Tetra/SwiftUI/RefreshableScrollView.swift | 58 +++++++------------ Tests/TetraTests/AsyncFlatMapTests.swift | 13 ++++- .../AsyncSequencePublisherTests.swift | 6 +- 17 files changed, 171 insertions(+), 122 deletions(-) create mode 100644 Sources/Tetra/Foundation/Suppress.swift diff --git a/Package.swift b/Package.swift index 4db774d..67c6414 100644 --- a/Package.swift +++ b/Package.swift @@ -1,4 +1,4 @@ -// swift-tools-version: 5.9 +// swift-tools-version: 6.0 // The swift-tools-version declares the minimum version of Swift required to build this package. import PackageDescription @@ -34,9 +34,9 @@ let package = Package( .product(name: "DequeModule", package: "swift-collections") ], swiftSettings: [ - .enableExperimentalFeature("StrictConcurrency=complete"), .enableUpcomingFeature("FullTypedThrows"), - .enableExperimentalFeature("IsolatedAny") + .enableExperimentalFeature("IsolatedAny"), + .swiftLanguageVersion(.v6) ] ), .testTarget( diff --git a/Sources/Tetra/Combine/AsyncSubscriber.swift b/Sources/Tetra/Combine/AsyncSubscriber.swift index 5610f4e..c02bcc9 100644 --- a/Sources/Tetra/Combine/AsyncSubscriber.swift +++ b/Sources/Tetra/Combine/AsyncSubscriber.swift @@ -55,7 +55,7 @@ internal struct AsyncSubscriber: Sendable, Subscriber, Cancellable func next( isolation: isolated (any Actor)? ) async -> Result? { - return await withUnsafeContinuation(isolation: isolation) { continuation in + return await withUnsafeContinuation { continuation in lock.withLockUnchecked{ $0.transition(.suspend(continuation)) }?.run() diff --git a/Sources/Tetra/Combine/AsyncSubscriberState.swift b/Sources/Tetra/Combine/AsyncSubscriberState.swift index 9ae8261..50fe620 100644 --- a/Sources/Tetra/Combine/AsyncSubscriberState.swift +++ b/Sources/Tetra/Combine/AsyncSubscriberState.swift @@ -66,12 +66,7 @@ struct AsyncSubscriberState { case .resumeValue(let continuation, let input): nonisolated(unsafe) let value = Result.success(consume input) - // just to suppress sendable warning - @inline(__always) - func send(_ value:sending Result) { - continuation.resume(returning: value) - } - send(value) + continuation.resume(returning: value) case .request(let subscription, let demand): subscription.request(demand) case .cancel(let array, let subscription): diff --git a/Sources/Tetra/Combine/Combine+Concurrency.swift b/Sources/Tetra/Combine/Combine+Concurrency.swift index 1d3fc2a..8b5c603 100644 --- a/Sources/Tetra/Combine/Combine+Concurrency.swift +++ b/Sources/Tetra/Combine/Combine+Concurrency.swift @@ -19,12 +19,8 @@ public extension Publisher { public extension TetraExtension where Base: Publisher { @inlinable - var values: some AsyncTypedSequence { - if #available(iOS 15.0, tvOS 15.0, watchOS 8.0, macCatalyst 15.0, macOS 12.0, *) { - return base.values - } else { - return CompatAsyncThrowingPublisher(publisher: base) - } + var values: CompatAsyncThrowingPublisher { + CompatAsyncThrowingPublisher(publisher: base) } } @@ -51,13 +47,12 @@ public extension Publisher { } -@available(macOS 9999, *) internal extension Publisher { func asyncFlatMap( maxTasks: Subscribers.Demand = .unlimited, transform: @escaping @Sendable @isolated(any) (Output) async throws(Err) -> Segment - ) -> AsyncFlatMap where Output:Sendable { + ) -> AsyncFlatMap where Output:Sendable { return AsyncFlatMap(maxTasks: maxTasks, upstream: self, transform: transform) } diff --git a/Sources/Tetra/Combine/CompatAsyncThrowingPublisher.swift b/Sources/Tetra/Combine/CompatAsyncThrowingPublisher.swift index c728475..93ba68c 100644 --- a/Sources/Tetra/Combine/CompatAsyncThrowingPublisher.swift +++ b/Sources/Tetra/Combine/CompatAsyncThrowingPublisher.swift @@ -12,7 +12,7 @@ import Foundation public struct CompatAsyncThrowingPublisher: AsyncTypedSequence { public typealias AsyncIterator = Iterator - + public typealias Failure = Iterator.Failure public var publisher:P @inlinable diff --git a/Sources/Tetra/Combine/ExperimentalMapTask.swift b/Sources/Tetra/Combine/ExperimentalMapTask.swift index e7d2352..ba1f663 100644 --- a/Sources/Tetra/Combine/ExperimentalMapTask.swift +++ b/Sources/Tetra/Combine/ExperimentalMapTask.swift @@ -223,9 +223,9 @@ extension MultiMapTask { Because polling add queueing taskGroup is implemented in Busy waiting atomic alogrithnm. */ nonisolated(unsafe) - let unsafe = group.makeAsyncIterator() + let unsafe = Suppress(value: group.makeAsyncIterator()) async let subTask:() = { - var iterator = unsafe + var iterator = unsafe.value while let _ = try await iterator.next() { } diff --git a/Sources/Tetra/Combine/Future+Concurrency.swift b/Sources/Tetra/Combine/Future+Concurrency.swift index 945366d..34ef811 100644 --- a/Sources/Tetra/Combine/Future+Concurrency.swift +++ b/Sources/Tetra/Combine/Future+Concurrency.swift @@ -20,31 +20,27 @@ public extension Combine.Future { @inlinable final var compatValue: Output { get async throws(Failure) { - if #available(iOS 15.0, tvOS 15.0, macCatalyst 15.0, watchOS 8.0, macOS 12.0, *) { - return try await value - } else { - let result: Result = await withCheckedContinuation { continuation in - self.subscribe(AnySubscriber( - receiveSubscription: { - $0.request(.max(1)) - }, - receiveValue: { (value: sending Output) in - continuation.resume(returning: .success(value)) - return .none - }, - receiveCompletion: { - if case let .failure(error) = $0 { - continuation.resume(returning: .failure(error)) - } + let result: Result = await withCheckedContinuation { continuation in + self.subscribe(AnySubscriber( + receiveSubscription: { + $0.request(.max(1)) + }, + receiveValue: { (value: sending Output) in + continuation.resume(returning: .success(value)) + return .none + }, + receiveCompletion: { + if case let .failure(error) = $0 { + continuation.resume(returning: .failure(error)) } - )) - } - switch result { - case .success(let success): - return success - case .failure(let failure): - throw failure - } + } + )) + } + switch result { + case .success(let success): + return success + case .failure(let failure): + throw failure } } } diff --git a/Sources/Tetra/Combine/Publishers+AsyncFlatMap.swift b/Sources/Tetra/Combine/Publishers+AsyncFlatMap.swift index b7b9ff7..272f336 100644 --- a/Sources/Tetra/Combine/Publishers+AsyncFlatMap.swift +++ b/Sources/Tetra/Combine/Publishers+AsyncFlatMap.swift @@ -8,10 +8,11 @@ import Foundation @preconcurrency import Combine -struct AsyncFlatMap: Publisher where Upstream.Output:Sendable { + +struct AsyncFlatMap: Publisher where Upstream.Output:Sendable { typealias Output = Segment.Element - typealias Failure = AsyncFlatMapError + typealias Failure = AsyncFlatMapError typealias Transform = @Sendable @isolated(any) (Upstream.Output) async throws(TransformFail) -> Segment let maxTasks:Subscribers.Demand let upstream:Upstream @@ -24,16 +25,30 @@ struct AsyncFlatMap Bool { - let nextResult = await wrapToResult(nil, &iterator) + let nextResult:Result? + do { + let value = if #available(macOS 15.0, iOS 18.0, watchOS 11.0, tvOS 18.0, visionOS 2.0, *) { + try await iterator.next(isolation: #isolation) + } else { + try await iterator.next() + } + if let value { + nextResult = .success(value) + } else { + nextResult = nil + } + } catch { + nextResult = .failure(error) + } switch nextResult { case .none: //finished @@ -310,7 +346,7 @@ extension AsyncFlatMap { } return false case .failure(let error): - send(completion: .failure(.segment(error))) + send(completion: .failure(.segment(error as! SegmentFail))) throw CancellationError() case .success(let value): try send(value) diff --git a/Sources/Tetra/Concurrency/AsyncSequencePublisher.swift b/Sources/Tetra/Concurrency/AsyncSequencePublisher.swift index 33e5259..be52e97 100644 --- a/Sources/Tetra/Concurrency/AsyncSequencePublisher.swift +++ b/Sources/Tetra/Concurrency/AsyncSequencePublisher.swift @@ -20,24 +20,29 @@ public extension AsyncSequence where Self:Sendable { public extension TetraExtension where Base: AsyncSequence & Sendable { @inlinable - var publisher:AsyncSequencePublisher { - .init(base: base) + var publisher:AsyncSequencePublisher { + .init(legacy: base) } } -public struct AsyncSequencePublisher: Publisher { +public struct AsyncSequencePublisher: Publisher { public typealias Output = Base.Element - public typealias Failure = Base.Failure public var base:Base + @available(macOS 15.0, iOS 18.0, watchOS 11.0, tvOS 18.0, visionOS 2.0, *) @inlinable - public init(base: Base) { + public init(base: Base) where Base.Failure == Failure { self.base = base } + @inlinable + public init(legacy: Base) where Failure == any Error { + self.base = legacy + } + public func receive(subscriber: S) where S : Subscriber, Failure == S.Failure, Base.Element == S.Input { let processor = Inner(subscriber: subscriber) let task = Task { [base] in @@ -48,8 +53,10 @@ public struct AsyncSequencePublisher: Publisher } + extension AsyncSequencePublisher: Sendable where Base: Sendable, Base.Element: Sendable {} + extension AsyncSequencePublisher { internal struct TaskState where S.Input == Output, S.Failure == Failure { @@ -137,13 +144,28 @@ extension AsyncSequencePublisher { for await var pending in demandSource.stream { while pending > .none { pending -= 1 - guard let result = await wrapToResult(nil, &iterator) else { + let result:Result? + do { + let value = if #available(macOS 15.0, iOS 18.0, watchOS 11.0, tvOS 18.0, visionOS 2.0, *) { + try await iterator.next(isolation: #isolation) + } else { + try await iterator.next() + } + if let value { + result = .success(value) + } else { + result = nil + } + } catch { + result = .failure(error) + } + guard let result else { send(completion: .finished) return } switch result { case .failure(let error): - send(completion: .failure(error)) + send(completion: .failure(error as! Failure)) return case .success(let value): if let newDemand = send(value) { diff --git a/Sources/Tetra/Concurrency/AsyncTypedSequence.swift b/Sources/Tetra/Concurrency/AsyncTypedSequence.swift index 72fec7d..10697ef 100644 --- a/Sources/Tetra/Concurrency/AsyncTypedSequence.swift +++ b/Sources/Tetra/Concurrency/AsyncTypedSequence.swift @@ -12,6 +12,7 @@ import Foundation public protocol AsyncTypedSequence:AsyncSequence {} + @available(iOS 15.0, macOS 12.0, macCatalyst 15.0, tvOS 15.0, watchOS 8.0, *) extension AsyncThrowingPublisher: AsyncTypedSequence {} @available(iOS 15.0, tvOS 15.0, macCatalyst 15.0, watchOS 8.0, macOS 12.0, *) diff --git a/Sources/Tetra/Concurrency/Notification+AsyncSequence.swift b/Sources/Tetra/Concurrency/Notification+AsyncSequence.swift index b9f88b0..11c0ff8 100644 --- a/Sources/Tetra/Concurrency/Notification+AsyncSequence.swift +++ b/Sources/Tetra/Concurrency/Notification+AsyncSequence.swift @@ -11,14 +11,12 @@ import _Concurrency extension NotificationCenter: TetraExtended {} + extension TetraExtension where Base: NotificationCenter { - func notifications(named: Notification.Name, object: AnyObject? = nil) -> some AsyncTypedSequence { - if #available(iOS 15.0, tvOS 15.0, macCatalyst 15.0, watchOS 8.0, macOS 12.0, *) { - return base.notifications(named: named, object: object) - } else { - return NotificationSequence(center: base, named: named, object: object) - } + @inlinable + func notifications(named: Notification.Name, object: AnyObject? = nil) -> NotificationSequence { + return NotificationSequence(center: base, named: named, object: object) } } @@ -28,7 +26,7 @@ extension TetraExtension where Base: NotificationCenter { public final class NotificationSequence: AsyncTypedSequence, Sendable { public typealias AsyncIterator = Iterator - + public typealias Failure = Never public func makeAsyncIterator() -> Iterator { Iterator(parent: self) @@ -86,13 +84,7 @@ public final class NotificationSequence: AsyncTypedSequence, Sendable { } return captured } - // just to suppress sendable warning - // it is unsafe to do this thing, since NotificationCenter broadcast and share Notifiaction among listeners, but for now there is no way to handle this clearly. - @inline(__always) - func resume(_ noti: sending Notification) { - continuation?.resume(returning: noti) - } - resume(noti2) + continuation?.resume(returning: noti2) } lock.withLockUnchecked{ $0.observer = observer @@ -120,7 +112,7 @@ public final class NotificationSequence: AsyncTypedSequence, Sendable { } func next(isolation: isolated (any Actor)?) async -> Notification? { - await withUnsafeContinuation(isolation: isolation) { continuation in + await withUnsafeContinuation { continuation in let (notification, isCancelled) = lock.withLockUnchecked { state in if !state.buffer.isEmpty { return (state.buffer.removeFirst() as Notification?, false) diff --git a/Sources/Tetra/Foundation/Mics.swift b/Sources/Tetra/Foundation/Mics.swift index 78376fa..5a1ec20 100644 --- a/Sources/Tetra/Foundation/Mics.swift +++ b/Sources/Tetra/Foundation/Mics.swift @@ -88,7 +88,7 @@ func wrapToResult(_ block: () throws(Failure) -> T) -> Result: @unchecked Sendable { + + var value:T + + @usableFromInline + init(value: T) { + self.value = value + } + +} diff --git a/Sources/Tetra/SwiftUI/Binding+Collection.swift b/Sources/Tetra/SwiftUI/Binding+Collection.swift index b610075..8c0955b 100644 --- a/Sources/Tetra/SwiftUI/Binding+Collection.swift +++ b/Sources/Tetra/SwiftUI/Binding+Collection.swift @@ -9,7 +9,7 @@ // import Foundation -import SwiftUI +@preconcurrency import SwiftUI @available(watchOS, deprecated: 8.0, message: "use Binding itself as Collection") @available(macOS, deprecated: 12.0, message: "use Binding itself as Collection") @@ -29,7 +29,7 @@ public extension Binding where Value: MutableCollection { @available(macCatalyst, deprecated: 15.0, message: "use Binding itself as Collection") @available(tvOS, deprecated: 15.0, message: "use Binding itself as Collection") @available(iOS, deprecated: 15.0, message: "use Binding itself as Collection") -public struct BindingCollection: Collection { +public struct BindingCollection: Collection, Sendable { @usableFromInline @Binding var collection:T @@ -43,11 +43,13 @@ public struct BindingCollection: Collection { if #available(iOS 15.0, tvOS 15.0, macCatalyst 15.0, macOS 12.0, watchOS 8.0, *) { return binding[position] } else { + nonisolated(unsafe) + let index = consume position return .init { - binding.wrappedValue[position] + binding.wrappedValue[index] } set: { newValue, transaction in withTransaction(transaction) { - binding.wrappedValue[position] = newValue + binding.wrappedValue[index] = newValue } } diff --git a/Sources/Tetra/SwiftUI/RefreshableScrollView.swift b/Sources/Tetra/SwiftUI/RefreshableScrollView.swift index d9bcae2..ccda360 100644 --- a/Sources/Tetra/SwiftUI/RefreshableScrollView.swift +++ b/Sources/Tetra/SwiftUI/RefreshableScrollView.swift @@ -81,56 +81,38 @@ public extension View { } @usableFromInline internal -struct RefreshActionModifier: EnvironmentalModifier { +struct RefreshActionModifier: ViewModifier { @Binding var task:Task? @Binding var refreshing:Bool + @Environment(\.self) private var environment @usableFromInline - func resolve(in environment: EnvironmentValues) -> ResolvedModifier { - var modifier = ResolvedModifier(task: $task, refreshing: refreshing) - if #available(iOS 15.0, tvOS 15.0, macOS 12.0, macCatalyst 15.0, watchOS 8.0, *), - let refresh = environment.refresh { - modifier.action = { - refreshing = true - await refresh() - refreshing = false - } - } else if let refresh = environment.refreshControl { - modifier.action = { - refreshing = true - await refresh.action() - refreshing = false - } - } - return modifier - } - - @usableFromInline - struct ResolvedModifier: ViewModifier { - @usableFromInline - @Binding var task:Task? - @usableFromInline - var refreshing:Bool - @usableFromInline - var action:( () async -> ())? - - @usableFromInline - func body(content: Content) -> some View { + func body(content: Content) -> some View { #if os(iOS) || targetEnvironment(macCatalyst) - content.background(Group{ + content.background( + Group{ + let action = if #available(iOS 15.0, tvOS 15.0, macOS 12.0, macCatalyst 15.0, watchOS 8.0, *) { + environment.refresh?.callAsFunction + } else { + environment.refreshControl?.action + } if let action { - ScrollRefreshImp(task: $task, refreshing: refreshing, operation: action) - .frame(width: 0, height: 0) + ScrollRefreshImp(task: $task, refreshing: refreshing) { + refreshing = true + defer { refreshing = false } + await action() + }.frame(width: 0, height: 0) } - }) + + } + ) #else - content + content #endif - } - } + } diff --git a/Tests/TetraTests/AsyncFlatMapTests.swift b/Tests/TetraTests/AsyncFlatMapTests.swift index 36925b3..9e0c802 100644 --- a/Tests/TetraTests/AsyncFlatMapTests.swift +++ b/Tests/TetraTests/AsyncFlatMapTests.swift @@ -190,7 +190,13 @@ final class AsyncFlatMapTests: XCTestCase { $0.finish() } }.mapError{ - $0.unwrap() + switch $0 { + case .transform(let error): + return error + case .segment(let error): + XCTFail("should not throw during segment") + return error + } }.sink { switch $0 { case .finished: @@ -221,7 +227,10 @@ final class AsyncFlatMapTests: XCTestCase { } } .mapError{ - $0.unwrap() + switch $0 { + case .segment(let error): + return error + } } .sink { switch $0 { diff --git a/Tests/TetraTests/AsyncSequencePublisherTests.swift b/Tests/TetraTests/AsyncSequencePublisherTests.swift index 19f1890..91962c1 100644 --- a/Tests/TetraTests/AsyncSequencePublisherTests.swift +++ b/Tests/TetraTests/AsyncSequencePublisherTests.swift @@ -22,7 +22,7 @@ class AsyncSequencePublisherTests: XCTestCase { source.forEach{ continuation.yield($0) } continuation.finish() } - let cancellable = AsyncSequencePublisher(base: stream) + let cancellable = AsyncSequencePublisher(legacy: stream) .catch{ _ in XCTFail() return Empty() @@ -59,7 +59,7 @@ class AsyncSequencePublisherTests: XCTestCase { } return value } - let pub = AsyncSequencePublisher(base: asyncSequence) + let pub = AsyncSequencePublisher(legacy: asyncSequence) .handleEvents( receiveCancel: { expect.fulfill() } ) @@ -88,7 +88,7 @@ class AsyncSequencePublisherTests: XCTestCase { } return value } - let cancellable = AsyncSequencePublisher(base: asyncSequence) + let cancellable = AsyncSequencePublisher(legacy: asyncSequence) .mapError{ $0 as! CancellationError } From 19664dceb789b6cb93b20c7221597befefd3d0fc Mon Sep 17 00:00:00 2001 From: park-byeong-gwan Date: Tue, 11 Jun 2024 11:55:53 +0900 Subject: [PATCH 25/63] ensure subscription deinitializer is called outside of lock --- .../Tetra/Combine/AsyncSubscriberState.swift | 23 ++++++++++++++----- .../Combine/AsyncSubscriptionState.swift | 7 +++++- 2 files changed, 23 insertions(+), 7 deletions(-) diff --git a/Sources/Tetra/Combine/AsyncSubscriberState.swift b/Sources/Tetra/Combine/AsyncSubscriberState.swift index 50fe620..b2ccbf9 100644 --- a/Sources/Tetra/Combine/AsyncSubscriberState.swift +++ b/Sources/Tetra/Combine/AsyncSubscriberState.swift @@ -56,23 +56,29 @@ struct AsyncSubscriberState { enum Effect { case resumeValue(Continuation, Input) - case resumeFailure([Continuation], Failure) + case resumeFailure([Continuation], Failure, discard: (any Subscription)?) case request(any Subscription, Subscribers.Demand) case cancel([Continuation], (any Subscription)?) + case cancelAndDiscard([Continuation], discard: (any Subscription)?) + case discard((any Subscription)?) @usableFromInline consuming func run() { switch consume self { + case .discard: + break case .resumeValue(let continuation, let input): nonisolated(unsafe) let value = Result.success(consume input) continuation.resume(returning: value) case .request(let subscription, let demand): subscription.request(demand) + case .cancelAndDiscard(let array, discard: _): + array.forEach{ $0.resume(returning: nil) } case .cancel(let array, let subscription): array.forEach{ $0.resume(returning: nil) } subscription?.cancel() - case .resumeFailure(let array, let failure): + case .resumeFailure(let array, let failure, _): array[0].resume(returning: .failure(failure)) array.dropFirst().forEach{ $0.resume(returning: nil) } } @@ -100,15 +106,20 @@ struct AsyncSubscriberState { private mutating func resume(completion: Subscribers.Completion) -> Effect? { let jobs = pending pending.removeAll() + let oldSubscription = if case let .subscribed(token) = subscription { + token + } else { + nil as (any Subscription)? + } switch subscription { case .awaitingSubscription, .subscribed: if !jobs.isEmpty { subscription = .terminal(nil) switch completion { case .finished: - return .cancel(jobs, nil) + return .cancelAndDiscard(jobs, discard: oldSubscription) case .failure(let failure): - return .resumeFailure(jobs, failure) + return .resumeFailure(jobs, failure, discard: oldSubscription) } } else { switch completion { @@ -117,7 +128,7 @@ struct AsyncSubscriberState { case .failure(let failure): subscription = .terminal(failure) } - return nil + return .discard(oldSubscription) } case .terminal: return .cancel(jobs, nil) @@ -149,7 +160,7 @@ struct AsyncSubscriberState { return .cancel([continuation], nil) case .terminal(let failure?): subscription = .terminal(nil) - return .resumeFailure([continuation], failure) + return .resumeFailure([continuation], failure, discard: nil) } } diff --git a/Sources/Tetra/Combine/AsyncSubscriptionState.swift b/Sources/Tetra/Combine/AsyncSubscriptionState.swift index deae9fc..9d6fde2 100644 --- a/Sources/Tetra/Combine/AsyncSubscriptionState.swift +++ b/Sources/Tetra/Combine/AsyncSubscriptionState.swift @@ -39,6 +39,8 @@ enum AsyncSubscriptionState { case resume(UnsafeContinuation) case raise(UnsafeContinuation) case cancel(any Subscription) + // ensure deinit is called outside of lock + case discard(any Subscription) consuming func run() { @@ -49,6 +51,8 @@ enum AsyncSubscriptionState { unsafeContinuation.resume(throwing: CancellationError()) case .cancel(let subscription): subscription.cancel() + case .discard: + break } } @@ -129,7 +133,8 @@ enum AsyncSubscriptionState { self = .finished return .resume(unsafeContinuation) case .cached(let subscription): - fallthrough + self = .finished + return .discard(subscription) case .waiting: self = .finished fallthrough From 44870059ac7525e6ef2ae6d851c408c2c83eefa7 Mon Sep 17 00:00:00 2001 From: park-byeong-gwan Date: Tue, 11 Jun 2024 18:51:45 +0900 Subject: [PATCH 26/63] AsyncFlatMap: adapting dedicated typedThrow --- .../Tetra/Combine/Combine+Concurrency.swift | 17 ++- .../Tetra/Combine/CompatAsyncPublisher.swift | 3 +- .../CompatAsyncThrowingPublisher.swift | 8 +- .../Tetra/Combine/ExperimentalMapTask.swift | 4 +- .../Combine/Publishers+AsyncFlatMap.swift | 62 +++++----- .../Tetra/Combine/Publishers+MapTask.swift | 8 +- .../Tetra/Combine/Publishers+TryMapTask.swift | 6 +- .../Concurrency/AsyncSequencePublisher.swift | 43 +++---- .../Concurrency/AsyncTypedSequence.swift | 42 +++++-- .../Notification+AsyncSequence.swift | 6 +- .../Concurrency/WrappedAsyncSequence.swift | 111 ++++++++++++++++++ Sources/Tetra/Foundation/Mics.swift | 24 ++-- Sources/Tetra/Foundation/Suppress.swift | 25 ++++ .../AsyncSequencePublisherTests.swift | 6 +- 14 files changed, 267 insertions(+), 98 deletions(-) create mode 100644 Sources/Tetra/Concurrency/WrappedAsyncSequence.swift diff --git a/Sources/Tetra/Combine/Combine+Concurrency.swift b/Sources/Tetra/Combine/Combine+Concurrency.swift index 8b5c603..fb66ba5 100644 --- a/Sources/Tetra/Combine/Combine+Concurrency.swift +++ b/Sources/Tetra/Combine/Combine+Concurrency.swift @@ -28,18 +28,25 @@ public extension TetraExtension where Base: Publisher { public extension Publisher { @inlinable - func mapTask(transform: @escaping @isolated(any) @Sendable (Output) async -> T) -> MapTask where Output:Sendable { + func mapTask( + transform: @escaping @isolated(any) @Sendable (Output) async -> sending T + ) -> some Publisher where Output:Sendable { MapTask(upstream: self, transform: transform) } @inlinable - func tryMapTask(transform: @escaping @isolated(any) @Sendable (Output) async throws -> T) -> TryMapTask where Output:Sendable { + func tryMapTask( + transform: @escaping @isolated(any) @Sendable (Output) async throws -> sending T + ) -> some Publisher where Output:Sendable { TryMapTask(upstream: self, transform: transform) } @_spi(Experimental) @inlinable - func multiMapTask(maxTasks: Subscribers.Demand = .max(1), transform: @escaping @Sendable @isolated(any) (Output) async throws(Self.Failure) -> T) -> MultiMapTask where Output: Sendable { + func multiMapTask( + maxTasks: Subscribers.Demand = .max(1), + transform: @escaping @Sendable @isolated(any) (Output) async throws(Failure) -> sending T + ) -> some Publisher where Output: Sendable { MultiMapTask(maxTasks: maxTasks, upstream: self, transform: transform) } @@ -51,8 +58,8 @@ internal extension Publisher { func asyncFlatMap( maxTasks: Subscribers.Demand = .unlimited, - transform: @escaping @Sendable @isolated(any) (Output) async throws(Err) -> Segment - ) -> AsyncFlatMap where Output:Sendable { + transform: @escaping @Sendable @isolated(any) (Output) async throws(Err) -> sending Segment + ) -> AsyncFlatMap, Err> where Output:Sendable { return AsyncFlatMap(maxTasks: maxTasks, upstream: self, transform: transform) } diff --git a/Sources/Tetra/Combine/CompatAsyncPublisher.swift b/Sources/Tetra/Combine/CompatAsyncPublisher.swift index 4a7a3f0..3d20dbc 100644 --- a/Sources/Tetra/Combine/CompatAsyncPublisher.swift +++ b/Sources/Tetra/Combine/CompatAsyncPublisher.swift @@ -27,7 +27,7 @@ public struct CompatAsyncPublisher: AsyncSequence where P.Failure = self.publisher = publisher } - public struct Iterator: AsyncIteratorProtocol { + public struct Iterator: TypedAsyncIteratorProtocol { public typealias Element = P.Output public typealias Failure = P.Failure @@ -42,6 +42,7 @@ public struct CompatAsyncPublisher: AsyncSequence where P.Failure = return await next(isolation: nil) } + @_implements(TypedAsyncIteratorProtocol, tetraNext(isolation:)) @inlinable public func next(isolation actor: isolated (any Actor)?) async -> P.Output? { let result: Result? = await withTaskCancellationHandler { [inner] in diff --git a/Sources/Tetra/Combine/CompatAsyncThrowingPublisher.swift b/Sources/Tetra/Combine/CompatAsyncThrowingPublisher.swift index 93ba68c..61b4401 100644 --- a/Sources/Tetra/Combine/CompatAsyncThrowingPublisher.swift +++ b/Sources/Tetra/Combine/CompatAsyncThrowingPublisher.swift @@ -9,10 +9,11 @@ import Foundation @preconcurrency import Combine -public struct CompatAsyncThrowingPublisher: AsyncTypedSequence { +public struct CompatAsyncThrowingPublisher: AsyncSequence { public typealias AsyncIterator = Iterator - public typealias Failure = Iterator.Failure + public typealias Failure = AsyncIterator.Failure + public var publisher:P @inlinable @@ -20,7 +21,7 @@ public struct CompatAsyncThrowingPublisher: AsyncTypedSequence { Iterator(source: publisher) } - public struct Iterator: AsyncIteratorProtocol { + public struct Iterator: TypedAsyncIteratorProtocol { public typealias Element = P.Output public typealias Failure = P.Failure @@ -29,6 +30,7 @@ public struct CompatAsyncThrowingPublisher: AsyncTypedSequence { @usableFromInline internal let reference:AnyCancellable + @_implements(TypedAsyncIteratorProtocol, tetraNext(isolation:)) @inlinable public mutating func next(isolation actor: isolated (any Actor)?) async throws(P.Failure) -> P.Output? { let result = await withTaskCancellationHandler { [inner] in diff --git a/Sources/Tetra/Combine/ExperimentalMapTask.swift b/Sources/Tetra/Combine/ExperimentalMapTask.swift index ba1f663..a4193ce 100644 --- a/Sources/Tetra/Combine/ExperimentalMapTask.swift +++ b/Sources/Tetra/Combine/ExperimentalMapTask.swift @@ -17,14 +17,14 @@ import Foundation precondition `maxTasks` must be none zero value */ @_spi(Experimental) -public struct MultiMapTask: Publisher where Upstream.Output:Sendable { +public struct MultiMapTask: Publisher where Upstream.Output:Sendable { public typealias Output = Output public typealias Failure = Upstream.Failure public let maxTasks:Subscribers.Demand public let upstream:Upstream - public let transform:@Sendable (Upstream.Output) async throws(Failure) -> Output + public let transform:@Sendable (Upstream.Output) async throws(Failure) -> sending Output public func receive(subscriber: S) where S : Subscriber, Upstream.Failure == S.Failure, Output == S.Input { let processor = Inner(maxTasks: maxTasks, subscriber: subscriber, transform: transform) diff --git a/Sources/Tetra/Combine/Publishers+AsyncFlatMap.swift b/Sources/Tetra/Combine/Publishers+AsyncFlatMap.swift index 272f336..a922d1f 100644 --- a/Sources/Tetra/Combine/Publishers+AsyncFlatMap.swift +++ b/Sources/Tetra/Combine/Publishers+AsyncFlatMap.swift @@ -9,16 +9,19 @@ import Foundation @preconcurrency import Combine -struct AsyncFlatMap: Publisher where Upstream.Output:Sendable { +struct AsyncFlatMap: Publisher where Upstream.Output:Sendable, Segment.AsyncIterator: TypedAsyncIteratorProtocol { typealias Output = Segment.Element - typealias Failure = AsyncFlatMapError - typealias Transform = @Sendable @isolated(any) (Upstream.Output) async throws(TransformFail) -> Segment + typealias Failure = AsyncFlatMapError + typealias Transform = @Sendable @isolated(any) (Upstream.Output) async throws(TransformFail) -> sending Segment let maxTasks:Subscribers.Demand let upstream:Upstream let transform:Transform func receive(subscriber: S) where S : Subscriber, Failure == S.Failure, Segment.Element == S.Input { + if #available(macOS 15.0, *) { + assert(Segment.Failure.self == Segment.AsyncIterator.TetraFailure.self, "assert") + } let processor = Inner(maxTasks: maxTasks, subscriber: subscriber, transform: transform) let task = Task(operation: processor.run) processor.resumeCondition(task) @@ -26,28 +29,45 @@ struct AsyncFlatMap( maxTasks: Subscribers.Demand, upstream: Upstream, - transform: @escaping @isolated(any) Transform - ) where SegmentFail == any Error { + transform: @escaping @isolated(any) @Sendable (Upstream.Output) async throws(TransformFail) -> sending Source + ) where Source: AsyncSequence, Segment == WrappedAsyncSequence, Segment.AsyncIterator.TetraFailure == any Error { + let block:Transform = { + return .init(base: try await transform($0)) + } self.maxTasks = maxTasks self.upstream = upstream - self.transform = transform + self.transform = block } - + +// @available(macOS 15.0, iOS 18.0, watchOS 11.0, tvOS 18.0, visionOS 2.0, *) +// @usableFromInline +// init( +// maxTasks: Subscribers.Demand, +// upstream: Upstream, +// typedTransform: @escaping @isolated(any) @Sendable (Upstream.Output) async throws(TransformFail) -> sending Source +// ) where Source: AsyncSequence, Segment == WrappedAsyncSequenceV2 { +// let block:Transform = { +// return .init(base: try await typedTransform($0)) +// } +// self.maxTasks = maxTasks +// self.upstream = upstream +// self.transform = block +// } + } @@ -56,7 +76,7 @@ extension AsyncFlatMap { struct Inner: Subscriber, Sendable, Subscription, CustomStringConvertible, CustomPlaygroundDisplayConvertible where Segment.Element == Down.Input, Down.Failure == AsyncFlatMap.Failure { - typealias Transformer = @Sendable (Upstream.Output) async throws(TransformFail) -> Segment + typealias Transformer = @Sendable (Upstream.Output) async throws(TransformFail) -> sending Segment typealias Input = Upstream.Output typealias Failure = Upstream.Failure @@ -271,7 +291,7 @@ extension AsyncFlatMap { valueSource.continuation.finish() } - private func makeSegment(_ input:Upstream.Output) async throws(CancellationError) -> Segment { + private func makeSegment(_ input:Upstream.Output) async throws(CancellationError) -> sending Segment { let result:Result do { let seg = try await transform(input) @@ -306,21 +326,7 @@ extension AsyncFlatMap { private func processNextSegment( iterator: inout Segment.AsyncIterator ) async throws(CancellationError) -> Bool { - let nextResult:Result? - do { - let value = if #available(macOS 15.0, iOS 18.0, watchOS 11.0, tvOS 18.0, visionOS 2.0, *) { - try await iterator.next(isolation: #isolation) - } else { - try await iterator.next() - } - if let value { - nextResult = .success(value) - } else { - nextResult = nil - } - } catch { - nextResult = .failure(error) - } + let nextResult = await wrapToResult(#isolation, &iterator) switch nextResult { case .none: //finished @@ -346,7 +352,7 @@ extension AsyncFlatMap { } return false case .failure(let error): - send(completion: .failure(.segment(error as! SegmentFail))) + send(completion: .failure(.segment(error))) throw CancellationError() case .success(let value): try send(value) diff --git a/Sources/Tetra/Combine/Publishers+MapTask.swift b/Sources/Tetra/Combine/Publishers+MapTask.swift index 22d1527..0647cf9 100644 --- a/Sources/Tetra/Combine/Publishers+MapTask.swift +++ b/Sources/Tetra/Combine/Publishers+MapTask.swift @@ -33,17 +33,17 @@ import Foundation ``` */ -public struct MapTask: Publisher where Upstream.Output:Sendable { +public struct MapTask: Publisher where Upstream.Output:Sendable { public typealias Output = Output public typealias Failure = Upstream.Failure public let upstream:Upstream - public var transform:@Sendable @isolated(any) (Upstream.Output) async -> Result + public var transform:@Sendable @isolated(any) (Upstream.Output) async -> sending Result public init( upstream: Upstream, - transform: @escaping @Sendable @isolated(any) (Upstream.Output) async -> Output + transform: @escaping @Sendable @isolated(any) (Upstream.Output) async -> sending Output ) { self.upstream = upstream self.transform = { @@ -53,7 +53,7 @@ public struct MapTask: Publisher where Upst public init( upstream: Upstream, - handler: @escaping @Sendable @isolated(any) (Upstream.Output) async -> Result + handler: @escaping @Sendable @isolated(any) (Upstream.Output) async -> sending Result ) { self.upstream = upstream self.transform = handler diff --git a/Sources/Tetra/Combine/Publishers+TryMapTask.swift b/Sources/Tetra/Combine/Publishers+TryMapTask.swift index 3ebea49..79a40cb 100644 --- a/Sources/Tetra/Combine/Publishers+TryMapTask.swift +++ b/Sources/Tetra/Combine/Publishers+TryMapTask.swift @@ -30,17 +30,17 @@ import _Concurrency ``` */ -public struct TryMapTask: Publisher where Upstream.Output:Sendable { +public struct TryMapTask: Publisher where Upstream.Output:Sendable { public typealias Output = Output public typealias Failure = any Error public let upstream:Upstream - public var transform: @isolated(any) @Sendable (Upstream.Output) async throws -> Output + public var transform: @isolated(any) @Sendable (Upstream.Output) async throws -> sending Output public init( upstream: Upstream, - transform: @escaping @isolated(any) @Sendable (Upstream.Output) async throws -> Output + transform: @escaping @isolated(any) @Sendable (Upstream.Output) async throws -> sending Output ) { self.upstream = upstream self.transform = transform diff --git a/Sources/Tetra/Concurrency/AsyncSequencePublisher.swift b/Sources/Tetra/Concurrency/AsyncSequencePublisher.swift index be52e97..b9279d0 100644 --- a/Sources/Tetra/Concurrency/AsyncSequencePublisher.swift +++ b/Sources/Tetra/Concurrency/AsyncSequencePublisher.swift @@ -20,27 +20,34 @@ public extension AsyncSequence where Self:Sendable { public extension TetraExtension where Base: AsyncSequence & Sendable { @inlinable - var publisher:AsyncSequencePublisher { - .init(legacy: base) + var publisher:some Publisher { + AsyncSequencePublisher(base: base) } } -public struct AsyncSequencePublisher: Publisher { + +public struct AsyncSequencePublisher: Publisher where Base.AsyncIterator: TypedAsyncIteratorProtocol { + public typealias Output = Base.Element + public typealias Failure = Base.AsyncIterator.TetraFailure public var base:Base - @available(macOS 15.0, iOS 18.0, watchOS 11.0, tvOS 18.0, visionOS 2.0, *) - @inlinable - public init(base: Base) where Base.Failure == Failure { + public init(base: Base) { self.base = base } - @inlinable - public init(legacy: Base) where Failure == any Error { - self.base = legacy + @available(macOS 15.0, iOS 18.0, watchOS 11.0, tvOS 18.0, visionOS 2.0, *) + public init(base:Source) where WrappedAsyncSequenceV2 == Base { + let source = WrappedAsyncSequenceV2(base: base) + self.base = source + } + + public init(base: Source) where Failure == any Error, WrappedAsyncSequence == Base { + let source = WrappedAsyncSequence(base: base) + self.base = source } public func receive(subscriber: S) where S : Subscriber, Failure == S.Failure, Base.Element == S.Input { @@ -144,28 +151,14 @@ extension AsyncSequencePublisher { for await var pending in demandSource.stream { while pending > .none { pending -= 1 - let result:Result? - do { - let value = if #available(macOS 15.0, iOS 18.0, watchOS 11.0, tvOS 18.0, visionOS 2.0, *) { - try await iterator.next(isolation: #isolation) - } else { - try await iterator.next() - } - if let value { - result = .success(value) - } else { - result = nil - } - } catch { - result = .failure(error) - } + let result = await wrapToResult(#isolation, &iterator) guard let result else { send(completion: .finished) return } switch result { case .failure(let error): - send(completion: .failure(error as! Failure)) + send(completion: .failure(error)) return case .success(let value): if let newDemand = send(value) { diff --git a/Sources/Tetra/Concurrency/AsyncTypedSequence.swift b/Sources/Tetra/Concurrency/AsyncTypedSequence.swift index 10697ef..dcdaca6 100644 --- a/Sources/Tetra/Concurrency/AsyncTypedSequence.swift +++ b/Sources/Tetra/Concurrency/AsyncTypedSequence.swift @@ -10,17 +10,33 @@ import _Concurrency import Foundation -public protocol AsyncTypedSequence:AsyncSequence {} - - -@available(iOS 15.0, macOS 12.0, macCatalyst 15.0, tvOS 15.0, watchOS 8.0, *) -extension AsyncThrowingPublisher: AsyncTypedSequence {} -@available(iOS 15.0, tvOS 15.0, macCatalyst 15.0, watchOS 8.0, macOS 12.0, *) -extension AsyncPublisher: AsyncTypedSequence {} - -extension AsyncThrowingStream: AsyncTypedSequence {} -extension AsyncStream: AsyncTypedSequence {} - -@available(iOS 15.0, tvOS 15.0, macCatalyst 15.0, watchOS 8.0, macOS 12.0, *) -extension NotificationCenter.Notifications: AsyncTypedSequence {} +/// Entry point of TypeFailure for `AsyncFlatMap` and `AsyncSequencePublisher +/// +/// Since generalized `AsyncSequence` is not available until Swift 6, this `protocol` is a entry point for async typed throw. +/// +///Even though it has isolation parameter, this parameter is rarely used since `AsyncFlatMap` and `AsyncSequencePublisher` runs in nonisolated Task space. +/// +/// `Use `WrappedAsyncSequence` or `WrappedAsyncSequenceV2` if possible which provides general implementation to adapt this protocol +/// +/// - postcondition: `TetraFailure` must be same as `Failure` +public protocol TypedAsyncIteratorProtocol: AsyncIteratorProtocol { + + associatedtype TetraFailure: Error = any Error + + @inlinable + mutating func tetraNext(isolation actor: isolated (any Actor)?) async throws(TetraFailure) -> Element? + +} + + + +@available(macOS 15.0, iOS 18.0, watchOS 11.0, tvOS 18.0, visionOS 2.0, *) +public extension TypedAsyncIteratorProtocol where TetraFailure == Failure { + + @inlinable + mutating func tetraNext(isolation actor: isolated (any Actor)?) async throws(Failure) -> Element? { + return try await next(isolation: actor) + } + +} diff --git a/Sources/Tetra/Concurrency/Notification+AsyncSequence.swift b/Sources/Tetra/Concurrency/Notification+AsyncSequence.swift index 11c0ff8..02cd7a8 100644 --- a/Sources/Tetra/Concurrency/Notification+AsyncSequence.swift +++ b/Sources/Tetra/Concurrency/Notification+AsyncSequence.swift @@ -23,7 +23,7 @@ extension TetraExtension where Base: NotificationCenter { -public final class NotificationSequence: AsyncTypedSequence, Sendable { +public final class NotificationSequence: AsyncSequence, Sendable { public typealias AsyncIterator = Iterator public typealias Failure = Never @@ -35,7 +35,7 @@ public final class NotificationSequence: AsyncTypedSequence, Sendable { let center: NotificationCenter private let lock:some UnfairStateLock = createUncheckedStateLock(uncheckedState: NotficationState()) - public struct Iterator: AsyncIteratorProtocol { + public struct Iterator: TypedAsyncIteratorProtocol { public typealias Element = Notification public typealias Failure = Never @@ -45,6 +45,7 @@ public final class NotificationSequence: AsyncTypedSequence, Sendable { await next(isolation: nil) } + @_implements(TypedAsyncIteratorProtocol, tetraNext(isolation:)) public func next(isolation actor: isolated (any Actor)?) async throws(Never) -> Notification? { // next를 호출한 동안에 task cancellation이 발생하면 observer Token이 무효화되는 것이 확인되므로 아래와 같이 canellation을 추가한다. await withTaskCancellationHandler( @@ -69,7 +70,6 @@ public final class NotificationSequence: AsyncTypedSequence, Sendable { named name: Notification.Name, object: AnyObject? = nil ) { - self.center = center let observer = center.addObserver(forName: name, object: object, queue: nil) { [lock] notification in nonisolated(unsafe) diff --git a/Sources/Tetra/Concurrency/WrappedAsyncSequence.swift b/Sources/Tetra/Concurrency/WrappedAsyncSequence.swift new file mode 100644 index 0000000..a84c2e9 --- /dev/null +++ b/Sources/Tetra/Concurrency/WrappedAsyncSequence.swift @@ -0,0 +1,111 @@ +// +// WrappedAsyncSequence.swift +// +// +// Created by 박병관 on 6/11/24. +// + + +public struct WrappedAsyncSequence: AsyncSequence { + + public typealias AsyncIterator = Iterator + + public typealias Element = Base.Element + public typealias Failure = any Error + + @usableFromInline + var base:Base + + @inlinable + public func makeAsyncIterator() -> AsyncIterator { + Iterator(base: base.makeAsyncIterator()) + } + + public struct Iterator: AsyncIteratorProtocol, TypedAsyncIteratorProtocol { + + public typealias Element = Base.Element + + public typealias Failure = any Error + + @usableFromInline + var base:Base.AsyncIterator + + @inlinable + public mutating func next() async throws -> Base.Element? { + try await base.next() + } + + @inlinable + @_implements(TypedAsyncIteratorProtocol, tetraNext(isolation:)) + public mutating func next(isolation actor: isolated (any Actor)?) async throws -> Base.Element? { + if #available(macOS 15.0, iOS 18.0, watchOS 11.0, tvOS 18.0, visionOS 2.0, *) { + return try await base.next(isolation: actor) + } else { + return try await base.advanceUnsafe() + } + } + + @usableFromInline + init(base: Base.AsyncIterator) { + self.base = base + } + + + } + + @inlinable + public init(base: Base) { + self.base = base + } + +} + +extension WrappedAsyncSequence: Sendable where Base:Sendable {} + +@available(macOS 15.0, iOS 18.0, watchOS 11.0, tvOS 18.0, visionOS 2.0, *) +public struct WrappedAsyncSequenceV2: AsyncSequence { + + public typealias Element = Base.Element + public typealias Failure = Base.Failure + + @inlinable + public func makeAsyncIterator() -> Iterator { + return Iterator(base: base.makeAsyncIterator()) + } + + @usableFromInline + var base:Base + + public struct Iterator: TypedAsyncIteratorProtocol { + + + @usableFromInline + var base:Base.AsyncIterator + + @inlinable + @_implements(TypedAsyncIteratorProtocol, tetraNext(isolation:)) + public mutating func next(isolation actor: isolated (any Actor)?) async throws(Base.Failure) -> Base.Element? { + return try await base.next(isolation: actor) + } + + @inlinable + public mutating func next() async throws -> Base.Element? { + return try await base.next() + } + + @usableFromInline + init(base: Base.AsyncIterator) { + self.base = base + } + + } + + @inlinable + public init(base: Base) { + self.base = base + } + +} + +@available(macOS 15.0, iOS 18.0, watchOS 11.0, tvOS 18.0, visionOS 2.0, *) +extension WrappedAsyncSequenceV2: Sendable where Base: Sendable {} diff --git a/Sources/Tetra/Foundation/Mics.swift b/Sources/Tetra/Foundation/Mics.swift index 5a1ec20..e3fe443 100644 --- a/Sources/Tetra/Foundation/Mics.swift +++ b/Sources/Tetra/Foundation/Mics.swift @@ -88,21 +88,29 @@ func wrapToResult(_ block: () throws(Failure) -> T) -> Result( +func wrapToResult( _ actor: isolated (any Actor)?, _ iterator: inout Base -) async -> Result? { +) async -> sending Result? { do { - let value = try await iterator.next(isolation: actor) - if let value { + if let value = try await iterator.tetraNext(isolation: actor) { return .success(value) - } else { - return nil } + return nil +// if #available(macOS 15.0, iOS 18.0, watchOS 11.0, tvOS 18.0, visionOS 2.0, *) { +// let value = try await iterator.next(isolation: actor) +// if let value { +// return .success(value) +// } else { +// return nil +// } +// } else { +// let result = try await iterator.next() +// return result +// } } catch { return .failure(error) } @@ -111,7 +119,7 @@ func wrapToResult( @inline(__always) @usableFromInline -internal func wrapToResult(_ value:T, _ transform: (T) async throws(Failure) -> U) async -> Result { +internal func wrapToResult(_ value:T, _ transform: (T) async throws(Failure) -> U) async -> sending Result { do { let success = try await transform(value) return .success(success) diff --git a/Sources/Tetra/Foundation/Suppress.swift b/Sources/Tetra/Foundation/Suppress.swift index d1928a2..ffcbf7c 100644 --- a/Sources/Tetra/Foundation/Suppress.swift +++ b/Sources/Tetra/Foundation/Suppress.swift @@ -5,10 +5,13 @@ // Created by 박병관 on 6/11/24. // +import Foundation + // just using to suppress sendable check for unsafe concurrent operation @usableFromInline struct Suppress: @unchecked Sendable { + @usableFromInline var value:T @usableFromInline @@ -17,3 +20,25 @@ struct Suppress: @unchecked Sendable { } } + +extension AsyncIteratorProtocol { + + + @usableFromInline + mutating func advanceUnsafe() async throws -> sending Element? { + nonisolated(unsafe) + var unsafe = Suppress(value: self) + defer { self = unsafe.value } + return try await unsafe.value.next() + } + +} + +extension Suppress where T: AsyncIteratorProtocol { + + @usableFromInline + mutating func advanceUnsafe() async throws -> sending T.Element? { + return try await value.next() + } + +} diff --git a/Tests/TetraTests/AsyncSequencePublisherTests.swift b/Tests/TetraTests/AsyncSequencePublisherTests.swift index 91962c1..19f1890 100644 --- a/Tests/TetraTests/AsyncSequencePublisherTests.swift +++ b/Tests/TetraTests/AsyncSequencePublisherTests.swift @@ -22,7 +22,7 @@ class AsyncSequencePublisherTests: XCTestCase { source.forEach{ continuation.yield($0) } continuation.finish() } - let cancellable = AsyncSequencePublisher(legacy: stream) + let cancellable = AsyncSequencePublisher(base: stream) .catch{ _ in XCTFail() return Empty() @@ -59,7 +59,7 @@ class AsyncSequencePublisherTests: XCTestCase { } return value } - let pub = AsyncSequencePublisher(legacy: asyncSequence) + let pub = AsyncSequencePublisher(base: asyncSequence) .handleEvents( receiveCancel: { expect.fulfill() } ) @@ -88,7 +88,7 @@ class AsyncSequencePublisherTests: XCTestCase { } return value } - let cancellable = AsyncSequencePublisher(legacy: asyncSequence) + let cancellable = AsyncSequencePublisher(base: asyncSequence) .mapError{ $0 as! CancellationError } From 5ae5fa7ae002a2e754db2ca6055a984d95033bf4 Mon Sep 17 00:00:00 2001 From: park-byeong-gwan Date: Tue, 11 Jun 2024 22:34:53 +0900 Subject: [PATCH 27/63] AsyncFlatMap: changing genernal protocol for better type conformance and reduce boliler plating --- .../Tetra/Combine/CompatAsyncPublisher.swift | 10 +-- .../CompatAsyncThrowingPublisher.swift | 9 +-- .../Combine/Publishers+AsyncFlatMap.swift | 37 +++++------ .../Concurrency/AsyncSequencePublisher.swift | 17 ++--- .../Concurrency/AsyncTypedSequence.swift | 65 ++++++++++++++----- .../Notification+AsyncSequence.swift | 7 +- .../Concurrency/WrappedAsyncSequence.swift | 19 +----- Sources/Tetra/Foundation/Mics.swift | 6 +- 8 files changed, 86 insertions(+), 84 deletions(-) diff --git a/Sources/Tetra/Combine/CompatAsyncPublisher.swift b/Sources/Tetra/Combine/CompatAsyncPublisher.swift index 3d20dbc..b68f063 100644 --- a/Sources/Tetra/Combine/CompatAsyncPublisher.swift +++ b/Sources/Tetra/Combine/CompatAsyncPublisher.swift @@ -27,7 +27,7 @@ public struct CompatAsyncPublisher: AsyncSequence where P.Failure = self.publisher = publisher } - public struct Iterator: TypedAsyncIteratorProtocol { + public struct Iterator: AsyncIteratorProtocol, TypedAsyncIteratorProtocol { public typealias Element = P.Output public typealias Failure = P.Failure @@ -36,13 +36,7 @@ public struct CompatAsyncPublisher: AsyncSequence where P.Failure = internal let inner = AsyncSubscriber

() @usableFromInline internal let reference:AnyCancellable - - @inlinable - public mutating func next() async -> P.Output? { - return await next(isolation: nil) - } - - @_implements(TypedAsyncIteratorProtocol, tetraNext(isolation:)) + @inlinable public func next(isolation actor: isolated (any Actor)?) async -> P.Output? { let result: Result? = await withTaskCancellationHandler { [inner] in diff --git a/Sources/Tetra/Combine/CompatAsyncThrowingPublisher.swift b/Sources/Tetra/Combine/CompatAsyncThrowingPublisher.swift index 61b4401..6cd0cf3 100644 --- a/Sources/Tetra/Combine/CompatAsyncThrowingPublisher.swift +++ b/Sources/Tetra/Combine/CompatAsyncThrowingPublisher.swift @@ -9,7 +9,7 @@ import Foundation @preconcurrency import Combine -public struct CompatAsyncThrowingPublisher: AsyncSequence { +public struct CompatAsyncThrowingPublisher: AsyncSequence, TypedAsyncSequence { public typealias AsyncIterator = Iterator public typealias Failure = AsyncIterator.Failure @@ -21,7 +21,7 @@ public struct CompatAsyncThrowingPublisher: AsyncSequence { Iterator(source: publisher) } - public struct Iterator: TypedAsyncIteratorProtocol { + public struct Iterator: AsyncIteratorProtocol, TypedAsyncIteratorProtocol { public typealias Element = P.Output public typealias Failure = P.Failure @@ -30,7 +30,6 @@ public struct CompatAsyncThrowingPublisher: AsyncSequence { @usableFromInline internal let reference:AnyCancellable - @_implements(TypedAsyncIteratorProtocol, tetraNext(isolation:)) @inlinable public mutating func next(isolation actor: isolated (any Actor)?) async throws(P.Failure) -> P.Output? { let result = await withTaskCancellationHandler { [inner] in @@ -49,10 +48,6 @@ public struct CompatAsyncThrowingPublisher: AsyncSequence { } } - public mutating func next() async throws(Failure) -> P.Output? { - try await next(isolation: nil) - } - @usableFromInline internal init(source: P) { self.reference = AnyCancellable(inner) diff --git a/Sources/Tetra/Combine/Publishers+AsyncFlatMap.swift b/Sources/Tetra/Combine/Publishers+AsyncFlatMap.swift index a922d1f..60d7b7b 100644 --- a/Sources/Tetra/Combine/Publishers+AsyncFlatMap.swift +++ b/Sources/Tetra/Combine/Publishers+AsyncFlatMap.swift @@ -9,19 +9,16 @@ import Foundation @preconcurrency import Combine -struct AsyncFlatMap: Publisher where Upstream.Output:Sendable, Segment.AsyncIterator: TypedAsyncIteratorProtocol { +struct AsyncFlatMap: Publisher where Upstream.Output:Sendable{ typealias Output = Segment.Element - typealias Failure = AsyncFlatMapError + typealias Failure = AsyncFlatMapError typealias Transform = @Sendable @isolated(any) (Upstream.Output) async throws(TransformFail) -> sending Segment let maxTasks:Subscribers.Demand let upstream:Upstream let transform:Transform func receive(subscriber: S) where S : Subscriber, Failure == S.Failure, Segment.Element == S.Input { - if #available(macOS 15.0, *) { - assert(Segment.Failure.self == Segment.AsyncIterator.TetraFailure.self, "assert") - } let processor = Inner(maxTasks: maxTasks, subscriber: subscriber, transform: transform) let task = Task(operation: processor.run) processor.resumeCondition(task) @@ -44,7 +41,7 @@ struct AsyncFlatMap sending Source - ) where Source: AsyncSequence, Segment == WrappedAsyncSequence, Segment.AsyncIterator.TetraFailure == any Error { + ) where Source: AsyncSequence, Segment == WrappedAsyncSequence, Segment.AsyncIterator.Failure == any Error { let block:Transform = { return .init(base: try await transform($0)) } @@ -53,20 +50,20 @@ struct AsyncFlatMap( -// maxTasks: Subscribers.Demand, -// upstream: Upstream, -// typedTransform: @escaping @isolated(any) @Sendable (Upstream.Output) async throws(TransformFail) -> sending Source -// ) where Source: AsyncSequence, Segment == WrappedAsyncSequenceV2 { -// let block:Transform = { -// return .init(base: try await typedTransform($0)) -// } -// self.maxTasks = maxTasks -// self.upstream = upstream -// self.transform = block -// } + @available(macOS 15.0, iOS 18.0, watchOS 11.0, tvOS 18.0, visionOS 2.0, *) + @usableFromInline + init( + maxTasks: Subscribers.Demand, + upstream: Upstream, + typedTransform: @escaping @isolated(any) @Sendable (Upstream.Output) async throws(TransformFail) -> sending Source + ) where Source: AsyncSequence, Segment == WrappedAsyncSequenceV2 { + let block:Transform = { + return .init(base: try await typedTransform($0)) + } + self.maxTasks = maxTasks + self.upstream = upstream + self.transform = block + } } diff --git a/Sources/Tetra/Concurrency/AsyncSequencePublisher.swift b/Sources/Tetra/Concurrency/AsyncSequencePublisher.swift index b9279d0..8d7f040 100644 --- a/Sources/Tetra/Concurrency/AsyncSequencePublisher.swift +++ b/Sources/Tetra/Concurrency/AsyncSequencePublisher.swift @@ -19,19 +19,20 @@ public extension AsyncSequence where Self:Sendable { public extension TetraExtension where Base: AsyncSequence & Sendable { - @inlinable - var publisher:some Publisher { - AsyncSequencePublisher(base: base) - } +// @inlinable +// var publisher:some Publisher { +// AsyncSequencePublisher(base: base) +// } } -public struct AsyncSequencePublisher: Publisher where Base.AsyncIterator: TypedAsyncIteratorProtocol { +internal struct AsyncSequencePublisher: Publisher { + + public typealias Output = Base.AsyncIterator.Element - public typealias Output = Base.Element - public typealias Failure = Base.AsyncIterator.TetraFailure + public typealias Failure = Base.AsyncIterator.Failure public var base:Base @@ -50,7 +51,7 @@ public struct AsyncSequencePublisher: Publisher self.base = source } - public func receive(subscriber: S) where S : Subscriber, Failure == S.Failure, Base.Element == S.Input { + public func receive(subscriber: S) where S : Subscriber, Failure == S.Failure, Base.AsyncIterator.Element == S.Input { let processor = Inner(subscriber: subscriber) let task = Task { [base] in await processor.run(base) diff --git a/Sources/Tetra/Concurrency/AsyncTypedSequence.swift b/Sources/Tetra/Concurrency/AsyncTypedSequence.swift index dcdaca6..b73d0ac 100644 --- a/Sources/Tetra/Concurrency/AsyncTypedSequence.swift +++ b/Sources/Tetra/Concurrency/AsyncTypedSequence.swift @@ -9,34 +9,67 @@ import Combine import _Concurrency import Foundation +/*** + Entry point of TypeFailure for `AsyncFlatMap` and `AsyncSequencePublisher -/// Entry point of TypeFailure for `AsyncFlatMap` and `AsyncSequencePublisher -/// -/// Since generalized `AsyncSequence` is not available until Swift 6, this `protocol` is a entry point for async typed throw. -/// -///Even though it has isolation parameter, this parameter is rarely used since `AsyncFlatMap` and `AsyncSequencePublisher` runs in nonisolated Task space. -/// -/// `Use `WrappedAsyncSequence` or `WrappedAsyncSequenceV2` if possible which provides general implementation to adapt this protocol -/// -/// - postcondition: `TetraFailure` must be same as `Failure` -public protocol TypedAsyncIteratorProtocol: AsyncIteratorProtocol { + Since generalized `AsyncSequence` is not available until Swift 6, this `protocol` is a entry point for async typed throw. + +Even though it has isolation parameter, this parameter is rarely used since `AsyncFlatMap` and `AsyncSequencePublisher` runs in nonisolated Task space. + +`Use `WrappedAsyncSequence` or `WrappedAsyncSequenceV2` if possible which provides general implementation to adapt this protocol + + - postcondition: `Failure` must be same as `AsyncIterator.Failure` + + + - adapting protocol to existing type, + + adapting protocols to exisiting type that user don't own is pretty easy too. By writing conformance like below, compiler now know the correct implemenation without recursive problem. +```` + extension AsyncStream.Iterator: TypedAsyncIteratorProtocol { + + @_implements(TypedAsyncIteratorProtocol, next(isolation:)) + mutating public func tetraNext(isolation actor: isolated (any Actor)?) async throws(Never) -> Self.Element? { + if #available(macOS 15.0, iOS 18.0, watchOS 11.0, tvOS 18.0, visionOS 2.0, *) { + var c: some AsyncIteratorProtocol = self + defer { + self = c as! Self + } + return await c.next(isolation: actor) + } else { + return try? await next() + } + } + + } + + ```` + */ +public protocol TypedAsyncIteratorProtocol { - associatedtype TetraFailure: Error = any Error + associatedtype Element + associatedtype Failure: Error + @inlinable - mutating func tetraNext(isolation actor: isolated (any Actor)?) async throws(TetraFailure) -> Element? + mutating func next(isolation actor: isolated (any Actor)?) async throws(Failure) -> Element? } +public protocol TypedAsyncSequence: AsyncSequence where AsyncIterator: TypedAsyncIteratorProtocol {} -@available(macOS 15.0, iOS 18.0, watchOS 11.0, tvOS 18.0, visionOS 2.0, *) -public extension TypedAsyncIteratorProtocol where TetraFailure == Failure { +public protocol TypedAsyncIteratorProtocol2: AsyncIteratorProtocol, TypedAsyncIteratorProtocol { +} + + +public extension TypedAsyncIteratorProtocol { @inlinable - mutating func tetraNext(isolation actor: isolated (any Actor)?) async throws(Failure) -> Element? { - return try await next(isolation: actor) + mutating func next() async throws(Failure) -> Element? { + try await next(isolation: nil) } } + + diff --git a/Sources/Tetra/Concurrency/Notification+AsyncSequence.swift b/Sources/Tetra/Concurrency/Notification+AsyncSequence.swift index 02cd7a8..2469cc3 100644 --- a/Sources/Tetra/Concurrency/Notification+AsyncSequence.swift +++ b/Sources/Tetra/Concurrency/Notification+AsyncSequence.swift @@ -35,17 +35,12 @@ public final class NotificationSequence: AsyncSequence, Sendable { let center: NotificationCenter private let lock:some UnfairStateLock = createUncheckedStateLock(uncheckedState: NotficationState()) - public struct Iterator: TypedAsyncIteratorProtocol { + public struct Iterator: AsyncIteratorProtocol, TypedAsyncIteratorProtocol { public typealias Element = Notification public typealias Failure = Never let parent:NotificationSequence - - public func next() async -> Notification? { - await next(isolation: nil) - } - @_implements(TypedAsyncIteratorProtocol, tetraNext(isolation:)) public func next(isolation actor: isolated (any Actor)?) async throws(Never) -> Notification? { // next를 호출한 동안에 task cancellation이 발생하면 observer Token이 무효화되는 것이 확인되므로 아래와 같이 canellation을 추가한다. await withTaskCancellationHandler( diff --git a/Sources/Tetra/Concurrency/WrappedAsyncSequence.swift b/Sources/Tetra/Concurrency/WrappedAsyncSequence.swift index a84c2e9..466ce97 100644 --- a/Sources/Tetra/Concurrency/WrappedAsyncSequence.swift +++ b/Sources/Tetra/Concurrency/WrappedAsyncSequence.swift @@ -6,7 +6,7 @@ // -public struct WrappedAsyncSequence: AsyncSequence { +public struct WrappedAsyncSequence: AsyncSequence, TypedAsyncSequence { public typealias AsyncIterator = Iterator @@ -29,14 +29,8 @@ public struct WrappedAsyncSequence: AsyncSequence { @usableFromInline var base:Base.AsyncIterator - - @inlinable - public mutating func next() async throws -> Base.Element? { - try await base.next() - } @inlinable - @_implements(TypedAsyncIteratorProtocol, tetraNext(isolation:)) public mutating func next(isolation actor: isolated (any Actor)?) async throws -> Base.Element? { if #available(macOS 15.0, iOS 18.0, watchOS 11.0, tvOS 18.0, visionOS 2.0, *) { return try await base.next(isolation: actor) @@ -63,7 +57,7 @@ public struct WrappedAsyncSequence: AsyncSequence { extension WrappedAsyncSequence: Sendable where Base:Sendable {} @available(macOS 15.0, iOS 18.0, watchOS 11.0, tvOS 18.0, visionOS 2.0, *) -public struct WrappedAsyncSequenceV2: AsyncSequence { +public struct WrappedAsyncSequenceV2: AsyncSequence, TypedAsyncSequence { public typealias Element = Base.Element public typealias Failure = Base.Failure @@ -76,23 +70,16 @@ public struct WrappedAsyncSequenceV2: AsyncSequence { @usableFromInline var base:Base - public struct Iterator: TypedAsyncIteratorProtocol { - + public struct Iterator: AsyncIteratorProtocol, TypedAsyncIteratorProtocol { @usableFromInline var base:Base.AsyncIterator @inlinable - @_implements(TypedAsyncIteratorProtocol, tetraNext(isolation:)) public mutating func next(isolation actor: isolated (any Actor)?) async throws(Base.Failure) -> Base.Element? { return try await base.next(isolation: actor) } - @inlinable - public mutating func next() async throws -> Base.Element? { - return try await base.next() - } - @usableFromInline init(base: Base.AsyncIterator) { self.base = base diff --git a/Sources/Tetra/Foundation/Mics.swift b/Sources/Tetra/Foundation/Mics.swift index e3fe443..4b918b0 100644 --- a/Sources/Tetra/Foundation/Mics.swift +++ b/Sources/Tetra/Foundation/Mics.swift @@ -91,12 +91,12 @@ func wrapToResult(_ block: () throws(Failure) -> T) -> Result( +func wrapToResult( _ actor: isolated (any Actor)?, _ iterator: inout Base -) async -> sending Result? { +) async -> sending Result? { do { - if let value = try await iterator.tetraNext(isolation: actor) { + if let value = try await iterator.next(isolation: actor) { return .success(value) } return nil From 78b19323d363c2da0d2d5d9d7333f483a87505ed Mon Sep 17 00:00:00 2001 From: park-byeong-gwan Date: Mon, 17 Jun 2024 18:36:18 +0900 Subject: [PATCH 28/63] fix DiscardingTaskGroup simulation stops too early. --- .../Tetra/Combine/ExperimentalMapTask.swift | 37 ++++++-- .../Concurrency/DiscardingTaskState.swift | 92 +++++++++++++++++++ 2 files changed, 120 insertions(+), 9 deletions(-) create mode 100644 Sources/Tetra/Concurrency/DiscardingTaskState.swift diff --git a/Sources/Tetra/Combine/ExperimentalMapTask.swift b/Sources/Tetra/Combine/ExperimentalMapTask.swift index a96d0a5..cbfae7b 100644 --- a/Sources/Tetra/Combine/ExperimentalMapTask.swift +++ b/Sources/Tetra/Combine/ExperimentalMapTask.swift @@ -79,7 +79,7 @@ extension MultiMapTask { private func localTask( subscription: any Subscription, group: inout some CompatThrowingDiscardingTaskGroup - ) async { + ) async throws { group.addTask(priority: nil) { for await demand in demandSource.stream { let nextDemand = receive(demand: demand) @@ -93,7 +93,7 @@ extension MultiMapTask { switch upstreamValue { case .failure(let failure): send(completion: .failure(failure), cancel: false) - break + throw CancellationError() case .success(let success): let flag = group.addTaskUnlessCancelled(priority: nil) { switch await transform(success) { @@ -220,7 +220,7 @@ extension MultiMapTask { if #available(iOS 17.0, tvOS 17.0, macCatalyst 17.0, macOS 14.0, watchOS 10.0, visionOS 1.0, *) { try? await withThrowingDiscardingTaskGroup(returning: Void.self) { group in defer { terminateStream() } - await localTask( + try await localTask( subscription: subscription, group: &group ) @@ -228,18 +228,37 @@ extension MultiMapTask { } else { try? await withThrowingTaskGroup(of: Void.self, returning: Void.self) { group in defer { terminateStream() } + let lock = createCheckedStateLock(checkedState: DiscardingTaskState.waiting) + group.addTask { + // keep at least one child task alive + try await withUnsafeThrowingContinuation{ continuation in + lock.withLock{ + $0.transition(.suspend(continuation)) + }?.run() + } + } var iterator = group.makeAsyncIterator() - let stream = AsyncThrowingStream(unfolding: { try await iterator.next() }) + let stream = AsyncThrowingStream(unfolding: { return try await iterator.next() }) async let subTask:() = { for try await _ in stream { } }() - await localTask( - subscription: subscription, - group: &group - ) - try await subTask + do { + try await localTask( + subscription: subscription, + group: &group + ) + lock.withLock{ + $0.transition(.finish) + }?.run() + try await subTask + } catch { + lock.withLock{ + $0.transition(.cancel) + }?.run() + throw error + } } } send(completion: .finished) diff --git a/Sources/Tetra/Concurrency/DiscardingTaskState.swift b/Sources/Tetra/Concurrency/DiscardingTaskState.swift new file mode 100644 index 0000000..8c89e90 --- /dev/null +++ b/Sources/Tetra/Concurrency/DiscardingTaskState.swift @@ -0,0 +1,92 @@ +// +// DiscardingTaskState.swift +// +// +// Created by 박병관 on 6/17/24. +// + +enum DiscardingTaskState { + + case waiting + case suspend(UnsafeContinuation) + case cancel + case finish + + enum Effect { + case resume(UnsafeContinuation) + case raise(UnsafeContinuation) + + func run() { + switch self { + case .resume(let unsafeContinuation): + unsafeContinuation.resume() + case .raise(let unsafeContinuation): + unsafeContinuation.resume(throwing: CancellationError()) + } + } + } + + enum Event { + case cancel + case suspend(UnsafeContinuation) + case finish + } + + mutating func transition(_ event:Event) -> Effect? { + switch event { + case .cancel: + return cancel() + case .suspend(let unsafeContinuation): + return suspend(unsafeContinuation) + case .finish: + return finish() + } + } + + private mutating func suspend(_ cont:UnsafeContinuation) -> Effect? { + switch self { + case .waiting: + self = .suspend(cont) + return nil + case .suspend(let unsafeContinuation): + self = .suspend(cont) + assertionFailure("received suspend more than once") + return .raise(unsafeContinuation) + case .cancel: + return .raise(cont) + case .finish: + return .resume(cont) + } + } + + private mutating func finish() -> Effect? { + switch self { + case .waiting: + self = .finish + return nil + case .suspend(let unsafeContinuation): + self = .finish + return .resume(unsafeContinuation) + case .cancel: + return nil + case .finish: + return nil + } + } + + private mutating func cancel() -> Effect? { + switch self { + case .waiting: + self = .cancel + return nil + case .suspend(let unsafeContinuation): + self = .cancel + return .raise(unsafeContinuation) + case .cancel: + return nil + case .finish: + return nil + } + } + +} From 2351dfd56f723991549f25d87991fdee5b22579a Mon Sep 17 00:00:00 2001 From: park-byeong-gwan Date: Mon, 17 Jun 2024 18:54:17 +0900 Subject: [PATCH 29/63] fix discardingTaskgroup simulation ends too early --- .../Tetra/Combine/ExperimentalMapTask.swift | 70 +++++++++++-------- .../CompatDiscardigTaskGroup.swift | 64 ++++++++++++++++- 2 files changed, 103 insertions(+), 31 deletions(-) diff --git a/Sources/Tetra/Combine/ExperimentalMapTask.swift b/Sources/Tetra/Combine/ExperimentalMapTask.swift index edc06c3..a118bec 100644 --- a/Sources/Tetra/Combine/ExperimentalMapTask.swift +++ b/Sources/Tetra/Combine/ExperimentalMapTask.swift @@ -24,11 +24,16 @@ public struct MultiMapTask: Publisher where Upstream public let maxTasks:Subscribers.Demand public let upstream:Upstream - public let transform:@Sendable (Upstream.Output) async throws(Failure) -> sending Output + public let transform: @isolated(any) @Sendable (Upstream.Output) async throws(Failure) -> sending Output + public let taskExecutor: (any Executor)? public func receive(subscriber: S) where S : Subscriber, Upstream.Failure == S.Failure, Output == S.Input { let processor = Inner(maxTasks: maxTasks, subscriber: subscriber, transform: transform) - let task = Task(operation: processor.run) + let task = if #available(macOS 15.0, iOS 18.0, watchOS 11.0, tvOS 18.0, visionOS 2.0, *), let executor = taskExecutor as? (any TaskExecutor) { + Task(executorPreference: executor, operation: processor.run) + } else { + Task(operation: processor.run) + } processor.resumeCondition(task) upstream.subscribe(processor) } @@ -43,6 +48,21 @@ public struct MultiMapTask: Publisher where Upstream self.maxTasks = maxTasks self.upstream = upstream self.transform = transform + self.taskExecutor = nil + } + + @available(macOS 15.0, iOS 18.0, watchOS 11.0, tvOS 18.0, visionOS 2.0, *) + public init( + maxTasks: Subscribers.Demand = .max(1), + executor:(any TaskExecutor)? = nil, + upstream: Upstream, + transform: @Sendable @escaping @isolated(any) (Upstream.Output) async throws(Failure) -> Output + ) { + precondition(maxTasks != .none, "maxTasks can not be zero") + self.maxTasks = maxTasks + self.upstream = upstream + self.transform = transform + self.taskExecutor = executor } } @@ -58,6 +78,7 @@ extension MultiMapTask { var condition = TaskValueContinuation.waiting } + // this can run serially or in parallel by using demand config, so use adding actor isolation seems quite odd here struct Inner: CustomCombineIdentifierConvertible where S.Failure == Failure, S.Input == Output { private let maxTasks:Subscribers.Demand @@ -79,11 +100,11 @@ extension MultiMapTask { } } - private func localTask( + internal func localTask( + isolation actor: isolated (any Actor)? = #isolation, group: inout some CompatThrowingDiscardingTaskGroup - ) async { - var iterator = valueSource.stream.makeAsyncIterator() - while let upstreamValue = await iterator.next() { + ) async throws { + for await upstreamValue in valueSource.stream { switch upstreamValue { case .failure(let failure): send(completion: .failure(failure), cancel: false) @@ -208,33 +229,12 @@ extension MultiMapTask { if #available(iOS 17.0, tvOS 17.0, macCatalyst 17.0, macOS 14.0, watchOS 10.0, visionOS 1.0, *) { try? await withThrowingDiscardingTaskGroup(returning: Void.self) { group in defer { terminateStream() } - await localTask( + try await localTask( group: &group ) } } else { - try? await withThrowingTaskGroup(of: Void.self, returning: Void.self) { group in - defer { terminateStream() } - /* - this is very unsafe operation, and there is no way to prove race problem to compiler for now. - - But at least version before `DiscardingTaskGroup` exist, this implementation is safe from race problem. - - Because polling add queueing taskGroup is implemented in Busy waiting atomic alogrithnm. - */ - nonisolated(unsafe) - let unsafe = Suppress(value: group.makeAsyncIterator()) - async let subTask:() = { - var iterator = unsafe.value - while let _ = try await iterator.next() { - - } - }() - await localTask( - group: &group - ) - try await subTask - } + try? await wrapForBackDeploy(isolation: SafetyRegion()) } send(completion: .finished) } onCancel: { @@ -242,8 +242,20 @@ extension MultiMapTask { } } + func wrapForBackDeploy( + isolation actor: isolated (some Actor) + ) async throws { + try await withThrowingTaskGroup(of: Void.self) { + defer { terminateStream() } + try await $0.simulateDiscarding(isolation: actor) { isolation, group in + try await localTask(isolation: isolation, group: &group) + } + } + } + } + } diff --git a/Sources/Tetra/Concurrency/CompatDiscardigTaskGroup.swift b/Sources/Tetra/Concurrency/CompatDiscardigTaskGroup.swift index ee7cbfc..43f872c 100644 --- a/Sources/Tetra/Concurrency/CompatDiscardigTaskGroup.swift +++ b/Sources/Tetra/Concurrency/CompatDiscardigTaskGroup.swift @@ -20,13 +20,73 @@ internal protocol CompatThrowingDiscardingTaskGroup { priority: TaskPriority?, operation: @escaping @Sendable () async throws -> Void ) - + + @available(macOS 15.0, iOS 18.0, watchOS 11.0, tvOS 18.0, visionOS 2.0, *) + mutating func addTask(executorPreference taskExecutor: (any TaskExecutor)?, priority: TaskPriority?, operation: @escaping @isolated(any) @Sendable () async throws -> Void) + + @available(macOS 15.0, iOS 18.0, watchOS 11.0, tvOS 18.0, visionOS 2.0, *) + mutating func addTaskUnlessCancelled(executorPreference taskExecutor: (any TaskExecutor)?, priority: TaskPriority?, operation: @escaping @isolated(any) @Sendable () async throws -> Void) -> Bool } + @available(macOS 14.0, iOS 17.0, watchOS 10.0, tvOS 17.0, macCatalyst 17.0, visionOS 1.0, *) extension ThrowingDiscardingTaskGroup: CompatThrowingDiscardingTaskGroup { } -extension ThrowingTaskGroup: CompatThrowingDiscardingTaskGroup where ChildTaskResult == Void { +extension ThrowingTaskGroup: CompatThrowingDiscardingTaskGroup where ChildTaskResult == Void, Failure == any Error { } + +/// Empty actor to isolate `ThrowingTaskGroup` to simulate DiscardingTaskGroup +@usableFromInline +actor SafetyRegion { + + @usableFromInline + init() { + + } + +} + + +extension ThrowingTaskGroup where ChildTaskResult == Void, Failure == any Error { + + /// work around for simulating Discarding TaskGroup + /// + /// TaskGroup is protected by the actor isolation + /// - warning: always call TaskGroup api while holding isolation + @usableFromInline + internal mutating func simulateDiscarding( + isolation actor: isolated (any Actor), + body: (isolated any Actor, inout Self) async throws -> Void + ) async throws { + let lock = createCheckedStateLock(checkedState: DiscardingTaskState.waiting) + addTask { + // keep at least one child task alive + try await withUnsafeThrowingContinuation { continuation in + lock.withLock{ + $0.transition(.suspend(continuation)) + }?.run() + } + } + async let subTask:Void = { + while let _ = try await next(isolation: actor) { + + } + }() + do { + try await body(actor, &self) + lock.withLock{ + $0.transition(.finish) + }?.run() + try await subTask + } catch { + lock.withLock{ + $0.transition(.cancel) + }?.run() + throw error + } + } + +} + From 13d0a48dfc80420c4638cc1561953e53efbfa038 Mon Sep 17 00:00:00 2001 From: park-byeong-gwan Date: Mon, 17 Jun 2024 20:07:23 +0900 Subject: [PATCH 30/63] refine keep alive Taskgroup to use actor state --- .../Tetra/Combine/ExperimentalMapTask.swift | 2 +- .../Combine/Publishers+AsyncFlatMap.swift | 33 ++++++------- .../CompatDiscardigTaskGroup.swift | 46 +++++++++++++------ 3 files changed, 45 insertions(+), 36 deletions(-) diff --git a/Sources/Tetra/Combine/ExperimentalMapTask.swift b/Sources/Tetra/Combine/ExperimentalMapTask.swift index a118bec..7dbe3d2 100644 --- a/Sources/Tetra/Combine/ExperimentalMapTask.swift +++ b/Sources/Tetra/Combine/ExperimentalMapTask.swift @@ -243,7 +243,7 @@ extension MultiMapTask { } func wrapForBackDeploy( - isolation actor: isolated (some Actor) + isolation actor: isolated SafetyRegion ) async throws { try await withThrowingTaskGroup(of: Void.self) { defer { terminateStream() } diff --git a/Sources/Tetra/Combine/Publishers+AsyncFlatMap.swift b/Sources/Tetra/Combine/Publishers+AsyncFlatMap.swift index 60d7b7b..d6a3834 100644 --- a/Sources/Tetra/Combine/Publishers+AsyncFlatMap.swift +++ b/Sources/Tetra/Combine/Publishers+AsyncFlatMap.swift @@ -130,26 +130,7 @@ extension AsyncFlatMap { isCancelled = void == nil } else { - let void:Void? = try? await withThrowingTaskGroup(of: Void.self) { group in - defer { terminateStream() } - /* - this is very unsafe operation, and there is no way to prove race problem to compiler for now. - - But at least version before `DiscardingTaskGroup` exist, this implementation is safe from race problem. - - Because polling add queueing taskGroup is implemented in Busy waiting atomic alogrithnm. - */ - nonisolated(unsafe) - let unsafe = Suppress(value: group.makeAsyncIterator()) - async let subTask:() = { - var iterator = unsafe.value - while let _ = try await iterator.next() { - - } - }() - try await localTask(group: &group) - try await subTask - } + let void:Void? = try? await wrapForBackDeploy(isolation: SafetyRegion()) isCancelled = void == nil } if !isCancelled { @@ -357,7 +338,19 @@ extension AsyncFlatMap { return true } + func wrapForBackDeploy( + isolation actor: isolated SafetyRegion + ) async throws { + try await withThrowingTaskGroup(of: Void.self) { + defer { terminateStream() } + try await $0.simulateDiscarding(isolation: actor) { isolation, group in + try await localTask(isolation: isolation, group: &group) + } + } + } + private func localTask( + isolation actor: isolated (any Actor)? = #isolation, group: inout some CompatThrowingDiscardingTaskGroup ) async throws { for await result in valueSource.stream { diff --git a/Sources/Tetra/Concurrency/CompatDiscardigTaskGroup.swift b/Sources/Tetra/Concurrency/CompatDiscardigTaskGroup.swift index 43f872c..b9fc05c 100644 --- a/Sources/Tetra/Concurrency/CompatDiscardigTaskGroup.swift +++ b/Sources/Tetra/Concurrency/CompatDiscardigTaskGroup.swift @@ -41,11 +41,37 @@ extension ThrowingTaskGroup: CompatThrowingDiscardingTaskGroup where ChildTaskRe @usableFromInline actor SafetyRegion { + private var isFinished = false + private var continuation: UnsafeContinuation? = nil + @usableFromInline init() { } + @usableFromInline + func markDone() { + guard !isFinished else { return } + isFinished = true + continuation?.resume() + continuation = nil + } + + @usableFromInline + func hold() async { + await withUnsafeContinuation { + if isFinished { + $0.resume() + } else { + if let old = self.continuation { + assertionFailure("received suspend more than once!") + old.resume() + } + self.continuation = $0 + } + } + } + } @@ -54,36 +80,26 @@ extension ThrowingTaskGroup where ChildTaskResult == Void, Failure == any Error /// work around for simulating Discarding TaskGroup /// /// TaskGroup is protected by the actor isolation - /// - warning: always call TaskGroup api while holding isolation + /// - important: always call TaskGroup api while holding isolation @usableFromInline internal mutating func simulateDiscarding( - isolation actor: isolated (any Actor), + isolation actor: isolated (SafetyRegion), body: (isolated any Actor, inout Self) async throws -> Void ) async throws { - let lock = createCheckedStateLock(checkedState: DiscardingTaskState.waiting) addTask { // keep at least one child task alive - try await withUnsafeThrowingContinuation { continuation in - lock.withLock{ - $0.transition(.suspend(continuation)) - }?.run() - } + await actor.hold() } async let subTask:Void = { while let _ = try await next(isolation: actor) { - } }() do { try await body(actor, &self) - lock.withLock{ - $0.transition(.finish) - }?.run() + actor.markDone() try await subTask } catch { - lock.withLock{ - $0.transition(.cancel) - }?.run() + actor.markDone() throw error } } From 7f0b99b91e1a697f523374b0203ee98f46cab860 Mon Sep 17 00:00:00 2001 From: park-byeong-gwan Date: Mon, 17 Jun 2024 20:45:25 +0900 Subject: [PATCH 31/63] Backport DiscardingTaskGroup: wrap defer stack with do block for immediate poop --- .../Concurrency/CompatDiscardigTaskGroup.swift | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/Sources/Tetra/Concurrency/CompatDiscardigTaskGroup.swift b/Sources/Tetra/Concurrency/CompatDiscardigTaskGroup.swift index b9fc05c..d5c6fff 100644 --- a/Sources/Tetra/Concurrency/CompatDiscardigTaskGroup.swift +++ b/Sources/Tetra/Concurrency/CompatDiscardigTaskGroup.swift @@ -87,21 +87,23 @@ extension ThrowingTaskGroup where ChildTaskResult == Void, Failure == any Error body: (isolated any Actor, inout Self) async throws -> Void ) async throws { addTask { - // keep at least one child task alive + /// keep at least one child task alive + /// so that subTask won't return await actor.hold() } + /// drain all the finished or failed Task async let subTask:Void = { while let _ = try await next(isolation: actor) { } }() + // wrap with do block so that `defer` pops before waiting subTask do { + /// release suspending Task + defer { actor.markDone() } + /// wrap the mutable TaskGroup with actor isolation try await body(actor, &self) - actor.markDone() - try await subTask - } catch { - actor.markDone() - throw error } + try await subTask } } From 507bffdbe9a8111684e3ae62c0a7ab433a99a6e7 Mon Sep 17 00:00:00 2001 From: park-byeong-gwan Date: Fri, 21 Jun 2024 18:43:12 +0900 Subject: [PATCH 32/63] MultiMapTask: add downstreamLock to match reactive stream spec --- .../Tetra/Combine/ExperimentalMapTask.swift | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/Sources/Tetra/Combine/ExperimentalMapTask.swift b/Sources/Tetra/Combine/ExperimentalMapTask.swift index cbfae7b..91a3786 100644 --- a/Sources/Tetra/Combine/ExperimentalMapTask.swift +++ b/Sources/Tetra/Combine/ExperimentalMapTask.swift @@ -65,7 +65,7 @@ extension MultiMapTask { private let demandSource = AsyncStream.makeStream() private let state: some UnfairStateLock> = createUncheckedStateLock(uncheckedState: TaskState()) private let transform:@Sendable (Upstream.Output) async -> Result - + private let downStreamLock = NSRecursiveLock() let combineIdentifier = CombineIdentifier() init(maxTasks:Subscribers.Demand, subscriber:S, transform: @escaping @Sendable (Upstream.Output) async -> Result) { @@ -135,16 +135,22 @@ extension MultiMapTask { return (old, effect) } effect?.run() - if let completion { - subscriber?.receive(completion: completion) + if let completion, let subscriber { + downStreamLock.withLock { + subscriber.receive(completion: completion) + } } } private func send(_ value: S.Input) -> Subscribers.Demand? { - let newDemand = state.withLockUnchecked{ + let subscriber = state.withLockUnchecked{ $0.subscriber - }?.receive(value) - guard let newDemand else { return nil } + } + guard let subscriber else { return nil } + + let newDemand = downStreamLock.withLock{ + subscriber.receive(value) + } if maxTasks == .unlimited { return newDemand From 39bb3b8ce818af804d3de3f6f56ee46aa3188e1c Mon Sep 17 00:00:00 2001 From: park-byeong-gwan Date: Fri, 21 Jun 2024 19:37:43 +0900 Subject: [PATCH 33/63] MapTasks: add locks to match Reactivestream spec on Subscriptions --- Sources/Tetra/Combine/ExperimentalMapTask.swift | 13 ++++++++++--- Sources/Tetra/Combine/Publishers+MapTask.swift | 9 +++++++-- Sources/Tetra/Combine/Publishers+TryMapTask.swift | 9 +++++++-- 3 files changed, 24 insertions(+), 7 deletions(-) diff --git a/Sources/Tetra/Combine/ExperimentalMapTask.swift b/Sources/Tetra/Combine/ExperimentalMapTask.swift index 91a3786..7e3e715 100644 --- a/Sources/Tetra/Combine/ExperimentalMapTask.swift +++ b/Sources/Tetra/Combine/ExperimentalMapTask.swift @@ -66,6 +66,7 @@ extension MultiMapTask { private let state: some UnfairStateLock> = createUncheckedStateLock(uncheckedState: TaskState()) private let transform:@Sendable (Upstream.Output) async -> Result private let downStreamLock = NSRecursiveLock() + private let outerLock = NSRecursiveLock() let combineIdentifier = CombineIdentifier() init(maxTasks:Subscribers.Demand, subscriber:S, transform: @escaping @Sendable (Upstream.Output) async -> Result) { @@ -84,7 +85,9 @@ extension MultiMapTask { for await demand in demandSource.stream { let nextDemand = receive(demand: demand) if nextDemand > .none { - subscription.request(nextDemand) + outerLock.withLock { + subscription.request(nextDemand) + } } } } @@ -103,7 +106,9 @@ extension MultiMapTask { case .success(let value): if let demand = send(value) { if demand > .none { - subscription.request(demand) + outerLock.withLock { + subscription.request(demand) + } } } else { throw CancellationError() @@ -269,7 +274,9 @@ extension MultiMapTask { } send(completion: .finished) } onCancel: { - subscription.cancel() + outerLock.withLock { + subscription.cancel() + } send(completion: nil, cancel: false) } } diff --git a/Sources/Tetra/Combine/Publishers+MapTask.swift b/Sources/Tetra/Combine/Publishers+MapTask.swift index 0aa82c9..2c62b21 100644 --- a/Sources/Tetra/Combine/Publishers+MapTask.swift +++ b/Sources/Tetra/Combine/Publishers+MapTask.swift @@ -81,6 +81,7 @@ extension MapTask { private let state: some UnfairStateLock> = createUncheckedStateLock(uncheckedState: .init()) private let transform:@Sendable (Upstream.Output) async -> Result let combineIdentifier = CombineIdentifier() + private let subscriptionLock = NSRecursiveLock() init( subscriber:S, @@ -177,7 +178,9 @@ extension MapTask { for await var demand in demandSource.stream { while demand > .none { demand -= 1 - subscription.request(.max(1)) + subscriptionLock.withLock { + subscription.request(.max(1)) + } let upstreamResult = await iterator.next() let upstreamValue:Upstream.Output switch upstreamResult { @@ -207,7 +210,9 @@ extension MapTask { } } onCancel: { - subscription.cancel() + subscriptionLock.withLock { + subscription.cancel() + } send(completion: nil) } diff --git a/Sources/Tetra/Combine/Publishers+TryMapTask.swift b/Sources/Tetra/Combine/Publishers+TryMapTask.swift index 6755584..a2e0202 100644 --- a/Sources/Tetra/Combine/Publishers+TryMapTask.swift +++ b/Sources/Tetra/Combine/Publishers+TryMapTask.swift @@ -71,6 +71,7 @@ extension TryMapTask { private let state: some UnfairStateLock> = createUncheckedStateLock(uncheckedState: .init()) private let transform:@Sendable (Upstream.Output) async throws -> Output let combineIdentifier = CombineIdentifier() + private let subscriptionLock = NSRecursiveLock() init( subscriber:S, @@ -169,7 +170,9 @@ extension TryMapTask { for await var demand in demandSource.stream { while demand > .none { demand -= 1 - subscription.request(.max(1)) + subscriptionLock.withLock { + subscription.request(.max(1)) + } let upstreamValue:Upstream.Output do { guard let value = try await iterator.next() else { @@ -197,7 +200,9 @@ extension TryMapTask { } } onCancel: { - subscription.cancel() + subscriptionLock.withLock { + subscription.cancel() + } send(completion: nil) } From 0edafc957ec3b3d88643008bb760d7478c458afb Mon Sep 17 00:00:00 2001 From: park-byeong-gwan Date: Wed, 26 Jun 2024 22:35:45 +0900 Subject: [PATCH 34/63] moving to swift 6, using package acces modifier. - backport various std asyncsequence operators for typed throwing feature - redesign mapTasks to use actor isolation to reduce contention (needs investigate further how to split dependancy and namespaces) --- Package.swift | 39 ++- .../AsyncCompactMapSequence.swift | 4 +- .../AsyncDropFirstSequence.swift | 3 +- .../AsyncDropWhileSequence.swift | 6 +- .../AsyncFilterSequence.swift | 6 +- .../AsyncFlatMapSequence.swift | 3 +- .../AsyncMapErrorSequence.swift | 8 +- .../AsyncMapSequence.swift | 72 ++-- .../AsyncPrefixSequence.swift | 3 +- .../AsyncPrefixWhileSequence.swift | 5 +- .../BackPortAsyncSequence/AsyncStream.swift | 20 +- .../AsyncThrowingStream.swift | 7 +- Sources/BackPortAsyncSequence/BackPort.swift | 85 ++--- .../ConvertTypeToAsyncSequence.swift | 3 +- .../LegacyTypedAsyncSequence.swift | 6 +- .../TypedAsyncIteratorProtocol.swift | 64 +++- .../WrappedAsyncSequence.swift | 5 +- Sources/BackPortAsyncSequence/operators.swift | 24 -- .../SafetyRegion.swift | 10 +- .../TaskGroup.swift | 49 ++- .../ThrowingTaskGroup.swift | 40 ++- .../conformance.swift | 121 +++++++ Sources/BackportDiscardingTaskGroup/imp.swift | 30 -- Sources/CriticalSection/Cell.swift | 40 +++ .../CriticalSection/ManagedUnfairLock.swift | 326 ++++++++++++++++++ .../Combine/AsyncFlatMapDemandState.swift | 2 +- Sources/Tetra/Combine/AsyncFlatMapError.swift | 97 ------ Sources/Tetra/Combine/AsyncSubscriber.swift | 1 + .../Combine/AsyncSubscriptionState.swift | 30 +- .../Tetra/Combine/Combine+Concurrency.swift | 15 +- .../Tetra/Combine/CompatAsyncPublisher.swift | 11 +- .../CompatAsyncThrowingPublisher.swift | 11 +- .../Tetra/Combine/DispatchTimePublisher.swift | 1 + .../Tetra/Combine/ExperimentalMapTask.swift | 177 ++++++---- .../Tetra/Combine/Future+Concurrency.swift | 54 ++- .../Tetra/Combine/PendingDemandState.swift | 5 +- .../Combine/Publishers+AsyncFlatMap.swift | 285 +++++++++------ .../Tetra/Combine/Publishers+MapTask.swift | 90 +++-- .../Tetra/Combine/Publishers+TryMapTask.swift | 120 +++++-- .../Combine/SchedulerTimePublisher.swift | 3 +- .../Concurrency/AsyncSequencePublisher.swift | 160 +++++---- .../Concurrency/AsyncTypedSequence.swift | 75 ---- .../CompatDiscardigTaskGroup.swift | 110 ------ .../CoreDataStack+Concurrency.swift | 16 +- .../Concurrency/DiscardingTaskState.swift | 92 ----- .../Concurrency/Dispatch+Extension.swift | 63 ++-- .../Notification+AsyncSequence.swift | 33 +- .../Concurrency/TaskValueContinuation.swift | 22 ++ .../URLSessionDownloadTask+Concurrency.swift | 1 + .../Concurrency/WrappedAsyncSequence.swift | 98 ------ Sources/Tetra/Foundation/ClosureHolder.swift | 24 +- Sources/Tetra/Foundation/EitherFailure.swift | 12 + .../Tetra/Foundation/ManagedUnfairLock.swift | 304 ---------------- Sources/Tetra/Foundation/Mics.swift | 34 +- .../Tetra/SwiftUI/Binding+Collection.swift | 2 +- Tests/TetraTests/AsyncFlatMapTests.swift | 96 +++--- .../AsyncSequencePublisherTests.swift | 8 +- Tests/TetraTests/MapTaskTests.swift | 6 +- Tests/TetraTests/MultiMapTaskTests.swift | 11 +- Tests/TetraTests/TetraTests.swift | 1 + Tests/TetraTests/TryMapTaskTests.swift | 17 +- 61 files changed, 1575 insertions(+), 1491 deletions(-) create mode 100644 Sources/BackportDiscardingTaskGroup/conformance.swift delete mode 100644 Sources/BackportDiscardingTaskGroup/imp.swift create mode 100644 Sources/CriticalSection/Cell.swift create mode 100644 Sources/CriticalSection/ManagedUnfairLock.swift delete mode 100644 Sources/Tetra/Combine/AsyncFlatMapError.swift delete mode 100644 Sources/Tetra/Concurrency/AsyncTypedSequence.swift delete mode 100644 Sources/Tetra/Concurrency/CompatDiscardigTaskGroup.swift delete mode 100644 Sources/Tetra/Concurrency/DiscardingTaskState.swift delete mode 100644 Sources/Tetra/Concurrency/WrappedAsyncSequence.swift create mode 100644 Sources/Tetra/Foundation/EitherFailure.swift diff --git a/Package.swift b/Package.swift index 67c6414..83b8684 100644 --- a/Package.swift +++ b/Package.swift @@ -18,20 +18,48 @@ let package = Package( .library( name: "Tetra", targets: ["Tetra"] - ) + ), ], dependencies: [ // Dependencies declare other packages that this package depends on. // .package(url: /* package url */, from: "1.0.0"), .package(url: "https://github.com/apple/swift-collections.git", .upToNextMajor(from: "1.1.0")), + .package( + url: "https://github.com/apple/swift-atomics.git", + .upToNextMajor(from: "1.2.0") // or `.upToNextMinor + ), + ], targets: [ // Targets are the basic building blocks of a package. A target can define a module or a test suite. // Targets can depend on other targets in this package, and on products in packages this package depends on. + .target( + name: "CriticalSection", + dependencies: [ + .product(name: "Atomics", package: "swift-atomics") + ], + swiftSettings: [ + .swiftLanguageVersion(.v6), + .enableExperimentalFeature("StaticExclusiveOnly"), + .enableExperimentalFeature("RawLayout"), + .enableExperimentalFeature("BuiltinModule") + ] + ), + .target( + name: "BackportDiscardingTaskGroup", + swiftSettings: [ + .enableUpcomingFeature("FullTypedThrows"), + .enableExperimentalFeature("IsolatedAny"), + .swiftLanguageVersion(.v6) + ] + ), .target( name: "Tetra", dependencies: [ - .product(name: "DequeModule", package: "swift-collections") + .product(name: "DequeModule", package: "swift-collections"), + "BackPortAsyncSequence", + "CriticalSection", + "BackportDiscardingTaskGroup", ], swiftSettings: [ .enableUpcomingFeature("FullTypedThrows"), @@ -39,6 +67,13 @@ let package = Package( .swiftLanguageVersion(.v6) ] ), + .target( + name: "BackPortAsyncSequence", + dependencies: [], + swiftSettings: [ + .swiftLanguageVersion(.v6), + ] + ), .testTarget( name: "TetraTests", dependencies: [ diff --git a/Sources/BackPortAsyncSequence/AsyncCompactMapSequence.swift b/Sources/BackPortAsyncSequence/AsyncCompactMapSequence.swift index 6cbda15..7d7173e 100644 --- a/Sources/BackPortAsyncSequence/AsyncCompactMapSequence.swift +++ b/Sources/BackPortAsyncSequence/AsyncCompactMapSequence.swift @@ -86,7 +86,7 @@ extension BackPort.AsyncCompactMapSequence.Iterator: AsyncIteratorProtocol, Type /// that transforms to a non-`nil` value. If calling the closure throws an /// error, the sequence ends and `next()` rethrows the error. @inlinable - public mutating func next(isolation actor: isolated (any Actor)?) async throws(Failure) -> Element? { + public mutating func next(isolation actor: isolated (any Actor)? = #isolation) async throws(Failure) -> Element? { while !finished { guard let element = try await baseIterator.next(isolation: actor) else { finished = true @@ -104,7 +104,7 @@ extension BackPort.AsyncCompactMapSequence.Iterator: AsyncIteratorProtocol, Type return nil } - + @_disfavoredOverload @inlinable public mutating func next() async throws(Failure) -> Element? { try await next(isolation: nil) diff --git a/Sources/BackPortAsyncSequence/AsyncDropFirstSequence.swift b/Sources/BackPortAsyncSequence/AsyncDropFirstSequence.swift index f65f901..48acb9f 100644 --- a/Sources/BackPortAsyncSequence/AsyncDropFirstSequence.swift +++ b/Sources/BackPortAsyncSequence/AsyncDropFirstSequence.swift @@ -57,7 +57,7 @@ extension BackPort.AsyncDropFirstSequence.Iterator: AsyncIteratorProtocol, Typed public typealias Failure = Base.AsyncIterator.Err @inlinable - public mutating func next(isolation actor: isolated (any Actor)?) async throws(Failure) -> Element? { + public mutating func next(isolation actor: isolated (any Actor)? = #isolation) async throws(Failure) -> Element? { var remainingToDrop = count while remainingToDrop > 0 { guard try await baseIterator.next(isolation: actor) != nil else { @@ -70,6 +70,7 @@ extension BackPort.AsyncDropFirstSequence.Iterator: AsyncIteratorProtocol, Typed return try await baseIterator.next(isolation: actor) } + @_disfavoredOverload @inlinable public mutating func next() async throws(Failure) -> Element? { try await next(isolation: nil) diff --git a/Sources/BackPortAsyncSequence/AsyncDropWhileSequence.swift b/Sources/BackPortAsyncSequence/AsyncDropWhileSequence.swift index b17da62..123b6c9 100644 --- a/Sources/BackPortAsyncSequence/AsyncDropWhileSequence.swift +++ b/Sources/BackPortAsyncSequence/AsyncDropWhileSequence.swift @@ -5,9 +5,6 @@ // Created by 박병관 on 6/13/24. // -import Foundation - - extension BackPort { public struct AsyncDropWhileSequence where Base.AsyncIterator: TypedAsyncIteratorProtocol { @@ -91,7 +88,7 @@ extension BackPort.AsyncDropWhileSequence.Iterator: AsyncIteratorProtocol, Typed /// received from its base iterator as-is, and never executes the predicate /// closure again. @inlinable - public mutating func next(isolation actor: isolated (any Actor)?) async throws(Failure) -> Element? { + public mutating func next(isolation actor: isolated (any Actor)? = #isolation) async throws(Failure) -> Element? { while !finished && !doneDropping { guard let element = try await baseIterator.next(isolation: actor) else { return nil @@ -112,6 +109,7 @@ extension BackPort.AsyncDropWhileSequence.Iterator: AsyncIteratorProtocol, Typed return try await baseIterator.next(isolation: actor) } + @_disfavoredOverload @inlinable public mutating func next() async throws(Failure) -> Element? { try await next(isolation: nil) diff --git a/Sources/BackPortAsyncSequence/AsyncFilterSequence.swift b/Sources/BackPortAsyncSequence/AsyncFilterSequence.swift index 555abff..5831951 100644 --- a/Sources/BackPortAsyncSequence/AsyncFilterSequence.swift +++ b/Sources/BackPortAsyncSequence/AsyncFilterSequence.swift @@ -5,9 +5,6 @@ // Created by 박병관 on 6/13/24. // - -import Foundation - extension BackPort { @@ -76,7 +73,7 @@ extension BackPort.AsyncFilterSequence.Iterator: AsyncIteratorProtocol, TypedAsy public typealias Failure = Base.AsyncIterator.Err @inlinable - public mutating func next(isolation actor: isolated (any Actor)?) async throws(Failure) -> Element? { + public mutating func next(isolation actor: isolated (any Actor)? = #isolation) async throws(Failure) -> Element? { while !finished { guard let element = try await baseIterator.next(isolation: actor) else { return nil @@ -94,6 +91,7 @@ extension BackPort.AsyncFilterSequence.Iterator: AsyncIteratorProtocol, TypedAsy return nil } + @_disfavoredOverload @inlinable public mutating func next() async throws(Failure) -> Element? { try await next(isolation: nil) diff --git a/Sources/BackPortAsyncSequence/AsyncFlatMapSequence.swift b/Sources/BackPortAsyncSequence/AsyncFlatMapSequence.swift index 5864f32..81d96c7 100644 --- a/Sources/BackPortAsyncSequence/AsyncFlatMapSequence.swift +++ b/Sources/BackPortAsyncSequence/AsyncFlatMapSequence.swift @@ -77,7 +77,7 @@ extension BackPort.AsyncFlatMapSequence.Iterator: AsyncIteratorProtocol, TypedAs public typealias Failure = Base.AsyncIterator.Err @inlinable - public mutating func next(isolation actor: isolated (any Actor)?) async throws(Failure) -> Element? { + public mutating func next(isolation actor: isolated (any Actor)? = #isolation) async throws(Failure) -> Element? { while !finished { if var iterator = currentIterator { do { @@ -116,6 +116,7 @@ extension BackPort.AsyncFlatMapSequence.Iterator: AsyncIteratorProtocol, TypedAs return nil } + @_disfavoredOverload @inlinable public mutating func next() async throws(Failure) -> Element? { try await next(isolation: nil) diff --git a/Sources/BackPortAsyncSequence/AsyncMapErrorSequence.swift b/Sources/BackPortAsyncSequence/AsyncMapErrorSequence.swift index 3ba4dcd..22db97b 100644 --- a/Sources/BackPortAsyncSequence/AsyncMapErrorSequence.swift +++ b/Sources/BackPortAsyncSequence/AsyncMapErrorSequence.swift @@ -40,7 +40,8 @@ public struct AsyncMapErrorSequence where Bas extension AsyncMapErrorSequence:AsyncSequence, TypedAsyncSequence { public typealias AsyncIterator = Iterator - public typealias Failure = AsyncIterator.Failure + public typealias Failure = Failure +// public typealias Failure = AsyncIterator.Failure public struct Iterator { @@ -73,9 +74,9 @@ extension AsyncMapErrorSequence:AsyncSequence, TypedAsyncSequence { extension AsyncMapErrorSequence.Iterator: AsyncIteratorProtocol, TypedAsyncIteratorProtocol { public typealias Element = Base.Element - + public typealias Failure = Failure @inlinable - public mutating func next(isolation actor: isolated (any Actor)?) async throws(Failure) -> Element? { + public mutating func next(isolation actor: isolated (any Actor)? = #isolation) async throws(Failure) -> Element? { do { let value = try await base?.next(isolation: actor) if value == nil { @@ -89,6 +90,7 @@ extension AsyncMapErrorSequence.Iterator: AsyncIteratorProtocol, TypedAsyncItera } } + @_disfavoredOverload @inlinable public mutating func next() async throws(Failure) -> Element? { try await next(isolation: nil) diff --git a/Sources/BackPortAsyncSequence/AsyncMapSequence.swift b/Sources/BackPortAsyncSequence/AsyncMapSequence.swift index d3bf867..1c06cbc 100644 --- a/Sources/BackPortAsyncSequence/AsyncMapSequence.swift +++ b/Sources/BackPortAsyncSequence/AsyncMapSequence.swift @@ -40,10 +40,7 @@ extension BackPort.AsyncMapSequence: AsyncSequence, TypedAsyncSequence { public typealias AsyncIterator = Iterator /// The iterator that produces elements of the map sequence. - public struct Iterator: AsyncIteratorProtocol, TypedAsyncIteratorProtocol { - - public typealias Element = Transformed - public typealias Failure = Base.AsyncIterator.Err + public struct Iterator { @usableFromInline var baseIterator: Base.AsyncIterator @@ -62,35 +59,6 @@ extension BackPort.AsyncMapSequence: AsyncSequence, TypedAsyncSequence { self.baseIterator = baseIterator self.transform = transform } - - /// Produces the next element in the map sequence. - /// - /// This iterator calls `next()` on its base iterator; if this call returns - /// `nil`, `next()` returns `nil`. Otherwise, `next()` returns the result of - /// calling the transforming closure on the received element. - @inlinable - public mutating func next() async throws(Failure) -> Element? { - try await next(isolation: nil) - } - - /// Produces the next element in the map sequence. - /// - /// This iterator calls `next(isolation:)` on its base iterator; if this - /// call returns `nil`, `next(isolation:)` returns `nil`. Otherwise, - /// `next(isolation:)` returns the result of calling the transforming - /// closure on the received element. - @inlinable - public mutating func next(isolation actor: isolated (any Actor)?) async throws(Failure) -> Element? { - guard !finished, let element = try await baseIterator.next(isolation: actor) else { - return nil - } - do { - return try await transform(element) - } catch { - finished = true - throw error - } - } } @@ -100,6 +68,43 @@ extension BackPort.AsyncMapSequence: AsyncSequence, TypedAsyncSequence { } } +extension BackPort.AsyncMapSequence.Iterator: AsyncIteratorProtocol, TypedAsyncIteratorProtocol { + + public typealias Element = Transformed + public typealias Failure = Base.AsyncIterator.Err + + /// Produces the next element in the map sequence. + /// + /// This iterator calls `next()` on its base iterator; if this call returns + /// `nil`, `next()` returns `nil`. Otherwise, `next()` returns the result of + /// calling the transforming closure on the received element. + @_disfavoredOverload + @inlinable + public mutating func next() async throws(Failure) -> Element? { + try await next(isolation: nil) + } + + /// Produces the next element in the map sequence. + /// + /// This iterator calls `next(isolation:)` on its base iterator; if this + /// call returns `nil`, `next(isolation:)` returns `nil`. Otherwise, + /// `next(isolation:)` returns the result of calling the transforming + /// closure on the received element. + @inlinable + public mutating func next(isolation actor: isolated (any Actor)? = #isolation) async throws(Failure) -> Element? { + guard !finished, let element = try await baseIterator.next(isolation: actor) else { + return nil + } + do { + return try await transform(element) + } catch { + finished = true + throw error + } + } + +} + extension BackPort.AsyncMapSequence: @unchecked Sendable where Base: Sendable, Base.Element: Sendable, @@ -112,6 +117,7 @@ where Base.AsyncIterator: Sendable, extension BackPort.AsyncMapSequence { + @inlinable package init ( _ source: Source, diff --git a/Sources/BackPortAsyncSequence/AsyncPrefixSequence.swift b/Sources/BackPortAsyncSequence/AsyncPrefixSequence.swift index 78921de..4c7deaf 100644 --- a/Sources/BackPortAsyncSequence/AsyncPrefixSequence.swift +++ b/Sources/BackPortAsyncSequence/AsyncPrefixSequence.swift @@ -59,7 +59,7 @@ extension BackPort.AsyncPrefixSequence.Iterator: AsyncIteratorProtocol, TypedAsy public typealias Failure = Base.AsyncIterator.Err @inlinable - public mutating func next(isolation actor: isolated (any Actor)?) async throws(Failure) -> Element? { + public mutating func next(isolation actor: isolated (any Actor)? = #isolation) async throws(Failure) -> Element? { if remaining != 0 { remaining &-= 1 return try await baseIterator.next(isolation: actor) @@ -68,6 +68,7 @@ extension BackPort.AsyncPrefixSequence.Iterator: AsyncIteratorProtocol, TypedAsy } } + @_disfavoredOverload @inlinable public mutating func next() async throws(Failure) -> Element? { try await next(isolation: nil) diff --git a/Sources/BackPortAsyncSequence/AsyncPrefixWhileSequence.swift b/Sources/BackPortAsyncSequence/AsyncPrefixWhileSequence.swift index dff13e1..d557b22 100644 --- a/Sources/BackPortAsyncSequence/AsyncPrefixWhileSequence.swift +++ b/Sources/BackPortAsyncSequence/AsyncPrefixWhileSequence.swift @@ -1,5 +1,5 @@ // -// error.swift +// AsyncPrefixWhileSequence.swift // // // Created by 박병관 on 6/13/24. @@ -87,6 +87,7 @@ extension BackPort.AsyncPrefixWhileSequence.Iterator: AsyncIteratorProtocol, Typ /// succeeds, this method passes along the element. Otherwise, it returns /// `nil`, ending the sequence. If calling the predicate closure throws an /// error, the sequence ends and `next()` rethrows the error. + @_disfavoredOverload @inlinable public mutating func next() async throws(Failure) -> Element? { try await next(isolation: nil) @@ -100,7 +101,7 @@ extension BackPort.AsyncPrefixWhileSequence.Iterator: AsyncIteratorProtocol, Typ /// `nil`, ending the sequence. If calling the predicate closure throws an /// error, the sequence ends and `next(isolation:)` rethrows the error. @inlinable - public mutating func next(isolation actor: isolated (any Actor)?) async throws(Failure) -> Base.Element? { + public mutating func next(isolation actor: isolated (any Actor)? = #isolation) async throws(Failure) -> Base.Element? { if !predicateHasFailed, let nextElement = try await baseIterator.next(isolation: actor) { do { if try await predicate(nextElement) { diff --git a/Sources/BackPortAsyncSequence/AsyncStream.swift b/Sources/BackPortAsyncSequence/AsyncStream.swift index d509ea4..0979b9a 100644 --- a/Sources/BackPortAsyncSequence/AsyncStream.swift +++ b/Sources/BackPortAsyncSequence/AsyncStream.swift @@ -21,6 +21,7 @@ extension AsyncTypedStream: AsyncSequence, TypedAsyncSequence { public typealias Failure = Never + @inlinable public func makeAsyncIterator() -> Iterator { Iterator(baseIterator: base.makeAsyncIterator()) } @@ -44,7 +45,7 @@ extension AsyncTypedStream.Iterator: AsyncIteratorProtocol, TypedAsyncIteratorPr public typealias Failure = Never @inlinable - public mutating func next(isolation actor: isolated (any Actor)?) async -> Element? { + public mutating func next(isolation actor: isolated (any Actor)? = #isolation) async -> Element? { if #available(macOS 15.0, iOS 18.0, watchOS 11.0, tvOS 18.0, visionOS 2.0, *) { return await baseIterator.next(isolation: actor) } else { @@ -52,9 +53,10 @@ extension AsyncTypedStream.Iterator: AsyncIteratorProtocol, TypedAsyncIteratorPr } } + @_disfavoredOverload @inlinable public mutating func next() async -> Element? { - await next(isolation: nil) + await baseIterator.next() } @inline(__always) @@ -65,15 +67,5 @@ extension AsyncTypedStream.Iterator: AsyncIteratorProtocol, TypedAsyncIteratorPr } -extension AsyncStream { - - func bridge() -> some TypedAsyncSequence { - AsyncTypedStream(base: self) - } - - func bridge2() -> some TypedAsyncSequence { - AsyncMapErrorSequence(base: LegacyTypedAsyncSequence(base: self)) { _ throws(Never) in - - } - } -} +extension AsyncTypedStream: Sendable where Element: Sendable {} + diff --git a/Sources/BackPortAsyncSequence/AsyncThrowingStream.swift b/Sources/BackPortAsyncSequence/AsyncThrowingStream.swift index 3061d05..ee2ffca 100644 --- a/Sources/BackPortAsyncSequence/AsyncThrowingStream.swift +++ b/Sources/BackPortAsyncSequence/AsyncThrowingStream.swift @@ -45,7 +45,7 @@ extension AsyncTypedThrowingStream: AsyncSequence, TypedAsyncSequence { extension AsyncTypedThrowingStream.Iterator: AsyncIteratorProtocol, TypedAsyncIteratorProtocol { @inlinable - public mutating func next(isolation actor: isolated (any Actor)?) async throws(Failure) -> Element? { + public mutating func next(isolation actor: isolated (any Actor)? = #isolation) async throws(Failure) -> Element? { if #available(macOS 15.0, iOS 18.0, watchOS 11.0, tvOS 18.0, visionOS 2.0, *) { return try await baseIterator.next(isolation: actor) } else { @@ -57,6 +57,7 @@ extension AsyncTypedThrowingStream.Iterator: AsyncIteratorProtocol, TypedAsyncIt } } + @_disfavoredOverload @inlinable public mutating func next() async throws(Failure) -> Element? { try await next(isolation: nil) @@ -70,6 +71,9 @@ extension AsyncTypedThrowingStream.Iterator: AsyncIteratorProtocol, TypedAsyncIt } +extension AsyncTypedThrowingStream: Sendable where Element: Sendable {} + + extension AsyncThrowingStream { func bridge() -> some TypedAsyncSequence { @@ -77,3 +81,4 @@ extension AsyncThrowingStream { } } + diff --git a/Sources/BackPortAsyncSequence/BackPort.swift b/Sources/BackPortAsyncSequence/BackPort.swift index 29cfabd..b1c95d6 100644 --- a/Sources/BackPortAsyncSequence/BackPort.swift +++ b/Sources/BackPortAsyncSequence/BackPort.swift @@ -4,48 +4,55 @@ // // Created by 박병관 on 6/13/24. // -import Darwin public enum BackPort { } +@available(macOS 15.0, iOS 18.0, watchOS 11.0, tvOS 18.0, visionOS 2.0, *) +extension AsyncStream.Iterator: TypedAsyncIteratorProtocol {} -import Synchronization -import Builtin -// -// -//@_rawLayout(like: os_unfair_lock, movesAsLike) -//@usableFromInline -//package struct UnfairPrimitive: ~Copyable { -// -// @inlinable -// package init() { -// pointer.initialize(to: .init()) -// } -// -// @usableFromInline -// internal var pointer:UnsafeMutablePointer { -// withUnsafePointer(to: self) { -// UnsafeMutableRawPointer(mutating: $0).assumingMemoryBound(to: os_unfair_lock.self) -// } -// } -// -// @inlinable -// borrowing package func lock() { -// UnsafeMutableRawPointer(Builtin.addressOfRawLayout(self)) -// os_unfair_lock_lock(pointer) -// } -// -// @inlinable -// borrowing package func unlock() { -// os_unfair_lock_unlock(pointer) -// } -// -// @inlinable -// borrowing package func tryLock() -> Bool { -// os_unfair_lock_trylock(pointer) -// } -// -// -//} +@available(macOS 15.0, iOS 18.0, watchOS 11.0, tvOS 18.0, visionOS 2.0, *) +extension AsyncThrowingStream.Iterator: TypedAsyncIteratorProtocol {} + +@available(macOS 15.0, iOS 18.0, watchOS 11.0, tvOS 18.0, visionOS 2.0, *) +extension AsyncMapSequence.Iterator: TypedAsyncIteratorProtocol {} + +@available(macOS 15.0, iOS 18.0, watchOS 11.0, tvOS 18.0, visionOS 2.0, *) +extension AsyncFlatMapSequence.Iterator: TypedAsyncIteratorProtocol {} + +@available(macOS 15.0, iOS 18.0, watchOS 11.0, tvOS 18.0, visionOS 2.0, *) +extension AsyncFilterSequence.Iterator: TypedAsyncIteratorProtocol {} + +@available(macOS 15.0, iOS 18.0, watchOS 11.0, tvOS 18.0, visionOS 2.0, *) +extension AsyncDropFirstSequence.Iterator: TypedAsyncIteratorProtocol {} + +@available(macOS 15.0, iOS 18.0, watchOS 11.0, tvOS 18.0, visionOS 2.0, *) +extension AsyncDropWhileSequence.Iterator: TypedAsyncIteratorProtocol {} + +@available(macOS 15.0, iOS 18.0, watchOS 11.0, tvOS 18.0, visionOS 2.0, *) +extension AsyncCompactMapSequence.Iterator: TypedAsyncIteratorProtocol {} + +@available(macOS 15.0, iOS 18.0, watchOS 11.0, tvOS 18.0, visionOS 2.0, *) +extension AsyncPrefixWhileSequence.Iterator: TypedAsyncIteratorProtocol {} + +@available(macOS 15.0, iOS 18.0, watchOS 11.0, tvOS 18.0, visionOS 2.0, *) +extension AsyncPrefixSequence.Iterator: TypedAsyncIteratorProtocol {} + +@available(macOS 15.0, iOS 18.0, watchOS 11.0, tvOS 18.0, visionOS 2.0, *) +extension AsyncThrowingMapSequence.Iterator: TypedAsyncIteratorProtocol {} + +@available(macOS 15.0, iOS 18.0, watchOS 11.0, tvOS 18.0, visionOS 2.0, *) +extension AsyncThrowingFlatMapSequence.Iterator: TypedAsyncIteratorProtocol {} + +@available(macOS 15.0, iOS 18.0, watchOS 11.0, tvOS 18.0, visionOS 2.0, *) +extension AsyncThrowingFilterSequence.Iterator: TypedAsyncIteratorProtocol {} + +@available(macOS 15.0, iOS 18.0, watchOS 11.0, tvOS 18.0, visionOS 2.0, *) +extension AsyncThrowingDropWhileSequence.Iterator: TypedAsyncIteratorProtocol {} + +@available(macOS 15.0, iOS 18.0, watchOS 11.0, tvOS 18.0, visionOS 2.0, *) +extension AsyncThrowingCompactMapSequence.Iterator: TypedAsyncIteratorProtocol {} + +@available(macOS 15.0, iOS 18.0, watchOS 11.0, tvOS 18.0, visionOS 2.0, *) +extension AsyncThrowingPrefixWhileSequence.Iterator: TypedAsyncIteratorProtocol {} diff --git a/Sources/BackPortAsyncSequence/ConvertTypeToAsyncSequence.swift b/Sources/BackPortAsyncSequence/ConvertTypeToAsyncSequence.swift index 9645308..7946d09 100644 --- a/Sources/BackPortAsyncSequence/ConvertTypeToAsyncSequence.swift +++ b/Sources/BackPortAsyncSequence/ConvertTypeToAsyncSequence.swift @@ -47,10 +47,11 @@ extension ConvertTypeToAsyncSequence.Iterator: AsyncIteratorProtocol, TypedAsync public typealias Element = Base.Element @inlinable - public mutating func next(isolation actor: isolated (any Actor)?) async throws(Failure) -> Element? { + public mutating func next(isolation actor: isolated (any Actor)? = #isolation) async throws(Failure) -> Element? { try await baseIterator.next(isolation: actor) } + @_disfavoredOverload @inlinable public mutating func next() async throws(Failure) -> Element? { try await next(isolation: nil) diff --git a/Sources/BackPortAsyncSequence/LegacyTypedAsyncSequence.swift b/Sources/BackPortAsyncSequence/LegacyTypedAsyncSequence.swift index 7a80c40..4dd6235 100644 --- a/Sources/BackPortAsyncSequence/LegacyTypedAsyncSequence.swift +++ b/Sources/BackPortAsyncSequence/LegacyTypedAsyncSequence.swift @@ -53,7 +53,7 @@ extension LegacyTypedAsyncSequence.Iterator: AsyncIteratorProtocol, TypedAsyncIt public typealias Failure = any Error @inlinable - public mutating func next(isolation actor: isolated (any Actor)?) async throws(Failure) -> Element? { + public mutating func next(isolation actor: isolated (any Actor)? = #isolation) async throws(Failure) -> Element? { if #available(macOS 15.0, iOS 18.0, watchOS 11.0, tvOS 18.0, visionOS 2.0, *) { return try await baseIterator.next(isolation: actor) } else { @@ -61,11 +61,13 @@ extension LegacyTypedAsyncSequence.Iterator: AsyncIteratorProtocol, TypedAsyncIt } } + @_disfavoredOverload @inlinable public mutating func next() async throws(Failure) -> Element? { - try await next(isolation: nil) + try await baseIterator.next() } + @inline(__always) @usableFromInline internal mutating func advance() async throws(Failure) -> sending Element? { try await baseIterator.next() diff --git a/Sources/BackPortAsyncSequence/TypedAsyncIteratorProtocol.swift b/Sources/BackPortAsyncSequence/TypedAsyncIteratorProtocol.swift index ac74f9f..1d93b55 100644 --- a/Sources/BackPortAsyncSequence/TypedAsyncIteratorProtocol.swift +++ b/Sources/BackPortAsyncSequence/TypedAsyncIteratorProtocol.swift @@ -1,15 +1,64 @@ // -// Untitled.swift +// TypedAsyncIteratorProtocol.swift // // // Created by 박병관 on 6/13/24. // +/*** + Entry point of TypeFailure for `AsyncFlatMap` and `AsyncSequencePublisher -public protocol TypedAsyncIteratorProtocol { + Since generalized `AsyncSequence` is not available until Swift 6, this `protocol` is a entry point for async typed throw. + +Even though it has isolation parameter, this parameter is rarely used since `AsyncFlatMap` and `AsyncSequencePublisher` runs in nonisolated Task space. + +`Use `WrappedAsyncSequence` or `LegacyTypedAsyncSequence` if possible which provides general implementation to adapt this protocol + + Use `TypedAsyncSequence` with Associated primarmy type generic to make use of AsyncSequence in abstract way. And later wrap the instance with `ConvertTypeToAsyncSequence` to use it as typed `AsyncSequence` +``` + + let source = AsyncStream(...) + let abstactSequence:some TypedAsyncSequence = AsyncTypedStream(base: source) + for await value in ConvertTypeToAsyncSequence(base: abstactSequence) { + // use value + // this sequence is fully typed throwing and actor isolation support + } + + +``` + + + - important: `Err` must be same as `AsyncIterator.Failure` + + + - adapting protocol to existing type, + + adapting protocols to exisiting type that user don't own is pretty easy too. By writing conformance like below, compiler now know the correct implemenation without recursive problem. +```` + extension AsyncStream.Iterator: TypedAsyncIteratorProtocol { + + @_implements(TypedAsyncIteratorProtocol, next(isolation:)) + mutating public func tetraNext(isolation actor: isolated (any Actor)?) async throws(Never) -> Self.Element? { + if #available(macOS 15.0, iOS 18.0, watchOS 11.0, tvOS 18.0, visionOS 2.0, *) { + var c: some AsyncIteratorProtocol = self + defer { + self = c as! Self + } + return await c.next(isolation: actor) + } else { + return try? await next() + } + } + + } + + ```` + */ + +public protocol TypedAsyncIteratorProtocol: ~Copyable { associatedtype Element associatedtype Err: Error - typealias Failure = Err +// typealias Failure = Err @inlinable mutating func next(isolation actor: isolated (any Actor)?) async throws(Err) -> Element? @@ -31,12 +80,3 @@ public protocol TypedAsyncSequence:AsyncSequence where AsyncIterat @inlinable func makeAsyncIterator() -> AsyncIterator } - -package extension TypedAsyncIteratorProtocol { - - @inlinable - mutating func next() async throws(Err) -> Element? { - try await next(isolation: nil) - } - -} diff --git a/Sources/BackPortAsyncSequence/WrappedAsyncSequence.swift b/Sources/BackPortAsyncSequence/WrappedAsyncSequence.swift index 6962417..0815a08 100644 --- a/Sources/BackPortAsyncSequence/WrappedAsyncSequence.swift +++ b/Sources/BackPortAsyncSequence/WrappedAsyncSequence.swift @@ -51,10 +51,11 @@ extension WrappedAsyncSequence.Iterator: AsyncIteratorProtocol, TypedAsyncIterat public typealias Failure = Base.Failure @inlinable - public mutating func next(isolation actor: isolated (any Actor)?) async throws(Failure) -> Element? { + public mutating func next(isolation actor: isolated (any Actor)? = #isolation) async throws(Failure) -> Element? { try await baseIterator.next(isolation: actor) } + @_disfavoredOverload @inlinable public mutating func next() async throws(Failure) -> Element? { try await next(isolation: nil) @@ -63,8 +64,6 @@ extension WrappedAsyncSequence.Iterator: AsyncIteratorProtocol, TypedAsyncIterat } -@available(macOS 15.0, iOS 18.0, watchOS 11.0, tvOS 18.0, visionOS 2.0, *) extension WrappedAsyncSequence: Sendable where Base: Sendable, Base.Element: Sendable {} -@available(macOS 15.0, iOS 18.0, watchOS 11.0, tvOS 18.0, visionOS 2.0, *) extension WrappedAsyncSequence.Iterator: Sendable where Base.AsyncIterator: Sendable, Base.Element: Sendable {} diff --git a/Sources/BackPortAsyncSequence/operators.swift b/Sources/BackPortAsyncSequence/operators.swift index e2795ba..d1365fc 100644 --- a/Sources/BackPortAsyncSequence/operators.swift +++ b/Sources/BackPortAsyncSequence/operators.swift @@ -30,27 +30,3 @@ extension AsyncSequence where AsyncIterator: TypedAsyncIteratorProtocol { } - - - - - - -@inline(__always) -@inlinable -package -func iteratorNextResult( - _ actor: isolated (any Actor)? = #isolation, - _ iterator: inout Base -) async -> sending Result? { - do { - if let value = try await iterator.next(isolation: actor) { - return .success(value) - } - return nil -// #available(macOS 15.0, iOS 18.0, watchOS 11.0, tvOS 18.0, visionOS - } catch { - return .failure(error) - } -} - diff --git a/Sources/BackportDiscardingTaskGroup/SafetyRegion.swift b/Sources/BackportDiscardingTaskGroup/SafetyRegion.swift index 0219d57..5b52f21 100644 --- a/Sources/BackportDiscardingTaskGroup/SafetyRegion.swift +++ b/Sources/BackportDiscardingTaskGroup/SafetyRegion.swift @@ -9,8 +9,10 @@ @usableFromInline package actor SafetyRegion { - private(set) var isFinished = false - private var continuation: UnsafeContinuation? = nil + @usableFromInline + internal(set) package var isFinished = false + @usableFromInline + internal var continuation: UnsafeContinuation? = nil @inlinable package init() { @@ -18,8 +20,8 @@ package actor SafetyRegion { } @usableFromInline - internal func markDone() { - guard !isFinished else { return } + package func markDone() { +// guard !isFinished else { return } isFinished = true continuation?.resume() continuation = nil diff --git a/Sources/BackportDiscardingTaskGroup/TaskGroup.swift b/Sources/BackportDiscardingTaskGroup/TaskGroup.swift index 03b5fbb..5a8c67e 100644 --- a/Sources/BackportDiscardingTaskGroup/TaskGroup.swift +++ b/Sources/BackportDiscardingTaskGroup/TaskGroup.swift @@ -15,12 +15,12 @@ extension TaskGroup where ChildTaskResult == Void { internal mutating func simulateDiscarding( isolation actor: isolated T, body: (isolated T, inout Self) async -> sending V - ) async -> V { + ) async -> V { let holder: SafetyRegion = actor as? SafetyRegion ?? .init() if await holder.isFinished { preconditionFailure("SafetyRegion is already used!") } - addTask { + addTask(priority: .background) { /// keep at least one child task alive /// so that subTask won't return await holder.hold() @@ -34,16 +34,21 @@ extension TaskGroup where ChildTaskResult == Void { } } }() - let value:V - // wrap with do block so that `defer` pops before waiting subTask - do { - /// release suspending Task - /// wrap the mutable TaskGroup with actor isolation - value = await body(actor, &self) + async let mainTask = { + let v = await runBlock(isolation: actor, body:body) await holder.markDone() - } + return Suppress(base: v) + }() await subTask - return value + return await mainTask.base + } + + @usableFromInline + internal mutating func runBlock( + isolation actor: isolated T, + body: (isolated T, inout Self) async throws(ErrorRef) -> sending V + ) async throws(ErrorRef) -> sending V { + try await body(actor, &self) } } @@ -52,13 +57,32 @@ extension TaskGroup where ChildTaskResult == Void { package func simuateDiscardingTaskGroup( isolation actor: isolated T = #isolation, body: @Sendable (isolated T, inout TaskGroup) async -> sending TaskResult -) async -> TaskResult { +) async -> sending TaskResult { await withTaskGroup(of: Void.self, returning: TaskResult.self) { await $0.simulateDiscarding(isolation: actor, body: body) } } +/// simulate `DiscardingTaskGroup` and return the TaskResult +/// +/// This function is a good workaround for globalActor isolated version of `simuateDiscardingTaskGroup` +/// +/// +///``` +///await simuateDiscardingTaskGroup { @MainActor group in +/// group.addTask{ ... } +/// group.addTask{ ... } +/// group.addTask{ ... } +/// return 0 +///} +///``` +/// +/// +/// - precondition: body can not be nonisolated +/// - Parameter body: DiscardingTaskGroup body +/// - Returns: which is returned from body +/// - SeeAlso: withDiscardingTaskGroup(returning:body:) @inlinable package func simuateDiscardingTaskGroup( body: @Sendable @isolated(any) (inout TaskGroup) async -> sending TaskResult @@ -74,6 +98,3 @@ package func simuateDiscardingTaskGroup( return await body(&unsafe.base) } } - - - diff --git a/Sources/BackportDiscardingTaskGroup/ThrowingTaskGroup.swift b/Sources/BackportDiscardingTaskGroup/ThrowingTaskGroup.swift index 38a4a62..d86f955 100644 --- a/Sources/BackportDiscardingTaskGroup/ThrowingTaskGroup.swift +++ b/Sources/BackportDiscardingTaskGroup/ThrowingTaskGroup.swift @@ -21,7 +21,7 @@ extension ThrowingTaskGroup where ChildTaskResult == Void, Failure == any Error if await holder.isFinished { preconditionFailure("SafetyRegion is already used!") } - addTask { + addTask(priority: .background) { /// keep at least one child task alive /// so that subTask won't return await holder.hold() @@ -34,22 +34,40 @@ extension ThrowingTaskGroup where ChildTaskResult == Void, Failure == any Error } } }() - let value:V - // wrap with do block so that `defer` pops before waiting subTask + async let mainTask = { + do { + let v = try await runBlock(isolation: actor, body:body) + await holder.markDone() + return Suppress(base: v) + } catch { + await holder.markDone() + throw error + } + }() + let errorRef:(any Error)? do { - /// release suspending Task - /// wrap the mutable TaskGroup with actor isolation - value = try await body(actor, &self) - await holder.markDone() - + // wait for subTask first to trigger priority elavation + // (release finished tasks as soon as possible) + try await subTask + errorRef = nil } catch { - await holder.markDone() - throw error + errorRef = error + } + let value = try await mainTask.base + if let errorRef { + throw errorRef } - try await subTask return value } + @usableFromInline + internal mutating func runBlock( + isolation actor: isolated T, + body: (isolated T, inout Self) async throws(ErrorRef) -> sending V + ) async throws(ErrorRef) -> sending V { + try await body(actor, &self) + } + } @inlinable diff --git a/Sources/BackportDiscardingTaskGroup/conformance.swift b/Sources/BackportDiscardingTaskGroup/conformance.swift new file mode 100644 index 0000000..1fcc159 --- /dev/null +++ b/Sources/BackportDiscardingTaskGroup/conformance.swift @@ -0,0 +1,121 @@ +// +// File.swift +// +// +// Created by 박병관 on 5/16/24. +// + +@available(macOS 14.0, iOS 17.0, watchOS 10.0, tvOS 17.0, macCatalyst 17.0, visionOS 1.0, *) +extension DiscardingTaskGroup: CompatDiscardingTaskGroup { + @usableFromInline + package typealias Failure = NoThrow + + @_disfavoredOverload + @inlinable + package mutating func addTaskUnlessCancelled(priority: TaskPriority?, operation: @escaping Block) -> Bool { + let block = { @Sendable () async throws(Never) -> Void in + try? await operation() + } + return addTaskUnlessCancelled(priority: priority, operation: block) + } + + @_disfavoredOverload + @inlinable + package mutating func addTask(priority: TaskPriority?, operation: @escaping Block) { + let block = { @Sendable () async throws(Never) -> Void in + try? await operation() + } + return addTask(priority: priority, operation: block) + } + + @_disfavoredOverload + @available(macOS 15.0, iOS 18.0, watchOS 11.0, tvOS 18.0, visionOS 2.0, *) + @inlinable + package mutating func addTask(executorPreference taskExecutor: (any TaskExecutor)?, priority: TaskPriority?, operation: @escaping Block) { + let block = { @Sendable () async throws(Never) -> Void in + try? await operation() + } + addTask(executorPreference: taskExecutor, priority: priority, operation: block) + } + + @_disfavoredOverload + @available(macOS 15.0, iOS 18.0, watchOS 11.0, tvOS 18.0, visionOS 2.0, *) + @inlinable + package mutating func addTaskUnlessCancelled(executorPreference taskExecutor: (any TaskExecutor)?, priority: TaskPriority?, operation: @escaping Block) -> Bool { + let block = { @Sendable () async throws(Never) -> Void in + try? await operation() + } + return addTaskUnlessCancelled(executorPreference: taskExecutor, priority: priority, operation: block) + + } + +} + +extension TaskGroup: CompatDiscardingTaskGroup where ChildTaskResult == Void { + + @_disfavoredOverload + @inlinable + package mutating func addTaskUnlessCancelled(priority: TaskPriority?, operation: @escaping Block) -> Bool { + let block = { @Sendable in + try? await operation() + return + } + return addTaskUnlessCancelled(priority: priority, operation: block) + } + + @_disfavoredOverload + @inlinable + package mutating func addTask(priority: TaskPriority?, operation: @escaping Block) { + let block = { @Sendable in + try? await operation() + return + } + return addTask(priority: priority, operation: block) + } + + @_disfavoredOverload + @available(macOS 15.0, iOS 18.0, watchOS 11.0, tvOS 18.0, visionOS 2.0, *) + @inlinable + package mutating func addTask(executorPreference taskExecutor: (any TaskExecutor)?, priority: TaskPriority?, operation: @escaping Block) { + let block = { @Sendable in + try? await operation() + return + } + addTask(executorPreference: taskExecutor, priority: priority, operation: block) + } + + @_disfavoredOverload + @available(macOS 15.0, iOS 18.0, watchOS 11.0, tvOS 18.0, visionOS 2.0, *) + @inlinable + package mutating func addTaskUnlessCancelled(executorPreference taskExecutor: (any TaskExecutor)?, priority: TaskPriority?, operation: @escaping Block) -> Bool { + let block = { @Sendable in + try? await operation() + return + } + return addTaskUnlessCancelled(executorPreference: taskExecutor, priority: priority, operation: block) + + } + + @usableFromInline + package typealias Failure = NoThrow + + +} + +@available(macOS 14.0, iOS 17.0, watchOS 10.0, tvOS 17.0, macCatalyst 17.0, visionOS 1.0, *) +extension ThrowingDiscardingTaskGroup: CompatDiscardingTaskGroup { + +} + + +extension ThrowingTaskGroup: CompatDiscardingTaskGroup where ChildTaskResult == Void { + +} + + +@usableFromInline +package enum NoThrow: Error { + + case failure(Never) + +} diff --git a/Sources/BackportDiscardingTaskGroup/imp.swift b/Sources/BackportDiscardingTaskGroup/imp.swift deleted file mode 100644 index da67ba8..0000000 --- a/Sources/BackportDiscardingTaskGroup/imp.swift +++ /dev/null @@ -1,30 +0,0 @@ -// -// File.swift -// -// -// Created by 박병관 on 5/16/24. -// - -//@available(macOS 14.0, iOS 17.0, watchOS 10.0, tvOS 17.0, macCatalyst 17.0, visionOS 1.0, *) -//extension DiscardingTaskGroup: CompatDiscardingTaskGroup { -// @usableFromInline -// package typealias Failure = Never -//} -// -//extension TaskGroup: CompatDiscardingTaskGroup where ChildTaskResult == Void { -// @usableFromInline -// package typealias Failure = Never -// -// -//} - -@available(macOS 14.0, iOS 17.0, watchOS 10.0, tvOS 17.0, macCatalyst 17.0, visionOS 1.0, *) -extension ThrowingDiscardingTaskGroup: CompatDiscardingTaskGroup { - -} - - -extension ThrowingTaskGroup: CompatDiscardingTaskGroup where ChildTaskResult == Void { - -} - diff --git a/Sources/CriticalSection/Cell.swift b/Sources/CriticalSection/Cell.swift new file mode 100644 index 0000000..a06b520 --- /dev/null +++ b/Sources/CriticalSection/Cell.swift @@ -0,0 +1,40 @@ +// +// Cell.swift +// +// +// Created by 박병관 on 6/26/24. +// +import Builtin + +#if $BuiltinAddressOfRawLayout + +@frozen +@usableFromInline +//@_rawLayout(like: Value, movesAsLike) +internal struct _Cell: ~Copyable { + + + @usableFromInline + internal var _address: UnsafeMutablePointer { + UnsafeMutablePointer(_rawAddress) + } + + @usableFromInline + internal var _rawAddress: Builtin.RawPointer { + Builtin.addressOfRawLayout(self) + } + + + @usableFromInline + internal init(_ initialValue: consuming Value) { + _address.initialize(to: initialValue) + } + + @inlinable + deinit { + _address.deinitialize(count: 1) + } + +} + +#endif diff --git a/Sources/CriticalSection/ManagedUnfairLock.swift b/Sources/CriticalSection/ManagedUnfairLock.swift new file mode 100644 index 0000000..397f61f --- /dev/null +++ b/Sources/CriticalSection/ManagedUnfairLock.swift @@ -0,0 +1,326 @@ +// +// ManagedUnfairLock.swift +// +// +// Created by pbk on 2022/12/14. +// + +import Foundation +import os + +@available(iOS, deprecated: 16.0, renamed: "OSAllocatedUnfairLock") +@available(tvOS, deprecated: 16.0, renamed: "OSAllocatedUnfairLock") +@available(macCatalyst, deprecated: 16.0, renamed: "OSAllocatedUnfairLock") +@available(watchOS, deprecated: 9.0, renamed: "OSAllocatedUnfairLock") +@available(macOS, deprecated: 13.0, renamed: "OSAllocatedUnfairLock") +public struct ManagedUnfairLock: @unchecked Sendable { + + @usableFromInline + internal let __lock:ManagedBuffer + + /// Initialize an SwiftUnfairLock with a non-sendable lock-protected + /// `initialState`. + /// + /// By initializing with a non-sendable type, the owner of this structure + /// must ensure the Sendable contract is upheld manually. + /// Non-sendable content from `State` should not be allowed + /// to escape from the lock. + /// + /// - Parameter initialState: An initial value to store that will be + /// protected under the lock. + /// + @inlinable + public init(uncheckedState initialState: State) { + __lock = .create(minimumCapacity: 1) { buffer in + buffer.withUnsafeMutablePointerToElements{ $0.initialize(to: .init()) } + return initialState + } + } + + /// Perform a closure while holding this lock. + /// This method does not enforce sendability requirement + /// on closure body and its return type. + /// The caller of this method is responsible for ensuring references + /// to non-sendables from closure uphold the Sendability contract. + /// + /// - Parameter body: A closure to invoke while holding this lock. + /// - Returns: The return value of `body`. + /// - Throws: Anything thrown by `body`. + /// + @inlinable + public func withLockUnchecked(_ body: (inout State) throws(Failure) -> R) throws(Failure) -> R { + try __lock.withUnsafeMutablePointers{ state, lock throws(Failure) in + os_unfair_lock_lock(lock) + defer { os_unfair_lock_unlock(lock) } + return try body(&state.pointee) + } + } + + /// Perform a sendable closure while holding this lock. + /// + /// + /// - Parameter body: A sendable closure to invoke while holding this lock. + /// - Returns: The sendable return value of `body`. + /// - Throws: Anything thrown by `body`. + /// + @inlinable + public func withLock(_ body: @Sendable (inout State) throws(Failure) -> R) throws(Failure) -> R where R : Sendable { + try withLockUnchecked(body) + } + + /// Attempt to acquire the lock, if successful, perform a closure while + /// holding the lock. + /// This method does not enforce sendability requirement + /// on closure body and its return type. + /// The caller of this method is responsible for ensuring references + /// to non-sendables from closure uphold the Sendability contract. + /// + /// - Parameter body: A closure to invoke while holding this lock. + /// - Returns: If the lock is acquired, the result of `body`. + /// If the lock is not acquired, nil. + /// - Throws: Anything thrown by `body`. + /// + @inlinable + public func withLockIfAvailableUnchecked(_ body: (inout State) throws(Failure) -> R) throws(Failure) -> R? { + try __lock.withUnsafeMutablePointers{ state, lock throws(Failure) in + guard os_unfair_lock_trylock(lock) else { return nil } + defer { os_unfair_lock_unlock(lock) } + return try body(&state.pointee) + } + } + + /// Attempt to acquire the lock, if successful, perform a sendable closure while + /// holding the lock. + /// + /// - Parameter body: A closure to invoke while holding this lock. + /// - Returns: If the lock is acquired, the result of `body`. + /// If the lock is not acquired, nil. + /// - Throws: Anything thrown by `body`. + /// + @inlinable + public func withLockIfAvailable(_ body: @Sendable (inout State) throws(Failure) -> R) throws(Failure) -> R? where R : Sendable { + try withLockIfAvailableUnchecked(body) + } + + @frozen + public enum Ownership: Sendable, Hashable { + case owner + case notOwner + } + + /// Check a precondition about whether the calling thread is the lock owner. + /// + /// - Parameter condition: An `Ownership` statement to check for the + /// current context. + /// - If the lock is currently owned by the calling thread: + /// - `.owner` - returns + /// - `.notOwner` - asserts and terminates the process + /// - If the lock is unlocked or owned by a different thread: + /// - `.owner` - asserts and terminates the process + /// - `.notOwner` - returns + /// + @inlinable + public func precondition(_ condition: Ownership) { + __lock.withUnsafeMutablePointerToElements { + switch condition { + case .notOwner: + os_unfair_lock_assert_not_owner($0) + case .owner: + os_unfair_lock_assert_owner($0) + } + } + } + +} + + +public extension ManagedUnfairLock where State == Void { + + /// Initialize an SwiftUnfairLock with no protected state. + @inlinable + init() { + __lock = .create(minimumCapacity: 1) { buffer in + buffer.withUnsafeMutablePointerToElements{ $0.initialize(to: .init()) } + } + } + + /// Acquire this lock. + @_unavailableFromAsync(message: "Use async-safe scoped locking instead") + @inlinable + func lock() { + __lock.withUnsafeMutablePointerToElements { + os_unfair_lock_lock($0) + } + } + + /// Unlock this lock. + @_unavailableFromAsync(message: "Use async-safe scoped locking instead") + @inlinable + func unlock() { + __lock.withUnsafeMutablePointerToElements{ os_unfair_lock_unlock($0) } + } + + /// Perform a sendable closure while holding this lock. + /// + /// - Parameter body: A sendable closure to invoke while holding this lock. + /// - Returns: The return value of `body`. + /// - Throws: Anything thrown by `body`. + /// + @inlinable + func withLock(_ body: @Sendable () throws(Failure) -> R) throws(Failure) -> R where R : Sendable { + try withLockUnchecked(body) + } + + /// Perform a closure while holding this lock. + /// This method does not enforce sendability requirement + /// on closure body and its return type. + /// The caller of this method is responsible for ensuring references + /// to non-sendables from closure uphold the Sendability contract. + /// + /// - Parameter body: A closure to invoke while holding this lock. + /// - Returns: The return value of `body`. + /// - Throws: Anything thrown by `body`. + /// + @inlinable + func withLockUnchecked(_ body: () throws(Failure) -> R) throws(Failure) -> R { + try __lock.withUnsafeMutablePointerToElements { lock throws(Failure) in + os_unfair_lock_lock(lock) + defer { os_unfair_lock_unlock(lock) } + return try body() + } + } + + /// Attempt to acquire the lock if it is not already locked. + /// + /// - Returns: `true` if the lock was succesfully locked, and + /// `false` if the lock attempt failed. + @available(*, noasync, message: "Use async-safe scoped locking instead") + @inlinable + func lockIfAvailable() -> Bool { + __lock.withUnsafeMutablePointerToElements { os_unfair_lock_trylock($0) } + } + + /// Attempt to acquire the lock, if successful, perform a sendable closure while + /// holding the lock. + /// + /// - Parameter body: A sendable closure to invoke while holding this lock. + /// - Returns: If the lock is acquired, the result of `body`. + /// If the lock is not acquired, nil. + /// - Throws: Anything thrown by `body`. + /// + @inlinable + func withLockIfAvailable(_ body: @Sendable () throws(Failure) -> R) throws(Failure) -> R? where R : Sendable { + try withLockIfAvailableUnchecked(body) + } + + /// Attempt to acquire the lock, if successful, perform a closure while + /// holding the lock. + /// This method does not enforce sendability requirement + /// on closure body and its return type. + /// The caller of this method is responsible for ensuring references + /// to non-sendables from closure uphold the Sendability contract. + /// + /// - Parameter body: A closure to invoke while holding this lock. + /// - Returns: If the lock is acquired, the result of `body`. + /// If the lock is not acquired, nil. + /// - Throws: Anything thrown by `body`. + /// + @inlinable + func withLockIfAvailableUnchecked(_ body: () throws(Failure) -> R) throws(Failure) -> R? { + try __lock.withUnsafeMutablePointerToElements{ lock throws(Failure) in + guard os_unfair_lock_trylock(lock) else { return nil } + defer { os_unfair_lock_unlock(lock) } + return try body() + } + } + +} + +public extension ManagedUnfairLock { + + /// Initialize an SwiftUnfairLock with a lock-protected sendable + /// `initialState`. + /// - Parameter initialState: An initial value to store that will be + /// protected under the lock. + @inlinable + init(initialState: State) where State:Sendable { + __lock = .create(minimumCapacity: 1) { buffer in + buffer.withUnsafeMutablePointerToElements{ $0.initialize(to: .init()) } + return initialState + } + } + +} + +@usableFromInline +package protocol UnfairStateLock: Sendable { + + associatedtype State + @inlinable + func withLock(_ body: @Sendable (inout State) throws -> R) rethrows -> R where R : Sendable + @inlinable + func withLockUnchecked(_ body: (inout State) throws -> R) rethrows -> R + @inlinable + func withLockIfAvailableUnchecked(_ body: (inout State) throws -> R) rethrows -> R? + @inlinable + init(uncheckedState initialState: State) + @inlinable + func withLockIfAvailable(_ body: @Sendable (inout State) throws -> R) rethrows -> R? where R: Sendable + +} + +@usableFromInline +package protocol UnfairLockProtocol: Sendable { + + @inlinable + init() + + @inlinable + @_unavailableFromAsync(message: "Use async-safe scoped locking instead") + func lock() + + @inlinable + @_unavailableFromAsync(message: "Use async-safe scoped locking instead") + func unlock() + + @inlinable + func withLockUnchecked(_ body: () throws -> R) rethrows -> R + + @inlinable + func withLock(_ body: @Sendable () throws -> R) rethrows -> R where R : Sendable + +} + +@available(iOS 16.0, tvOS 16.0, macOS 13.0, macCatalyst 16.0, watchOS 9.0, *) +extension OSAllocatedUnfairLock: UnfairStateLock {} +@available(iOS 16.0, tvOS 16.0, macOS 13.0, macCatalyst 16.0, watchOS 9.0, *) +extension OSAllocatedUnfairLock: UnfairLockProtocol {} +extension ManagedUnfairLock: UnfairStateLock {} +extension ManagedUnfairLock: UnfairLockProtocol {} + +@inlinable +package func createUncheckedStateLock(uncheckedState initialState:State) -> some UnfairStateLock { + if #available(iOS 16.0, tvOS 16.0, macCatalyst 16.0, watchOS 9.0, macOS 13.0, *) { + return OSAllocatedUnfairLock(uncheckedState: initialState) + } else { + return ManagedUnfairLock(uncheckedState: initialState) + } +} + +@inlinable +package func createCheckedStateLock(checkedState initialState:State) -> some UnfairStateLock { + if #available(iOS 16.0, tvOS 16.0, macCatalyst 16.0, watchOS 9.0, macOS 13.0, *) { + return OSAllocatedUnfairLock(initialState: initialState) + } else { + return ManagedUnfairLock(initialState: initialState) + } +} + +@inlinable +package func createUnfairLock() -> some UnfairLockProtocol { + if #available(iOS 16.0, tvOS 16.0, macCatalyst 16.0, watchOS 9.0, macOS 13.0, *) { + return OSAllocatedUnfairLock() + } else { + return ManagedUnfairLock() + } +} diff --git a/Sources/Tetra/Combine/AsyncFlatMapDemandState.swift b/Sources/Tetra/Combine/AsyncFlatMapDemandState.swift index 8955a2a..00f625c 100644 --- a/Sources/Tetra/Combine/AsyncFlatMapDemandState.swift +++ b/Sources/Tetra/Combine/AsyncFlatMapDemandState.swift @@ -7,7 +7,7 @@ import Foundation import Combine -import DequeModule +internal import DequeModule struct AsyncFlatMapDemandState: Sendable { diff --git a/Sources/Tetra/Combine/AsyncFlatMapError.swift b/Sources/Tetra/Combine/AsyncFlatMapError.swift deleted file mode 100644 index 566682c..0000000 --- a/Sources/Tetra/Combine/AsyncFlatMapError.swift +++ /dev/null @@ -1,97 +0,0 @@ -// -// AsyncFlatMapError.swift -// -// -// Created by 박병관 on 6/9/24. -// - -import Foundation - - -@usableFromInline -enum AsyncFlatMapError: Error { - - case upstream(First) - case transform(Second) - case segment(Third) - -} - - -extension AsyncFlatMapError{ - - @inlinable - func unwrap() -> Never where First == Second, Second == Third, Third == Never { - fatalError() - } - - @inlinable - func unwrap() -> First where First == Second, Second == Third { - switch self { - case .upstream(let error): - fallthrough - case .segment(let error): - fallthrough - case .transform(let error): - return (error) - } - } - - @inlinable - func unwrap() -> First where First == Second, Third == Never { - switch self { - case .upstream(let error): - fallthrough - case .transform(let error): - return error - - } - } - - @inlinable - func unwrap() -> Second where Second == Third, First == Never { - switch self { - case .transform(let error): - fallthrough - case .segment(let error): - return error - } - } - - @inlinable - func unwrap() -> Third where First == Third, Second == Never { - switch self { - case .upstream(let error): - fallthrough - case .segment(let error): - return error - } - } - - @inlinable - func unwrap() -> First where Second == Third, Third == Never { - switch self { - case .upstream(let error): - return error - } - } - - @inlinable - func unwrap() -> Second where First == Third, First == Never { - switch self { - case .transform(let error): - return error - } - } - - @inlinable - func unwrap() -> Third where First == Second, Second == Never { - switch self { - case .segment(let error): - return error - } - } - - -} - diff --git a/Sources/Tetra/Combine/AsyncSubscriber.swift b/Sources/Tetra/Combine/AsyncSubscriber.swift index c02bcc9..ca9a596 100644 --- a/Sources/Tetra/Combine/AsyncSubscriber.swift +++ b/Sources/Tetra/Combine/AsyncSubscriber.swift @@ -7,6 +7,7 @@ import Foundation @preconcurrency import Combine +internal import CriticalSection @usableFromInline internal struct AsyncSubscriber: Sendable, Subscriber, Cancellable { diff --git a/Sources/Tetra/Combine/AsyncSubscriptionState.swift b/Sources/Tetra/Combine/AsyncSubscriptionState.swift index 9d6fde2..543e637 100644 --- a/Sources/Tetra/Combine/AsyncSubscriptionState.swift +++ b/Sources/Tetra/Combine/AsyncSubscriptionState.swift @@ -58,6 +58,27 @@ enum AsyncSubscriptionState { } + @usableFromInline + func shouldMutate(_ event: Event) -> Bool { + switch self { + case .waiting: + return true + case .suspending(_): + if case .suspend = event { + return false + } else { + return true + } + case .cached(_): + if case .resume = event { + return false + } else { + return true + } + case .cancelled, .finished: + return false + } + } mutating func transition(_ event:Event) -> sending Effect? { switch event { @@ -98,10 +119,9 @@ enum AsyncSubscriptionState { case .suspending(let unsafeContinuation): self = .cached(subscription) return .resume(unsafeContinuation) - case .cached(let old): - self = .cached(subscription) + case .cached: assertionFailure("Received Subscription more than Once") - return .cancel(old) + return .cancel(subscription) case .cancelled: fallthrough case .finished: @@ -115,9 +135,9 @@ enum AsyncSubscriptionState { self = .suspending(continuation) return nil case .suspending(let unsafeContinuation): - self = .suspending(continuation) + self = .suspending(unsafeContinuation) assertionFailure("Received Continuation more than Once") - return .raise(unsafeContinuation) + return .raise(continuation) case .cancelled: return .raise(continuation) case .finished: diff --git a/Sources/Tetra/Combine/Combine+Concurrency.swift b/Sources/Tetra/Combine/Combine+Concurrency.swift index fb66ba5..722d848 100644 --- a/Sources/Tetra/Combine/Combine+Concurrency.swift +++ b/Sources/Tetra/Combine/Combine+Concurrency.swift @@ -7,6 +7,7 @@ import Foundation import Combine +internal import BackPortAsyncSequence public extension Publisher { @inlinable @@ -54,12 +55,20 @@ public extension Publisher { } + internal extension Publisher { +// +// func asyncFlatMap( +// maxTasks: Subscribers.Demand = .unlimited, +// transform: @escaping @Sendable @isolated(any) (Output) async throws(any Error) -> sending Segment +// ) -> AsyncFlatMap> where Output:Sendable { +// return AsyncFlatMap(maxTasks: maxTasks, upstream: self, transform: transform) +// } - func asyncFlatMap( + func asyncFlatMap( maxTasks: Subscribers.Demand = .unlimited, - transform: @escaping @Sendable @isolated(any) (Output) async throws(Err) -> sending Segment - ) -> AsyncFlatMap, Err> where Output:Sendable { + transform: @escaping @Sendable @isolated(any) (Output) async throws(Failure) -> sending Segment + ) -> AsyncFlatMap where Output:Sendable, Segment.Err == Failure { return AsyncFlatMap(maxTasks: maxTasks, upstream: self, transform: transform) } diff --git a/Sources/Tetra/Combine/CompatAsyncPublisher.swift b/Sources/Tetra/Combine/CompatAsyncPublisher.swift index b68f063..14cc637 100644 --- a/Sources/Tetra/Combine/CompatAsyncPublisher.swift +++ b/Sources/Tetra/Combine/CompatAsyncPublisher.swift @@ -8,6 +8,7 @@ import Foundation @preconcurrency import Combine +internal import BackPortAsyncSequence public struct CompatAsyncPublisher: AsyncSequence where P.Failure == Never { @@ -36,9 +37,15 @@ public struct CompatAsyncPublisher: AsyncSequence where P.Failure = internal let inner = AsyncSubscriber

() @usableFromInline internal let reference:AnyCancellable - + + @_disfavoredOverload + @inlinable + public mutating func next() async throws(Never) -> P.Output? { + await next(isolation: nil) + } + @inlinable - public func next(isolation actor: isolated (any Actor)?) async -> P.Output? { + public func next(isolation actor: isolated (any Actor)? = #isolation) async -> P.Output? { let result: Result? = await withTaskCancellationHandler { [inner] in await inner.next(isolation: actor) } onCancel: { [reference] in diff --git a/Sources/Tetra/Combine/CompatAsyncThrowingPublisher.swift b/Sources/Tetra/Combine/CompatAsyncThrowingPublisher.swift index 6cd0cf3..47c7918 100644 --- a/Sources/Tetra/Combine/CompatAsyncThrowingPublisher.swift +++ b/Sources/Tetra/Combine/CompatAsyncThrowingPublisher.swift @@ -8,11 +8,12 @@ import Foundation @preconcurrency import Combine +internal import BackPortAsyncSequence public struct CompatAsyncThrowingPublisher: AsyncSequence, TypedAsyncSequence { public typealias AsyncIterator = Iterator - public typealias Failure = AsyncIterator.Failure + public typealias Failure = P.Failure public var publisher:P @@ -31,7 +32,7 @@ public struct CompatAsyncThrowingPublisher: AsyncSequence, TypedAsy internal let reference:AnyCancellable @inlinable - public mutating func next(isolation actor: isolated (any Actor)?) async throws(P.Failure) -> P.Output? { + public mutating func next(isolation actor: isolated (any Actor)? = #isolation) async throws(P.Failure) -> P.Output? { let result = await withTaskCancellationHandler { [inner] in await inner.next(isolation: actor) } onCancel: { [reference] in @@ -48,6 +49,12 @@ public struct CompatAsyncThrowingPublisher: AsyncSequence, TypedAsy } } + @_disfavoredOverload + @inlinable + public mutating func next() async throws(P.Failure) -> P.Output? { + try await next(isolation: nil) + } + @usableFromInline internal init(source: P) { self.reference = AnyCancellable(inner) diff --git a/Sources/Tetra/Combine/DispatchTimePublisher.swift b/Sources/Tetra/Combine/DispatchTimePublisher.swift index f39b307..2675abc 100644 --- a/Sources/Tetra/Combine/DispatchTimePublisher.swift +++ b/Sources/Tetra/Combine/DispatchTimePublisher.swift @@ -8,6 +8,7 @@ import Foundation import Dispatch import Combine +internal import CriticalSection extension DispatchSource: TetraExtended {} diff --git a/Sources/Tetra/Combine/ExperimentalMapTask.swift b/Sources/Tetra/Combine/ExperimentalMapTask.swift index b6b40eb..9a67599 100644 --- a/Sources/Tetra/Combine/ExperimentalMapTask.swift +++ b/Sources/Tetra/Combine/ExperimentalMapTask.swift @@ -7,9 +7,8 @@ import Foundation @preconcurrency import Combine - - - +internal import CriticalSection +internal import BackportDiscardingTaskGroup /** Manage Multiple Child Task. provides similair behavior of `flatMap`'s `maxPublisher` @@ -21,8 +20,8 @@ public struct MultiMapTask: Publisher where Upstream public typealias Output = Output public typealias Failure = Upstream.Failure - - public let maxTasks:Subscribers.Demand + public var priority:TaskPriority? = nil + public var maxTasks:Subscribers.Demand public let upstream:Upstream public let transform: @isolated(any) @Sendable (Upstream.Output) async throws(Failure) -> sending Output public let taskExecutor: (any Executor)? @@ -30,9 +29,9 @@ public struct MultiMapTask: Publisher where Upstream public func receive(subscriber: S) where S : Subscriber, Upstream.Failure == S.Failure, Output == S.Input { let processor = Inner(maxTasks: maxTasks, subscriber: subscriber, transform: transform) let task = if #available(macOS 15.0, iOS 18.0, watchOS 11.0, tvOS 18.0, visionOS 2.0, *), let executor = taskExecutor as? (any TaskExecutor) { - Task(executorPreference: executor, operation: processor.run) + Task(executorPreference: executor, priority: priority, operation: processor.run) } else { - Task(operation: processor.run) + Task(priority: priority, operation: processor.run) } processor.resumeCondition(task) upstream.subscribe(processor) @@ -40,6 +39,7 @@ public struct MultiMapTask: Publisher where Upstream public init( + priority: TaskPriority? = nil, maxTasks: Subscribers.Demand = .max(1), upstream: Upstream, transform: @Sendable @escaping @isolated(any) (Upstream.Output) async throws(Failure) -> Output @@ -49,10 +49,12 @@ public struct MultiMapTask: Publisher where Upstream self.upstream = upstream self.transform = transform self.taskExecutor = nil + self.priority = priority } @available(macOS 15.0, iOS 18.0, watchOS 11.0, tvOS 18.0, visionOS 2.0, *) public init( + priority:TaskPriority? = nil, maxTasks: Subscribers.Demand = .max(1), executor:(any TaskExecutor)? = nil, upstream: Upstream, @@ -63,6 +65,7 @@ public struct MultiMapTask: Publisher where Upstream self.upstream = upstream self.transform = transform self.taskExecutor = executor + self.priority = priority } } @@ -83,9 +86,10 @@ extension MultiMapTask { private let maxTasks:Subscribers.Demand private let valueSource = AsyncStream>.makeStream() + // accessed from Combine intferface or isolated Actor + // which ever guarantee serialized access private let state: some UnfairStateLock> = createUncheckedStateLock(uncheckedState: TaskState()) private let transform:@Sendable (Upstream.Output) async throws(Failure) -> Output - let combineIdentifier = CombineIdentifier() init( @@ -100,24 +104,48 @@ extension MultiMapTask { } } +// private func prepareTermination(cancel:Bool) { +// let effect = state.withLockUnchecked { +// let effect1 = $0.condition.transition(cancel ? .cancel : .finish) +// let effect2 = $0.upstreamSubscription.transition(cancel ? .cancel : .finish) +// return (effect1, effect2) +// } +// effect.0?.run() +// effect.1?.run() +// } + internal func localTask( isolation actor: isolated (any Actor)? = #isolation, - group: inout some CompatThrowingDiscardingTaskGroup - ) async throws { - for await upstreamValue in valueSource.stream { - switch upstreamValue { + group: inout some CompatDiscardingTaskGroup + ) async { + let barrier = actor as? SafetyRegion ?? SafetyRegion() + + for await event in valueSource.stream { + if await barrier.isFinished { + break + } + switch event { case .failure(let failure): - send(completion: .failure(failure), cancel: false) - throw CancellationError() + await barrier.markDone() + // no contention except `request` and `cancel` + await send(barrier: barrier, completion: .failure(failure), cancel: false) + break case .success(let success): let flag = group.addTaskUnlessCancelled(priority: nil) { let result = await wrapToResult(success, transform) switch result { case .failure(let error): - send(completion: .failure(error), cancel: true) - throw CancellationError() + await barrier.markDone() + // no contention except `request` and `cancel` + await send(barrier: barrier, completion: .failure(error), cancel: true) case .success(let success): - try send(success) + do { + // no contention except `request` and `cancel` + try await send(isolation: barrier, success) + } catch { + await barrier.markDone() +// token.store(true, ordering: .releasing) + } } } if !flag { @@ -131,47 +159,59 @@ extension MultiMapTask { valueSource.continuation.finish() } - private func send(completion: Subscribers.Completion?, cancel:Bool = false) { + private func send( + barrier: isolated (some Actor)? = #isolation, + completion: Subscribers.Completion, + cancel:Bool = false + ) { terminateStream() - let (subscriber, effect) = state.withLockUnchecked{ + let (subscriber, taskEffect) = state.withLockUnchecked{ let old = $0.subscriber $0.subscriber = nil - let effect = if cancel { - $0.upstreamSubscription.transition(.cancel) + let taskEffect = if cancel { + $0.condition.transition(.cancel) } else { - $0.upstreamSubscription.transition(.finish) + $0.condition.transition(.finish) } - return (old, effect) + return (old, taskEffect) } - effect?.run() - if let completion, let subscriber { - downStreamLock.withLock { - subscriber.receive(completion: completion) - } + if let subscriber { + subscriber.receive(completion: completion) } + taskEffect?.run() } - private func send(_ value: S.Input) throws { + + private func send( + isolation actor: isolated some Actor, + _ value: S.Input + ) async throws { + let (subscriber, subscription) = state.withLockUnchecked{ - return ($0.subscriber, $0.upstreamSubscription.subscription) } - guard let subscriber, let subscription else { + guard let subscriber else { throw CancellationError() } + // subscriber might call extra `request` or `cancel` but as we don't acquire the lock, it is safe to do so. var demand = subscriber.receive(value) - guard maxTasks != .unlimited else { + // subscription can be null, if upstream is already completed + guard let subscription else { + return + } + defer { if demand > .none { subscription.request(demand) } + } + guard maxTasks != .unlimited else { return } + // yield so that other task can access to subscriber for a while. + await Task.yield() demand = state.withLockUnchecked{ $0.demand.transistion(maxTasks: maxTasks, demand, reduce: true) } - if demand > .none { - subscription.request(demand) - } } private func waitForUpStream() async throws { @@ -210,6 +250,7 @@ extension MultiMapTask { @Sendable nonisolated func run() async { + // contention can happen with `resumeCondition(_ :) ` let token:Void? = try? await waitForCondition() if token == nil { withUnsafeCurrentTask{ @@ -219,6 +260,7 @@ extension MultiMapTask { defer { clearCondition() } + // contention can happen with `receive(subscription:) let success:Void? = try? await waitForUpStream() state.withLockUnchecked{ $0.subscriber @@ -227,33 +269,21 @@ extension MultiMapTask { terminateStream() return } - await withTaskCancellationHandler { - if #available(iOS 17.0, tvOS 17.0, macCatalyst 17.0, macOS 14.0, watchOS 10.0, visionOS 1.0, *) { - try? await withThrowingDiscardingTaskGroup(returning: Void.self) { group in - defer { terminateStream() } - try await localTask( - group: &group - ) - } - } else { - try? await wrapForBackDeploy(isolation: SafetyRegion()) + if #available(iOS 17.0, tvOS 17.0, macCatalyst 17.0, macOS 14.0, watchOS 10.0, visionOS 1.0, *) { + await withDiscardingTaskGroup(returning: Void.self) { group in + await localTask( + group: &group + ) } - send(completion: .finished) - } onCancel: { - send(completion: nil, cancel: true) - } - } - - func wrapForBackDeploy( - isolation actor: isolated SafetyRegion - ) async throws { - try await withThrowingTaskGroup(of: Void.self) { - defer { terminateStream() } - try await $0.simulateDiscarding(isolation: actor) { isolation, group in - try await localTask(isolation: isolation, group: &group) + } else { + await simuateDiscardingTaskGroup(isolation: SafetyRegion()) { actor, group in + await localTask(isolation: actor, group: &group) } - send(completion: nil, cancel: false) } + // we assume no one is accessing other state + // except `Subscription.cancel()` + await send(barrier: SafetyRegion?.none, completion: .finished, cancel: false) + } } @@ -270,13 +300,17 @@ extension MultiMapTask.Inner: Subscriber { } func receive(completion: Subscribers.Completion) { + // release upstream subscription + state.withLockUnchecked{ + $0.upstreamSubscription.transition(.finish) + }?.run() switch completion { case .finished: break case .failure(let failure): valueSource.continuation.yield(.failure(failure)) } - valueSource.continuation.finish() + terminateStream() } typealias Input = Upstream.Output @@ -296,8 +330,12 @@ extension MultiMapTask.Inner: Subscription { func request(_ demand: Subscribers.Demand) { let (subscription, nextDemand) = state.withLock{ let subscription = $0.upstreamSubscription.subscription - let demand = $0.demand.transistion(maxTasks: maxTasks, demand, reduce: false) - return (subscription, demand) + let newDemand = if subscription == nil { + Subscribers.Demand.none + } else { + $0.demand.transistion(maxTasks: maxTasks, demand, reduce: false) + } + return (subscription, newDemand) } if let subscription, nextDemand > .none { subscription.request(nextDemand) @@ -305,9 +343,18 @@ extension MultiMapTask.Inner: Subscription { } func cancel() { - state.withLock{ - $0.condition.transition(.cancel) - }?.run() + terminateStream() + let (task, subscription, subscriber) = state.withLockUnchecked{ + let taskEffect = $0.condition.transition(.cancel) + let subscriptionEffect = $0.upstreamSubscription.transition(.cancel) + let downstream = $0.subscriber + $0.subscriber = nil + return (taskEffect, subscriptionEffect, downstream) + } + withExtendedLifetime(subscriber) { + subscription?.run() + task?.run() + } } } diff --git a/Sources/Tetra/Combine/Future+Concurrency.swift b/Sources/Tetra/Combine/Future+Concurrency.swift index 34ef811..261d6ce 100644 --- a/Sources/Tetra/Combine/Future+Concurrency.swift +++ b/Sources/Tetra/Combine/Future+Concurrency.swift @@ -9,40 +9,34 @@ import Foundation import Combine +extension TetraExtension { -public extension Combine.Future { - - @available(iOS, deprecated: 15.0, renamed: "value") - @available(iOS, deprecated: 15.0, renamed: "value") - @available(iOS, deprecated: 15.0, renamed: "value") - @available(watchOS, deprecated: 8, renamed: "value") - @available(macOS, deprecated: 12.0, renamed: "value") @inlinable - final var compatValue: Output { - get async throws(Failure) { - let result: Result = await withCheckedContinuation { continuation in - self.subscribe(AnySubscriber( - receiveSubscription: { - $0.request(.max(1)) - }, - receiveValue: { (value: sending Output) in - continuation.resume(returning: .success(value)) - return .none - }, - receiveCompletion: { - if case let .failure(error) = $0 { - continuation.resume(returning: .failure(error)) - } + public func next() async throws(Failure) -> Output where Base == Combine.Future { + let result: Result = await withCheckedContinuation { continuation in + base.subscribe(AnySubscriber( + receiveSubscription: { + $0.request(.max(1)) + }, + receiveValue: { (value: sending Output) in + continuation.resume(returning: .success(value)) + return .none + }, + receiveCompletion: { + if case let .failure(error) = $0 { + continuation.resume(returning: .failure(error)) } - )) - } - switch result { - case .success(let success): - return success - case .failure(let failure): - throw failure - } + } + )) + } + switch result { + case .success(let success): + return success + case .failure(let failure): + throw failure } } + + } diff --git a/Sources/Tetra/Combine/PendingDemandState.swift b/Sources/Tetra/Combine/PendingDemandState.swift index cc63bf9..b78a41e 100644 --- a/Sources/Tetra/Combine/PendingDemandState.swift +++ b/Sources/Tetra/Combine/PendingDemandState.swift @@ -8,7 +8,8 @@ import Foundation import Combine -struct PendingDemandState { +struct PendingDemandState{ + private var taskCount:Int private var pendingDemand:Subscribers.Demand @@ -51,7 +52,5 @@ struct PendingDemandState { return snapShot } } - - } diff --git a/Sources/Tetra/Combine/Publishers+AsyncFlatMap.swift b/Sources/Tetra/Combine/Publishers+AsyncFlatMap.swift index d6a3834..c07779d 100644 --- a/Sources/Tetra/Combine/Publishers+AsyncFlatMap.swift +++ b/Sources/Tetra/Combine/Publishers+AsyncFlatMap.swift @@ -7,30 +7,35 @@ import Foundation @preconcurrency import Combine +internal import BackPortAsyncSequence +internal import CriticalSection +internal import BackportDiscardingTaskGroup - -struct AsyncFlatMap: Publisher where Upstream.Output:Sendable{ +struct AsyncFlatMap: Publisher where Upstream.Output:Sendable, Segment.AsyncIterator.Err == Upstream.Failure, Segment.AsyncIterator: TypedAsyncIteratorProtocol { typealias Output = Segment.Element - typealias Failure = AsyncFlatMapError - typealias Transform = @Sendable @isolated(any) (Upstream.Output) async throws(TransformFail) -> sending Segment - let maxTasks:Subscribers.Demand + typealias Failure = Upstream.Failure + typealias Transform = @Sendable (Upstream.Output) async throws(Failure) -> sending Segment + var priority: TaskPriority? = nil + var maxTasks:Subscribers.Demand let upstream:Upstream let transform:Transform func receive(subscriber: S) where S : Subscriber, Failure == S.Failure, Segment.Element == S.Input { let processor = Inner(maxTasks: maxTasks, subscriber: subscriber, transform: transform) - let task = Task(operation: processor.run) + let task = Task(priority: priority, operation: processor.run) processor.resumeCondition(task) upstream.subscribe(processor) } @usableFromInline init( + priority: TaskPriority? = nil, maxTasks: Subscribers.Demand, upstream: Upstream, - transform: @escaping @isolated(any) Transform + transform: @escaping Transform ) { + self.priority = priority self.maxTasks = maxTasks self.upstream = upstream self.transform = transform @@ -38,31 +43,33 @@ struct AsyncFlatMap( + priority: TaskPriority? = nil, maxTasks: Subscribers.Demand, upstream: Upstream, - transform: @escaping @isolated(any) @Sendable (Upstream.Output) async throws(TransformFail) -> sending Source - ) where Source: AsyncSequence, Segment == WrappedAsyncSequence, Segment.AsyncIterator.Failure == any Error { - let block:Transform = { - return .init(base: try await transform($0)) - } + transform: @escaping @Sendable (Upstream.Output) async throws(Failure) -> sending Source + ) where Source: AsyncSequence, Segment == LegacyTypedAsyncSequence, Failure == any Error { + self.priority = priority self.maxTasks = maxTasks self.upstream = upstream - self.transform = block + self.transform = { (value) throws(Failure) in + .init(base: try await transform(value)) + } } @available(macOS 15.0, iOS 18.0, watchOS 11.0, tvOS 18.0, visionOS 2.0, *) @usableFromInline init( + priority: TaskPriority? = nil, maxTasks: Subscribers.Demand, upstream: Upstream, - typedTransform: @escaping @isolated(any) @Sendable (Upstream.Output) async throws(TransformFail) -> sending Source - ) where Source: AsyncSequence, Segment == WrappedAsyncSequenceV2 { - let block:Transform = { - return .init(base: try await typedTransform($0)) - } + typedTransform: @escaping @isolated(any) @Sendable (Upstream.Output) async throws(Failure) -> sending Source + ) where Source: AsyncSequence, Segment == WrappedAsyncSequence { + self.priority = priority self.maxTasks = maxTasks self.upstream = upstream - self.transform = block + self.transform = { (value) throws(Failure) in + .init(base: try await typedTransform(value)) + } } @@ -73,11 +80,10 @@ extension AsyncFlatMap { struct Inner: Subscriber, Sendable, Subscription, CustomStringConvertible, CustomPlaygroundDisplayConvertible where Segment.Element == Down.Input, Down.Failure == AsyncFlatMap.Failure { - typealias Transformer = @Sendable (Upstream.Output) async throws(TransformFail) -> sending Segment + typealias Transformer = @Sendable (Upstream.Output) async throws(Failure) -> sending Segment typealias Input = Upstream.Output typealias Failure = Upstream.Failure - let lock:some UnfairStateLock = createUncheckedStateLock(uncheckedState: TaskState()) struct TaskState { @@ -120,25 +126,21 @@ extension AsyncFlatMap { terminateStream() return } - await withTaskCancellationHandler { - let isCancelled:Bool - if #available(iOS 17.0, tvOS 17.0, macCatalyst 17.0, macOS 14.0, watchOS 10.0, visionOS 1.0, *) { - let void:Void? = try? await withThrowingDiscardingTaskGroup { group in - defer { terminateStream() } - try await localTask(group: &group) - } - - isCancelled = void == nil - } else { - let void:Void? = try? await wrapForBackDeploy(isolation: SafetyRegion()) - isCancelled = void == nil + if #available(iOS 17.0, tvOS 17.0, macCatalyst 17.0, macOS 14.0, watchOS 10.0, visionOS 1.0, *) { + try? await withThrowingDiscardingTaskGroup { group in + defer { terminateStream() } + await localTask( + group: &group + ) } - if !isCancelled { - send(completion: .finished) + } else { + try? await simuateThrowingDiscardingTaskGroup(isolation: SafetyRegion()) { barrier, group in + defer { terminateStream() } + await localTask(isolation: barrier, group: &group) } - } onCancel: { - send(completion: nil) } + send(completion: .finished) + } var playgroundDescription: Any { description } @@ -152,6 +154,9 @@ extension AsyncFlatMap { } func receive(completion: Subscribers.Completion) { + lock.withLockUnchecked{ + $0.upstreamSubscription.transition(.finish) + }?.run() switch completion { case .finished: break @@ -186,21 +191,61 @@ extension AsyncFlatMap { } func cancel() { - lock.withLockUnchecked{ - $0.taskCondition.transition(.cancel) - }?.run() + send(completion: nil) + } + + // almost uncontented call + private func handleDownStream( + isolation actor: isolated some Actor, + event: Result?, EitherFailure> + ) async { + switch event { + case .success(.none): + //finished + // check and request more transformer + // request one more from upstream subscription + if maxTasks != .unlimited { + let (subscription, effect) = lock.withLockUnchecked{ + let effect = if $0.demandState.pending == .unlimited { + $0.demandState.transition(.resume(.none)) + } else { + // reclaim discarded demand + $0.demandState.transition(.resume(.max(1))) + } + let subscription = $0.upstreamSubscription.subscription + return (subscription, effect) + } + if let subscription { + subscription.request(.max(1)) + effect?.run() + } + } + case .success(let success?): + send( + isolation: actor, + success.value + ) + return + case .failure(let failure): + send(completion: .failure(failure)) + return + } + return } - private func send(completion: Subscribers.Completion?) { + // almost uncontented call + private func send( + completion: Subscribers.Completion>? + ) { valueSource.continuation.finish() let shouldCancel:Bool switch completion { - case .none, .failure(.segment(_)), .failure(.transform(_)): + case .none, .failure(.second(_)): shouldCancel = true - case .failure(.upstream(_)), .finished: + case .failure(.first(_)), .finished: shouldCancel = false } - let (subscriber, effect, interruption) = lock.withLockUnchecked{ + let (subscriber, effect, interruption, taskEffect) = lock.withLockUnchecked{ let old = $0.subscriber $0.subscriber = nil let effect = if shouldCancel { @@ -209,28 +254,44 @@ extension AsyncFlatMap { $0.upstreamSubscription.transition(.finish) } let interruption = $0.demandState.transition(.interrupt) - return (old, effect, interruption) - } - if let completion { - subscriber?.receive(completion: completion) + let taskEffect = if shouldCancel { + $0.taskCondition.transition(.cancel) + } else { + $0.taskCondition.transition(.finish) + } + return (old, effect, interruption, taskEffect) } effect?.run() interruption?.run() + switch completion { + case .finished: + subscriber?.receive(completion: .finished) + case .failure(.first(let error)), .failure(.second(let error)): + subscriber?.receive(completion: .failure(error)) + case nil: + break + } + taskEffect?.run() } - - private func send(_ value:Down.Input) throws(CancellationError) { + // almost uncontented call + private func send( + isolation actor: isolated some Actor, + _ value:Down.Input + ) { + // use lock but this is isolated so we expect uncontended let subscriber = lock.withLockUnchecked { $0.subscriber } guard let newDemand = subscriber?.receive(value) else { - throw CancellationError() + return } lock.withLockUnchecked{ $0.demandState.transition(.resume(newDemand)) }?.run() } + // contention case private func waitForUpStream() async throws { try await withTaskCancellationHandler { try await withUnsafeThrowingContinuation { coninuation in @@ -245,12 +306,14 @@ extension AsyncFlatMap { } } + // contention case func resumeCondition(_ task:Task) { lock.withLock{ $0.taskCondition.transition(.resume(task)) }?.run() } + // contention case private func waitForCondition() async throws { try await withUnsafeThrowingContinuation{ continuation in lock.withLock{ @@ -258,7 +321,7 @@ extension AsyncFlatMap { }?.run() } } - + // almost uncontented call private func clearCondition() { lock.withLock{ $0.taskCondition.transition(.finish) @@ -269,27 +332,24 @@ extension AsyncFlatMap { valueSource.continuation.finish() } - private func makeSegment(_ input:Upstream.Output) async throws(CancellationError) -> sending Segment { - let result:Result + private func makeSegment(_ input:Upstream.Output) async -> sending Result { + let result:Result do { let seg = try await transform(input) result = .success(seg) } catch { result = .failure(error ) } - switch result { - case .success(let success): - return success - case .failure(let failure): - send(completion: .failure(.transform(failure))) - throw CancellationError() - } + return result } + // almost uncontented call /// whether demand is unlimited /// - Returns: `true` if demand is unlimited, `false` if demand is just `1`. /// - throws: `CancellationError` if internal state reached cancellation - private func nextDemand() async throws -> Bool { + private func nextDemand( + barrier:isolated some Actor + ) async throws -> Bool { try await withUnsafeThrowingContinuation { continuation in lock.withLockUnchecked{ $0.demandState.transition(.suspend(continuation)) @@ -297,79 +357,76 @@ extension AsyncFlatMap { } } - + /// process next segment and send event to downstream /// - Returns: `false` if iterator reached termination otherwise `true` - /// - throws: `CancellationError` if internal state reached cancellation private func processNextSegment( - iterator: inout Segment.AsyncIterator - ) async throws(CancellationError) -> Bool { - let nextResult = await wrapToResult(#isolation, &iterator) - switch nextResult { - case .none: - //finished - // check and request more transformer - // request one more from upstream subscription - if maxTasks != .unlimited { - let (subscription, effect) = lock.withLockUnchecked{ - let effect = if $0.demandState.pending == .unlimited { - $0.demandState.transition(.resume(.none)) - } else { - // reclaim discarded demand - $0.demandState.transition(.resume(.max(1))) - } - let subscription = $0.upstreamSubscription.subscription - return (subscription, effect) - } - if let subscription { - subscription.request(.max(1)) - effect?.run() - } else { - throw CancellationError() - } + iterator: inout Segment.AsyncIterator, + barrier: some Actor + ) async -> Bool { + let result:Result? + do { + if let value = try await iterator.next(isolation: nil) { + result = .success(value) + } else { + result = nil } + } catch { + result = .failure(error) + } + switch result { + case .none: + await handleDownStream(isolation: barrier, event: .success(.none)) return false case .failure(let error): - send(completion: .failure(.segment(error))) - throw CancellationError() + await handleDownStream(isolation: barrier, event: .failure(.second(error))) + return false case .success(let value): - try send(value) - } - return true - } - - func wrapForBackDeploy( - isolation actor: isolated SafetyRegion - ) async throws { - try await withThrowingTaskGroup(of: Void.self) { - defer { terminateStream() } - try await $0.simulateDiscarding(isolation: actor) { isolation, group in - try await localTask(isolation: isolation, group: &group) - } + await handleDownStream(isolation: barrier, event: .success(.init(value: value))) + return true } } private func localTask( isolation actor: isolated (any Actor)? = #isolation, - group: inout some CompatThrowingDiscardingTaskGroup - ) async throws { + group: inout some CompatDiscardingTaskGroup + ) async { + let barrier = actor as? SafetyRegion ?? .init() for await result in valueSource.stream { + if await barrier.isFinished { + break + } switch result { case .failure(let failure): - send(completion: .failure(.upstream(failure))) - throw CancellationError() + await handleDownStream( + isolation: barrier, + event: .failure(.first(failure)) + ) + return case .success(let value): let isSuccess = group.addTaskUnlessCancelled(priority: nil) { - let segment = try await makeSegment(value) - var iterator = segment.makeAsyncIterator() + let segmentResult = await makeSegment(value) + var iterator:Segment.AsyncIterator + switch segmentResult { + case .failure(let failure): + await barrier.markDone() + await handleDownStream( + isolation: barrier, + event: .failure(.second(failure)) + ) + return + case .success(let source): + iterator = source.makeAsyncIterator() + } while true { - let isUnlimited = try await nextDemand() + let isUnlimited = try await nextDemand(barrier: barrier) if isUnlimited { - while try await processNextSegment(iterator: &iterator) { + while await processNextSegment(iterator: &iterator, barrier: barrier) { + } return } else { - let hasNext = try await processNextSegment(iterator: &iterator) + let hasNext = await processNextSegment(iterator: &iterator, barrier: barrier) if !hasNext { return } @@ -377,7 +434,7 @@ extension AsyncFlatMap { } } if !isSuccess { - throw CancellationError() + return } } diff --git a/Sources/Tetra/Combine/Publishers+MapTask.swift b/Sources/Tetra/Combine/Publishers+MapTask.swift index 0647cf9..93d2c42 100644 --- a/Sources/Tetra/Combine/Publishers+MapTask.swift +++ b/Sources/Tetra/Combine/Publishers+MapTask.swift @@ -7,6 +7,7 @@ import Foundation @preconcurrency import Combine +internal import CriticalSection /** @@ -37,14 +38,17 @@ public struct MapTask: Publisher where Upstream.Outp public typealias Output = Output public typealias Failure = Upstream.Failure - + public typealias Transform = @Sendable @isolated(any) (Upstream.Output) async -> sending Result + public var priority: TaskPriority? = nil public let upstream:Upstream - public var transform:@Sendable @isolated(any) (Upstream.Output) async -> sending Result + public var transform:Transform public init( + priority: TaskPriority? = nil, upstream: Upstream, transform: @escaping @Sendable @isolated(any) (Upstream.Output) async -> sending Output ) { + self.priority = priority self.upstream = upstream self.transform = { Result.success(await transform($0)) @@ -52,16 +56,19 @@ public struct MapTask: Publisher where Upstream.Outp } public init( + priority: TaskPriority? = nil, upstream: Upstream, - handler: @escaping @Sendable @isolated(any) (Upstream.Output) async -> sending Result + handler: @escaping Transform ) { + self.priority = priority self.upstream = upstream self.transform = handler } public func receive(subscriber: S) where S : Subscriber, Upstream.Failure == S.Failure, Output == S.Input { + let processor = Inner(subscriber: subscriber, transform: transform) - let task = Task(operation: processor.run) + let task = Task(priority: priority, operation: processor.run) processor.resumeCondition(task) upstream.subscribe(processor) } @@ -86,19 +93,20 @@ extension MapTask { private let valueSource = AsyncStream>.makeStream(bufferingPolicy: .bufferingNewest(2)) private let state: some UnfairStateLock> = createUncheckedStateLock(uncheckedState: .init()) - private let transform:@Sendable (Upstream.Output) async -> Result + private let transform: Transform let combineIdentifier = CombineIdentifier() init( subscriber:S, - transform: @Sendable @escaping (Upstream.Output) async -> Result + transform: @escaping Transform ) { self.transform = transform state.withLockUnchecked{ $0.subscriber = subscriber } } + private func send(completion: Subscribers.Completion?, cancel:Bool = false) { - let (subscriber, effect) = state.withLockUnchecked{ + let (subscriber, effect, taskEffect) = state.withLockUnchecked{ let old = $0.subscriber $0.subscriber = nil let effect = if cancel { @@ -106,12 +114,18 @@ extension MapTask { } else { $0.upstreamSubscription.transition(.finish) } - return (old, effect) + let taskEffect = if cancel { + $0.condition.transition(.cancel) + } else { + $0.condition.transition(.finish) + } + return (old, effect, taskEffect) } effect?.run() if let completion { subscriber?.receive(completion: completion) } + taskEffect?.run() } private func send(_ value:Output) throws { @@ -183,31 +197,42 @@ extension MapTask { guard success != nil else { return } - try? await withTaskCancellationHandler { - for await upstreamResult in valueSource.stream { - let upValue: Upstream.Output - switch upstreamResult { - case .failure(let error): - send(completion: .failure(error), cancel: false) - throw CancellationError() - case .success(let value): - upValue = value - } - // enqueue to separate task to prevent transformer cancelling root task using `UnsafeCurrentTask` - async let job = transform(upValue) - switch (await job) { - case .failure(let error): - send(completion: .failure(error), cancel: true) - throw CancellationError() - case .success(let value): + await runIn(isolation: transform.isolation) + + } + + internal func runIn( + isolation actor:isolated (any Actor)? = #isolation + ) async { + let block = { @Sendable in + let value = await transform($0) + return value.map(Suppress.init) + } + for await upstreamResult in valueSource.stream { + let upValue: Upstream.Output + switch upstreamResult { + case .failure(let error): + send(completion: .failure(error), cancel: false) + return + case .success(let value): + upValue = value + } + + // enqueue to separate task to prevent transformer cancelling root task using `UnsafeCurrentTask` + async let job = block(upValue) + switch (await job).map(\.value) { + case .failure(let error): + send(completion: .failure(error), cancel: true) + return + case .success(let value): + do { try send(value) + } catch { + return } } - send(completion: .finished) - } onCancel: { - send(completion: nil, cancel: true) } - + send(completion: .finished) } } @@ -238,6 +263,9 @@ extension MapTask.Inner: Subscriber { } func receive(completion: Subscribers.Completion) { + state.withLockUnchecked { + $0.upstreamSubscription.transition(.finish) + }?.run() switch completion { case .finished: break @@ -253,9 +281,7 @@ extension MapTask.Inner: Subscriber { extension MapTask.Inner: Subscription { func cancel() { - state.withLock{ - $0.condition.transition(.cancel) - }?.run() + send(completion: nil, cancel: true) } func request(_ demand: Subscribers.Demand) { diff --git a/Sources/Tetra/Combine/Publishers+TryMapTask.swift b/Sources/Tetra/Combine/Publishers+TryMapTask.swift index 79a40cb..921d58f 100644 --- a/Sources/Tetra/Combine/Publishers+TryMapTask.swift +++ b/Sources/Tetra/Combine/Publishers+TryMapTask.swift @@ -8,6 +8,7 @@ import Foundation @preconcurrency import Combine import _Concurrency +internal import CriticalSection /** @@ -34,28 +35,70 @@ public struct TryMapTask: Publisher where Upstream.O public typealias Output = Output public typealias Failure = any Error - + public typealias Transform = @isolated(any) @Sendable (Upstream.Output) async throws -> sending Output public let upstream:Upstream - public var transform: @isolated(any) @Sendable (Upstream.Output) async throws -> sending Output + public var transform: Transform + public var priority: TaskPriority? = nil public init( + priority:TaskPriority? = nil, upstream: Upstream, - transform: @escaping @isolated(any) @Sendable (Upstream.Output) async throws -> sending Output + transform: @escaping Transform ) { self.upstream = upstream self.transform = transform + self.priority = priority } public func receive(subscriber: S) where S : Subscriber, Failure == S.Failure, Output == S.Input { - let processor = Inner(subscriber: subscriber, transform: transform) - let task = Task(operation: processor.run) - processor.resumeCondition(task) - upstream.subscribe(processor) +// let processor = Inner(subscriber: subscriber, transform: transform) +// let task = Task(priority: priority, operation: processor.run) +// processor.resumeCondition(task) +// upstream.subscribe(processor) + MultiMapTask( + priority: priority, + maxTasks: .max(1), + upstream: upstream.mapError{ $0 as any Error }, + transform: transform + ).subscribe(TryMapTaskInner(downstream: subscriber, upstream: nil)) } } +struct TryMapTaskInner: Subscription, Subscriber, CustomStringConvertible, CustomPlaygroundDisplayConvertible { + + var description: String { "TryMapTask" } + + var playgroundDescription: Any { description } + + var downstream:S + var upstream:Subscription? = nil + var combineIdentifier: CombineIdentifier { downstream.combineIdentifier } + + func receive(_ input: S.Input) -> Subscribers.Demand { + downstream.receive(input) + } + + func receive(subscription: any Subscription) { + let newSubscription = Self(downstream: downstream, upstream: subscription) + downstream.receive(subscription: newSubscription) + } + + func receive(completion: Subscribers.Completion) { + downstream.receive(completion: completion) + } + + func request(_ demand: Subscribers.Demand) { + upstream?.request(demand) + } + + func cancel() { + upstream?.cancel() + } + +} + extension TryMapTask: Sendable where Upstream: Sendable {} extension TryMapTask { @@ -69,16 +112,16 @@ extension TryMapTask { var pending = Subscribers.Demand.none } - internal struct Inner: CustomCombineIdentifierConvertible where S.Failure == Failure, S.Input == Output { + internal struct Inner: Sendable, CustomCombineIdentifierConvertible where S.Failure == Failure, S.Input == Output { private let valueSource = AsyncThrowingStream.makeStream(bufferingPolicy: .bufferingNewest(2)) private let state: some UnfairStateLock> = createUncheckedStateLock(uncheckedState: .init()) - private let transform:@Sendable (Upstream.Output) async throws -> Output + private let transform: Transform let combineIdentifier = CombineIdentifier() init( subscriber:S, - transform: @escaping @Sendable (Upstream.Output) async throws -> Output + transform: @escaping Transform ) { self.transform = transform @@ -87,20 +130,26 @@ extension TryMapTask { private func send(completion: Subscribers.Completion?, cancel:Bool = false) { terminateStream() - let (subscriber, effect) = state.withLockUnchecked{ + let (subscriber, effect, taskEffect) = state.withLockUnchecked{ let old = $0.subscriber $0.subscriber = nil let effect = if cancel { + $0.upstreamSubscription.transition(.cancel) + } else { + $0.upstreamSubscription.transition(.finish) + } + let taskEffect = if cancel { $0.condition.transition(.cancel) } else { $0.condition.transition(.finish) } - return (old, effect) + return (old, effect, taskEffect) } effect?.run() if let completion { subscriber?.receive(completion: completion) } + taskEffect?.run() } private func send(_ value:Output) throws { @@ -119,6 +168,7 @@ extension TryMapTask { valueSource.continuation.finish() } + nonisolated private func waitForUpStream() async throws { try await withTaskCancellationHandler { try await withUnsafeThrowingContinuation { coninuation in @@ -139,6 +189,7 @@ extension TryMapTask { }?.run() } + nonisolated private func waitForCondition() async throws { try await withUnsafeThrowingContinuation{ continuation in state.withLock{ @@ -172,24 +223,22 @@ extension TryMapTask { guard subscription != nil else { return } - var iterator = valueSource.stream.makeAsyncIterator() - await withTaskCancellationHandler { - while true { - let upValue:Upstream.Output - do { - guard let value = try await iterator.next() else { - send(completion: .finished, cancel: false) - return - } - upValue = value - } catch { - send(completion: .failure(error), cancel: false) - return - } + await runInIsolation(isolation: transform.isolation) + } + + func runInIsolation( + isolation actor: isolated (any Actor)? = #isolation + ) async { + let block = { @Sendable in + let value = try await transform($0) + return Suppress(value: value) + } + do { + for try await upValue in valueSource.stream { do { // enqueue to separate task to prevent transformer cancelling root task using `UnsafeCurrentTask` - async let job = transform(upValue) - let value = try await job + async let job = block(upValue) + let value = try await job.value guard let _ = try? send(value) else { return } @@ -198,10 +247,10 @@ extension TryMapTask { return } } - } onCancel: { - send(completion: nil, cancel: true) + send(completion: .finished, cancel: false) + } catch { + send(completion: .failure(error), cancel: false) } - } } @@ -212,9 +261,7 @@ extension TryMapTask { extension TryMapTask.Inner: Subscription { func cancel() { - state.withLock{ - $0.condition.transition(.cancel) - }?.run() + send(completion: nil, cancel: true) } func request(_ demand: Subscribers.Demand) { @@ -256,9 +303,12 @@ extension TryMapTask.Inner: Subscriber { } func receive(completion: Subscribers.Completion) { + state.withLockUnchecked { + $0.upstreamSubscription.transition(.finish) + }?.run() switch completion { case .finished: - valueSource.continuation.finish() + valueSource.continuation.finish(throwing: nil) case .failure(let failure): valueSource.continuation.finish(throwing: failure) } diff --git a/Sources/Tetra/Combine/SchedulerTimePublisher.swift b/Sources/Tetra/Combine/SchedulerTimePublisher.swift index c89b475..9a9bbbb 100644 --- a/Sources/Tetra/Combine/SchedulerTimePublisher.swift +++ b/Sources/Tetra/Combine/SchedulerTimePublisher.swift @@ -7,6 +7,7 @@ import Foundation @preconcurrency import Combine +internal import CriticalSection public struct SchedulerTimePublisher: Publisher { @@ -45,7 +46,7 @@ public struct SchedulerTimePublisher: Publisher { final class Inner: Subscription, CustomStringConvertible, CustomPlaygroundDisplayConvertible where S.Input == Output, S.Failure == Failure { var description: String { - "SchedulerTimer<\(type(of: publisher.scheduler))>" + "SchedulerTimePublisher" } var playgroundDescription: Any { description } diff --git a/Sources/Tetra/Concurrency/AsyncSequencePublisher.swift b/Sources/Tetra/Concurrency/AsyncSequencePublisher.swift index 8d7f040..3bc39d0 100644 --- a/Sources/Tetra/Concurrency/AsyncSequencePublisher.swift +++ b/Sources/Tetra/Concurrency/AsyncSequencePublisher.swift @@ -7,8 +7,10 @@ import Foundation @preconcurrency import Combine +public import BackPortAsyncSequence +internal import CriticalSection -public extension AsyncSequence where Self:Sendable { +public extension AsyncSequence { @inlinable var tetra:TetraExtension { @@ -17,44 +19,57 @@ public extension AsyncSequence where Self:Sendable { } -public extension TetraExtension where Base: AsyncSequence & Sendable { +public extension TetraExtension where Base: AsyncSequence { -// @inlinable -// var publisher:some Publisher { -// AsyncSequencePublisher(base: base) -// } + @_disfavoredOverload + @inlinable + var publisher:some Publisher { + AsyncSequencePublisher(base: LegacyTypedAsyncSequence(base: base)) + } } +public extension TetraExtension where Base: AsyncSequence, Base.AsyncIterator: TypedAsyncIteratorProtocol { + + @inlinable + var publisher:some Publisher { + AsyncSequencePublisher(base: base) + } + +} -internal struct AsyncSequencePublisher: Publisher { +public struct AsyncSequencePublisher: Publisher where Base.AsyncIterator: TypedAsyncIteratorProtocol { public typealias Output = Base.AsyncIterator.Element - public typealias Failure = Base.AsyncIterator.Failure + public typealias Failure = Base.AsyncIterator.Err public var base:Base + public var barrier: (any Actor)? = nil + public var priority: TaskPriority? = nil - public init(base: Base) { + @inlinable + public init( + base: Base, + barrier: (any Actor)? = nil, + priority: TaskPriority? = nil + ) { self.base = base - } - - @available(macOS 15.0, iOS 18.0, watchOS 11.0, tvOS 18.0, visionOS 2.0, *) - public init(base:Source) where WrappedAsyncSequenceV2 == Base { - let source = WrappedAsyncSequenceV2(base: base) - self.base = source - } - - public init(base: Source) where Failure == any Error, WrappedAsyncSequence == Base { - let source = WrappedAsyncSequence(base: base) - self.base = source + self.barrier = barrier + self.priority = priority } public func receive(subscriber: S) where S : Subscriber, Failure == S.Failure, Base.AsyncIterator.Element == S.Input { let processor = Inner(subscriber: subscriber) - let task = Task { [base] in - await processor.run(base) + // transfer the AsyncIterator to the Task + // can not tell the compiler that this is safe, + // but this transfer is safe from data race + nonisolated(unsafe) + let unsafe = Suppress(value: base.makeAsyncIterator()) + let task = Task(priority: priority) { [capture = consume unsafe, barrier] in + var iter = capture.value + await processor.run(barrier, &iter) } processor.resumeCondition(task) } @@ -71,6 +86,9 @@ extension AsyncSequencePublisher { var subscriber:S? = nil var condition = TaskValueContinuation.waiting + var demand = Subscribers.Demand.none + var continuation:UnsafeContinuation? = nil + var terminated = false } @@ -80,7 +98,6 @@ extension AsyncSequencePublisher { var playgroundDescription: Any { description } let combineIdentifier = CombineIdentifier() - private let demandSource = AsyncStream.makeStream() private let state:some UnfairStateLock> = createUncheckedStateLock(uncheckedState: .init()) init(subscriber:S) { @@ -89,15 +106,25 @@ extension AsyncSequencePublisher { } } - func cancel() { - state.withLock{ - return $0.condition.transition(.cancel) - }?.run() + send(completion: nil) } func request(_ demand: Subscribers.Demand) { - demandSource.continuation.yield(demand) + let tuple:(UnsafeContinuation, Subscribers.Demand)? = state.withLock{ + $0.demand += demand + let old = $0.continuation + $0.continuation = nil + if let old, $0.demand > .none { + let newDemand = $0.demand + $0.demand = .none + return (old, newDemand) + } else { + return .none + } + } + guard let (token, newDemand) = tuple else { return } + token.resume(returning: newDemand) } private func send(_ value:S.Input) -> Subscribers.Demand? { @@ -105,14 +132,20 @@ extension AsyncSequencePublisher { } private func send(completion: Subscribers.Completion?) { - let subscriber = state.withLockUnchecked{ + let (subscriber, effect, token) = state.withLockUnchecked{ let old = $0.subscriber $0.subscriber = nil - return old + $0.terminated = true + let effect = $0.condition.transition(completion == nil ? .cancel : .finish) + let token = $0.continuation + $0.continuation = nil + return (old, effect, token) } if let completion { subscriber?.receive(completion: completion) } + token?.resume(returning: nil) + effect?.run() } func resumeCondition(_ task:Task) { @@ -129,53 +162,62 @@ extension AsyncSequencePublisher { } } - private func clearCondition() { - state.withLock{ - $0.condition.transition(.finish) - }?.run() + private func nextDemand() async -> Subscribers.Demand? { + await withUnsafeContinuation{ continuation in + let demand:Subscribers.Demand? = state.withLock{ + if $0.demand > .none { + let newDemand = $0.demand + $0.demand = .none + return newDemand + } else if $0.continuation == nil { + $0.continuation = continuation + return Subscribers.Demand.none + } else { + assertionFailure("received \(#function) twice at the same time") + return nil + } + + } + if let demand, demand > .none { + continuation.resume(returning: demand) + } + if demand == nil { + continuation.resume(returning: nil) + } + + } } + - func run(_ base: Base) async { + func run( + _ actor: isolated (any Actor)? = #isolation, + _ iterator: inout sending Base.AsyncIterator + ) async { let token:Void? = try? await waitForCondition() - defer { - demandSource.continuation.finish() - } if token == nil { - withUnsafeCurrentTask { $0?.cancel() } + send(completion: nil) } - defer { clearCondition() } state.withLockUnchecked{ $0.subscriber }?.receive(subscription: self) - var iterator = base.makeAsyncIterator() - await withTaskCancellationHandler { - for await var pending in demandSource.stream { + do { + while var pending = await nextDemand() { while pending > .none { pending -= 1 - let result = await wrapToResult(#isolation, &iterator) - guard let result else { + guard let value = try await iterator.next(isolation: actor) + else { send(completion: .finished) return } - switch result { - case .failure(let error): - send(completion: .failure(error)) + guard let newDemand = send(value) else { return - case .success(let value): - if let newDemand = send(value) { - pending += newDemand - } else { - return - } } + pending += newDemand } } - send(completion: .finished) - } onCancel: { - demandSource.continuation.finish() - send(completion: nil) + } catch { + send(completion: .failure(error)) } - } } diff --git a/Sources/Tetra/Concurrency/AsyncTypedSequence.swift b/Sources/Tetra/Concurrency/AsyncTypedSequence.swift deleted file mode 100644 index b73d0ac..0000000 --- a/Sources/Tetra/Concurrency/AsyncTypedSequence.swift +++ /dev/null @@ -1,75 +0,0 @@ -// -// AsyncTypedSequence.swift -// -// -// Created by pbk on 2022/09/26. -// - -import Combine -import _Concurrency -import Foundation - -/*** - Entry point of TypeFailure for `AsyncFlatMap` and `AsyncSequencePublisher - - Since generalized `AsyncSequence` is not available until Swift 6, this `protocol` is a entry point for async typed throw. - -Even though it has isolation parameter, this parameter is rarely used since `AsyncFlatMap` and `AsyncSequencePublisher` runs in nonisolated Task space. - -`Use `WrappedAsyncSequence` or `WrappedAsyncSequenceV2` if possible which provides general implementation to adapt this protocol - - - postcondition: `Failure` must be same as `AsyncIterator.Failure` - - - - adapting protocol to existing type, - - adapting protocols to exisiting type that user don't own is pretty easy too. By writing conformance like below, compiler now know the correct implemenation without recursive problem. -```` - extension AsyncStream.Iterator: TypedAsyncIteratorProtocol { - - @_implements(TypedAsyncIteratorProtocol, next(isolation:)) - mutating public func tetraNext(isolation actor: isolated (any Actor)?) async throws(Never) -> Self.Element? { - if #available(macOS 15.0, iOS 18.0, watchOS 11.0, tvOS 18.0, visionOS 2.0, *) { - var c: some AsyncIteratorProtocol = self - defer { - self = c as! Self - } - return await c.next(isolation: actor) - } else { - return try? await next() - } - } - - } - - ```` - */ -public protocol TypedAsyncIteratorProtocol { - - associatedtype Element - associatedtype Failure: Error - - - @inlinable - mutating func next(isolation actor: isolated (any Actor)?) async throws(Failure) -> Element? - -} - -public protocol TypedAsyncSequence: AsyncSequence where AsyncIterator: TypedAsyncIteratorProtocol {} - - -public protocol TypedAsyncIteratorProtocol2: AsyncIteratorProtocol, TypedAsyncIteratorProtocol { -} - - -public extension TypedAsyncIteratorProtocol { - - @inlinable - mutating func next() async throws(Failure) -> Element? { - try await next(isolation: nil) - } - -} - - - diff --git a/Sources/Tetra/Concurrency/CompatDiscardigTaskGroup.swift b/Sources/Tetra/Concurrency/CompatDiscardigTaskGroup.swift deleted file mode 100644 index d5c6fff..0000000 --- a/Sources/Tetra/Concurrency/CompatDiscardigTaskGroup.swift +++ /dev/null @@ -1,110 +0,0 @@ -// -// File.swift -// -// -// Created by 박병관 on 5/16/24. -// - -import Foundation - -internal protocol CompatThrowingDiscardingTaskGroup { - - var isCancelled:Bool { get } - var isEmpty:Bool { get } - func cancelAll() - mutating func addTaskUnlessCancelled( - priority: TaskPriority?, - operation: @escaping @Sendable () async throws -> Void - ) -> Bool - mutating func addTask( - priority: TaskPriority?, - operation: @escaping @Sendable () async throws -> Void - ) - - @available(macOS 15.0, iOS 18.0, watchOS 11.0, tvOS 18.0, visionOS 2.0, *) - mutating func addTask(executorPreference taskExecutor: (any TaskExecutor)?, priority: TaskPriority?, operation: @escaping @isolated(any) @Sendable () async throws -> Void) - - @available(macOS 15.0, iOS 18.0, watchOS 11.0, tvOS 18.0, visionOS 2.0, *) - mutating func addTaskUnlessCancelled(executorPreference taskExecutor: (any TaskExecutor)?, priority: TaskPriority?, operation: @escaping @isolated(any) @Sendable () async throws -> Void) -> Bool -} - -@available(macOS 14.0, iOS 17.0, watchOS 10.0, tvOS 17.0, macCatalyst 17.0, visionOS 1.0, *) -extension ThrowingDiscardingTaskGroup: CompatThrowingDiscardingTaskGroup { - -} - -extension ThrowingTaskGroup: CompatThrowingDiscardingTaskGroup where ChildTaskResult == Void, Failure == any Error { - -} - -/// Empty actor to isolate `ThrowingTaskGroup` to simulate DiscardingTaskGroup -@usableFromInline -actor SafetyRegion { - - private var isFinished = false - private var continuation: UnsafeContinuation? = nil - - @usableFromInline - init() { - - } - - @usableFromInline - func markDone() { - guard !isFinished else { return } - isFinished = true - continuation?.resume() - continuation = nil - } - - @usableFromInline - func hold() async { - await withUnsafeContinuation { - if isFinished { - $0.resume() - } else { - if let old = self.continuation { - assertionFailure("received suspend more than once!") - old.resume() - } - self.continuation = $0 - } - } - } - -} - - -extension ThrowingTaskGroup where ChildTaskResult == Void, Failure == any Error { - - /// work around for simulating Discarding TaskGroup - /// - /// TaskGroup is protected by the actor isolation - /// - important: always call TaskGroup api while holding isolation - @usableFromInline - internal mutating func simulateDiscarding( - isolation actor: isolated (SafetyRegion), - body: (isolated any Actor, inout Self) async throws -> Void - ) async throws { - addTask { - /// keep at least one child task alive - /// so that subTask won't return - await actor.hold() - } - /// drain all the finished or failed Task - async let subTask:Void = { - while let _ = try await next(isolation: actor) { - } - }() - // wrap with do block so that `defer` pops before waiting subTask - do { - /// release suspending Task - defer { actor.markDone() } - /// wrap the mutable TaskGroup with actor isolation - try await body(actor, &self) - } - try await subTask - } - -} - diff --git a/Sources/Tetra/Concurrency/CoreDataStack+Concurrency.swift b/Sources/Tetra/Concurrency/CoreDataStack+Concurrency.swift index 46e22de..879577d 100644 --- a/Sources/Tetra/Concurrency/CoreDataStack+Concurrency.swift +++ b/Sources/Tetra/Concurrency/CoreDataStack+Concurrency.swift @@ -20,15 +20,13 @@ extension TetraExtension where Base: NSPersistentStoreCoordinator { @usableFromInline internal func _perform(_ body: () throws(Failure) -> T) async throws(Failure) -> T { let value:Result = await withoutActuallyEscaping(body) { escapingClosure in - let holder = ClosureHolder(closure: escapingClosure) + let block = ClosureHolder(closure: escapingClosure) defer { - withExtendedLifetime(holder, {}) + withExtendedLifetime(block, {}) } return await withUnsafeContinuation { continuation in - base.perform { [unowned holder, continuation] in - nonisolated(unsafe) - let result = wrapToResult(holder.closure) - continuation.resume(returning: result) + base.perform { [unowned block, continuation] in + continuation.resume(returning: block()) } } } @@ -44,7 +42,7 @@ extension TetraExtension where Base: NSPersistentStoreCoordinator { withExtendedLifetime(block, {}) } return try await base.perform{ [unowned block] in - try block() + try block().get() } } } else { @@ -149,7 +147,7 @@ extension TetraExtension where Base: NSManagedObjectContext { withExtendedLifetime(block, {}) } return try await base.perform(schedule: schedule.platformValue) { [unowned block] in - return try block() + return try block().get() } } } else if schedule == .enqueued { @@ -195,7 +193,7 @@ extension TetraExtension where Base: NSPersistentContainer { let block = CoreDataContextClosureHolder(closure: $0) defer { withExtendedLifetime(block, {}) } return try await base.performBackgroundTask{ [unowned block] in - try block($0) + try block($0).get() } } } else { diff --git a/Sources/Tetra/Concurrency/DiscardingTaskState.swift b/Sources/Tetra/Concurrency/DiscardingTaskState.swift deleted file mode 100644 index 8c89e90..0000000 --- a/Sources/Tetra/Concurrency/DiscardingTaskState.swift +++ /dev/null @@ -1,92 +0,0 @@ -// -// DiscardingTaskState.swift -// -// -// Created by 박병관 on 6/17/24. -// - -enum DiscardingTaskState { - - case waiting - case suspend(UnsafeContinuation) - case cancel - case finish - - enum Effect { - case resume(UnsafeContinuation) - case raise(UnsafeContinuation) - - func run() { - switch self { - case .resume(let unsafeContinuation): - unsafeContinuation.resume() - case .raise(let unsafeContinuation): - unsafeContinuation.resume(throwing: CancellationError()) - } - } - } - - enum Event { - case cancel - case suspend(UnsafeContinuation) - case finish - } - - mutating func transition(_ event:Event) -> Effect? { - switch event { - case .cancel: - return cancel() - case .suspend(let unsafeContinuation): - return suspend(unsafeContinuation) - case .finish: - return finish() - } - } - - private mutating func suspend(_ cont:UnsafeContinuation) -> Effect? { - switch self { - case .waiting: - self = .suspend(cont) - return nil - case .suspend(let unsafeContinuation): - self = .suspend(cont) - assertionFailure("received suspend more than once") - return .raise(unsafeContinuation) - case .cancel: - return .raise(cont) - case .finish: - return .resume(cont) - } - } - - private mutating func finish() -> Effect? { - switch self { - case .waiting: - self = .finish - return nil - case .suspend(let unsafeContinuation): - self = .finish - return .resume(unsafeContinuation) - case .cancel: - return nil - case .finish: - return nil - } - } - - private mutating func cancel() -> Effect? { - switch self { - case .waiting: - self = .cancel - return nil - case .suspend(let unsafeContinuation): - self = .cancel - return .raise(unsafeContinuation) - case .cancel: - return nil - case .finish: - return nil - } - } - -} diff --git a/Sources/Tetra/Concurrency/Dispatch+Extension.swift b/Sources/Tetra/Concurrency/Dispatch+Extension.swift index 9eb3f34..e9bc907 100644 --- a/Sources/Tetra/Concurrency/Dispatch+Extension.swift +++ b/Sources/Tetra/Concurrency/Dispatch+Extension.swift @@ -7,6 +7,7 @@ @preconcurrency import Foundation import Dispatch +internal import CriticalSection extension Task: TetraExtended {} @@ -18,9 +19,9 @@ public extension TetraExtension where Base == Task { - Throws: `CancellationError` if task is cancelled */ @inlinable - static func sleep(deadline: DispatchTime, tolerance: DispatchTimeInterval? = nil) async throws { + static func sleep(deadline: DispatchTime, tolerance: DispatchTimeInterval? = nil) async throws(CancellationError) { let source = DispatchSource.makeTimerSource(flags: [.strict]) - + defer { source.cancel() } source.schedule(deadline: deadline, repeating: .never, leeway: tolerance ?? .nanoseconds(0)) try await dispatchTimerSleep(source: source) } @@ -31,8 +32,9 @@ public extension TetraExtension where Base == Task { - Throws: `CancellationError` if task is cancelled */ @inlinable - static func sleep(wallDeadline: DispatchWallTime, tolerance: DispatchTimeInterval? = nil) async throws { + static func sleep(wallDeadline: DispatchWallTime, tolerance: DispatchTimeInterval? = nil) async throws(CancellationError) { let source = DispatchSource.makeTimerSource(flags: [.strict]) + defer { source.cancel() } source.schedule(wallDeadline: wallDeadline, repeating: .never, leeway: tolerance ?? .nanoseconds(0)) try await dispatchTimerSleep(source: source) } @@ -41,7 +43,7 @@ public extension TetraExtension where Base == Task { @usableFromInline -internal func dispatchTimerSleep(source:DispatchSourceTimer) async throws { +internal func dispatchTimerSleep(source:DispatchSourceTimer) async throws(CancellationError) { let lock = createCheckedStateLock(checkedState: DispatchSleepState.waiting) source.setEventHandler{ @@ -54,37 +56,40 @@ internal func dispatchTimerSleep(source:DispatchSourceTimer) async throws { $0.take() }?.resume(throwing: CancellationError()) } - return try await withTaskCancellationHandler { - return try await withUnsafeThrowingContinuation{ continuation in - let snapShot = lock.withLock{ - let oldValue = $0 - switch oldValue { + do { + try await withTaskCancellationHandler { + return try await withUnsafeThrowingContinuation{ continuation in + let snapShot = lock.withLock{ + let oldValue = $0 + switch oldValue { + case .finished: + break + case .waiting, .continuation: + $0 = .continuation(continuation) + + } + return oldValue + } + switch snapShot { + case .continuation(let unsafeContinuation): + assertionFailure("reached unexpected state") + unsafeContinuation.resume(throwing: CancellationError()) case .finished: + continuation.resume(throwing: CancellationError()) + case .waiting: break - case .waiting, .continuation: - $0 = .continuation(continuation) - } - return oldValue - } - switch snapShot { - case .continuation(let unsafeContinuation): - assertionFailure("reached unexpected state") - unsafeContinuation.resume(throwing: CancellationError()) - case .finished: - continuation.resume(throwing: CancellationError()) - case .waiting: - break + source.activate() } - source.activate() + } onCancel: { + lock.withLock{ + $0.take() + }?.resume(throwing: CancellationError()) + source.cancel() } - } onCancel: { - lock.withLock{ - $0.take() - }?.resume(throwing: CancellationError()) - source.cancel() + } catch { + throw CancellationError() } - } diff --git a/Sources/Tetra/Concurrency/Notification+AsyncSequence.swift b/Sources/Tetra/Concurrency/Notification+AsyncSequence.swift index 2469cc3..a6cab53 100644 --- a/Sources/Tetra/Concurrency/Notification+AsyncSequence.swift +++ b/Sources/Tetra/Concurrency/Notification+AsyncSequence.swift @@ -8,9 +8,11 @@ import Foundation import _Concurrency +internal import BackPortAsyncSequence -extension NotificationCenter: TetraExtended {} +public import CriticalSection +extension NotificationCenter: TetraExtended {} extension TetraExtension where Base: NotificationCenter { @@ -22,8 +24,7 @@ extension TetraExtension where Base: NotificationCenter { } - -public final class NotificationSequence: AsyncSequence, Sendable { +public final class NotificationSequence: AsyncSequence, Sendable, TypedAsyncSequence { public typealias AsyncIterator = Iterator public typealias Failure = Never @@ -32,16 +33,20 @@ public final class NotificationSequence: AsyncSequence, Sendable { Iterator(parent: self) } + @usableFromInline let center: NotificationCenter - private let lock:some UnfairStateLock = createUncheckedStateLock(uncheckedState: NotficationState()) + @usableFromInline + let lock:some UnfairStateLock = createUncheckedStateLock(uncheckedState: NotficationState()) public struct Iterator: AsyncIteratorProtocol, TypedAsyncIteratorProtocol { public typealias Element = Notification public typealias Failure = Never + @usableFromInline let parent:NotificationSequence - public func next(isolation actor: isolated (any Actor)?) async throws(Never) -> Notification? { + @inlinable + public func next(isolation actor: isolated (any Actor)? = #isolation) async throws(Never) -> Notification? { // next를 호출한 동안에 task cancellation이 발생하면 observer Token이 무효화되는 것이 확인되므로 아래와 같이 canellation을 추가한다. await withTaskCancellationHandler( operation: { [parent] in @@ -50,16 +55,26 @@ public final class NotificationSequence: AsyncSequence, Sendable { onCancel: parent.cancel ) } + + @_disfavoredOverload + @inlinable + public func next() async throws(Never) -> Notification? { + await next(isolation: nil) + } } - private struct NotficationState { + @usableFromInline + internal struct NotficationState { + @usableFromInline var buffer:[Notification] = [] + @usableFromInline var pending:[UnsafeContinuation] = [] + @usableFromInline var observer:NSObjectProtocol? } - + @inlinable public init( center: NotificationCenter, named name: Notification.Name, @@ -86,11 +101,12 @@ public final class NotificationSequence: AsyncSequence, Sendable { } } - + @inlinable deinit { cancel() } + @usableFromInline @Sendable func cancel() { let snapShot = lock.withLockUnchecked { @@ -106,6 +122,7 @@ public final class NotificationSequence: AsyncSequence, Sendable { snapShot.pending.forEach{ $0.resume(returning: nil) } } + @usableFromInline func next(isolation: isolated (any Actor)?) async -> Notification? { await withUnsafeContinuation { continuation in let (notification, isCancelled) = lock.withLockUnchecked { state in diff --git a/Sources/Tetra/Concurrency/TaskValueContinuation.swift b/Sources/Tetra/Concurrency/TaskValueContinuation.swift index b74b878..c8622ae 100644 --- a/Sources/Tetra/Concurrency/TaskValueContinuation.swift +++ b/Sources/Tetra/Concurrency/TaskValueContinuation.swift @@ -53,6 +53,28 @@ enum TaskValueContinuation: Sendable { } } + func shouldMutate(_ event: Event) -> Bool { + switch self { + case .waiting: + return true + case .suspending(let unsafeContinuation): + if case .suspend(_) = event { + return false + } else { + return true + } + case .cached(let task): + if case .resume(_) = event { + return false + } else { + return true + } + case .cancelled, .finished: + return false + + } + } + private mutating func suspend(_ continuation:UnsafeContinuation) -> Effect? { switch self { case .waiting: diff --git a/Sources/Tetra/Concurrency/URLSessionDownloadTask+Concurrency.swift b/Sources/Tetra/Concurrency/URLSessionDownloadTask+Concurrency.swift index 006c7b7..4ca105e 100644 --- a/Sources/Tetra/Concurrency/URLSessionDownloadTask+Concurrency.swift +++ b/Sources/Tetra/Concurrency/URLSessionDownloadTask+Concurrency.swift @@ -8,6 +8,7 @@ import Foundation import Dispatch import _Concurrency +internal import CriticalSection @usableFromInline internal func randomDownloadFileURL() -> URL { diff --git a/Sources/Tetra/Concurrency/WrappedAsyncSequence.swift b/Sources/Tetra/Concurrency/WrappedAsyncSequence.swift deleted file mode 100644 index 466ce97..0000000 --- a/Sources/Tetra/Concurrency/WrappedAsyncSequence.swift +++ /dev/null @@ -1,98 +0,0 @@ -// -// WrappedAsyncSequence.swift -// -// -// Created by 박병관 on 6/11/24. -// - - -public struct WrappedAsyncSequence: AsyncSequence, TypedAsyncSequence { - - public typealias AsyncIterator = Iterator - - public typealias Element = Base.Element - public typealias Failure = any Error - - @usableFromInline - var base:Base - - @inlinable - public func makeAsyncIterator() -> AsyncIterator { - Iterator(base: base.makeAsyncIterator()) - } - - public struct Iterator: AsyncIteratorProtocol, TypedAsyncIteratorProtocol { - - public typealias Element = Base.Element - - public typealias Failure = any Error - - @usableFromInline - var base:Base.AsyncIterator - - @inlinable - public mutating func next(isolation actor: isolated (any Actor)?) async throws -> Base.Element? { - if #available(macOS 15.0, iOS 18.0, watchOS 11.0, tvOS 18.0, visionOS 2.0, *) { - return try await base.next(isolation: actor) - } else { - return try await base.advanceUnsafe() - } - } - - @usableFromInline - init(base: Base.AsyncIterator) { - self.base = base - } - - - } - - @inlinable - public init(base: Base) { - self.base = base - } - -} - -extension WrappedAsyncSequence: Sendable where Base:Sendable {} - -@available(macOS 15.0, iOS 18.0, watchOS 11.0, tvOS 18.0, visionOS 2.0, *) -public struct WrappedAsyncSequenceV2: AsyncSequence, TypedAsyncSequence { - - public typealias Element = Base.Element - public typealias Failure = Base.Failure - - @inlinable - public func makeAsyncIterator() -> Iterator { - return Iterator(base: base.makeAsyncIterator()) - } - - @usableFromInline - var base:Base - - public struct Iterator: AsyncIteratorProtocol, TypedAsyncIteratorProtocol { - - @usableFromInline - var base:Base.AsyncIterator - - @inlinable - public mutating func next(isolation actor: isolated (any Actor)?) async throws(Base.Failure) -> Base.Element? { - return try await base.next(isolation: actor) - } - - @usableFromInline - init(base: Base.AsyncIterator) { - self.base = base - } - - } - - @inlinable - public init(base: Base) { - self.base = base - } - -} - -@available(macOS 15.0, iOS 18.0, watchOS 11.0, tvOS 18.0, visionOS 2.0, *) -extension WrappedAsyncSequenceV2: Sendable where Base: Sendable {} diff --git a/Sources/Tetra/Foundation/ClosureHolder.swift b/Sources/Tetra/Foundation/ClosureHolder.swift index d708003..345cbd9 100644 --- a/Sources/Tetra/Foundation/ClosureHolder.swift +++ b/Sources/Tetra/Foundation/ClosureHolder.swift @@ -11,7 +11,7 @@ import CoreData #endif @usableFromInline -internal final class ClosureHolder { +internal final class ClosureHolder: @unchecked Sendable { @usableFromInline let closure: () throws(Failure) -> R @inlinable @@ -20,14 +20,21 @@ internal final class ClosureHolder { } @inlinable - func callAsFunction() throws(Failure) -> R { - return try closure() + func callAsFunction() -> Result { + do { + let value = try closure() + return .success(value) + } catch { + return .failure(error) + } } + } + #if canImport(CoreData) @usableFromInline -internal final class CoreDataContextClosureHolder { +internal final class CoreDataContextClosureHolder { @usableFromInline let closure: (NSManagedObjectContext) throws(Failure) -> R @inlinable @@ -36,8 +43,13 @@ internal final class CoreDataContextClosureHolder { } @inlinable - func callAsFunction(_ context:NSManagedObjectContext) throws(Failure) -> R { - return try closure(context) + func callAsFunction(_ context:NSManagedObjectContext) -> Result { + do { + let value = try closure(context) + return .success(value) + } catch { + return .failure(error) + } } } #endif diff --git a/Sources/Tetra/Foundation/EitherFailure.swift b/Sources/Tetra/Foundation/EitherFailure.swift new file mode 100644 index 0000000..a222b47 --- /dev/null +++ b/Sources/Tetra/Foundation/EitherFailure.swift @@ -0,0 +1,12 @@ +// +// EitherFailure.swift +// +// +// Created by 박병관 on 6/22/24. +// + +enum EitherFailure:Error { + + case first(First) + case second(Second) +} diff --git a/Sources/Tetra/Foundation/ManagedUnfairLock.swift b/Sources/Tetra/Foundation/ManagedUnfairLock.swift index b619e0b..8b13789 100644 --- a/Sources/Tetra/Foundation/ManagedUnfairLock.swift +++ b/Sources/Tetra/Foundation/ManagedUnfairLock.swift @@ -1,305 +1 @@ -// -// ManagedUnfairLock.swift -// -// -// Created by pbk on 2022/12/14. -// -import Foundation -import os - -@available(iOS, deprecated: 16.0, renamed: "OSAllocatedUnfairLock") -@available(tvOS, deprecated: 16.0, renamed: "OSAllocatedUnfairLock") -@available(macCatalyst, deprecated: 16.0, renamed: "OSAllocatedUnfairLock") -@available(watchOS, deprecated: 9.0, renamed: "OSAllocatedUnfairLock") -@available(macOS, deprecated: 13.0, renamed: "OSAllocatedUnfairLock") -public struct ManagedUnfairLock: @unchecked Sendable { - - private let __lock:ManagedBuffer - - /// Initialize an SwiftUnfairLock with a non-sendable lock-protected - /// `initialState`. - /// - /// By initializing with a non-sendable type, the owner of this structure - /// must ensure the Sendable contract is upheld manually. - /// Non-sendable content from `State` should not be allowed - /// to escape from the lock. - /// - /// - Parameter initialState: An initial value to store that will be - /// protected under the lock. - /// - public init(uncheckedState initialState: State) { - __lock = .create(minimumCapacity: 1) { buffer in - buffer.withUnsafeMutablePointerToElements{ $0.initialize(to: .init()) } - return initialState - } - } - - /// Perform a closure while holding this lock. - /// This method does not enforce sendability requirement - /// on closure body and its return type. - /// The caller of this method is responsible for ensuring references - /// to non-sendables from closure uphold the Sendability contract. - /// - /// - Parameter body: A closure to invoke while holding this lock. - /// - Returns: The return value of `body`. - /// - Throws: Anything thrown by `body`. - /// - public func withLockUnchecked(_ body: (inout State) throws -> R) rethrows -> R { - try __lock.withUnsafeMutablePointers{ state, lock in - os_unfair_lock_lock(lock) - defer { os_unfair_lock_unlock(lock) } - return try body(&state.pointee) - } - } - - /// Perform a sendable closure while holding this lock. - /// - /// - /// - Parameter body: A sendable closure to invoke while holding this lock. - /// - Returns: The sendable return value of `body`. - /// - Throws: Anything thrown by `body`. - /// - public func withLock(_ body: @Sendable (inout State) throws -> R) rethrows -> R where R : Sendable { - try withLockUnchecked(body) - } - - /// Attempt to acquire the lock, if successful, perform a closure while - /// holding the lock. - /// This method does not enforce sendability requirement - /// on closure body and its return type. - /// The caller of this method is responsible for ensuring references - /// to non-sendables from closure uphold the Sendability contract. - /// - /// - Parameter body: A closure to invoke while holding this lock. - /// - Returns: If the lock is acquired, the result of `body`. - /// If the lock is not acquired, nil. - /// - Throws: Anything thrown by `body`. - /// - public func withLockIfAvailableUnchecked(_ body: (inout State) throws -> R) rethrows -> R? { - try __lock.withUnsafeMutablePointers{ state, lock in - guard os_unfair_lock_trylock(lock) else { return nil } - defer { os_unfair_lock_unlock(lock) } - return try body(&state.pointee) - } - } - - /// Attempt to acquire the lock, if successful, perform a sendable closure while - /// holding the lock. - /// - /// - Parameter body: A closure to invoke while holding this lock. - /// - Returns: If the lock is acquired, the result of `body`. - /// If the lock is not acquired, nil. - /// - Throws: Anything thrown by `body`. - /// - public func withLockIfAvailable(_ body: @Sendable (inout State) throws -> R) rethrows -> R? where R : Sendable { - try withLockIfAvailableUnchecked(body) - } - - @frozen - public enum Ownership: Sendable, Hashable { - case owner - case notOwner - } - - /// Check a precondition about whether the calling thread is the lock owner. - /// - /// - Parameter condition: An `Ownership` statement to check for the - /// current context. - /// - If the lock is currently owned by the calling thread: - /// - `.owner` - returns - /// - `.notOwner` - asserts and terminates the process - /// - If the lock is unlocked or owned by a different thread: - /// - `.owner` - asserts and terminates the process - /// - `.notOwner` - returns - /// - public func precondition(_ condition: Ownership) { - __lock.withUnsafeMutablePointerToElements { - switch condition { - case .notOwner: - os_unfair_lock_assert_not_owner($0) - case .owner: - os_unfair_lock_assert_owner($0) - } - } - } - -} - - -public extension ManagedUnfairLock where State == Void { - - /// Initialize an SwiftUnfairLock with no protected state. - init() { - __lock = .create(minimumCapacity: 1) { buffer in - buffer.withUnsafeMutablePointerToElements{ $0.initialize(to: .init()) } - } - } - - /// Acquire this lock. - @_unavailableFromAsync(message: "Use async-safe scoped locking instead") - func lock() { - __lock.withUnsafeMutablePointerToElements { - os_unfair_lock_lock($0) - } - } - - /// Unlock this lock. - @_unavailableFromAsync(message: "Use async-safe scoped locking instead") - func unlock() { - __lock.withUnsafeMutablePointerToElements{ os_unfair_lock_unlock($0) } - } - - /// Perform a sendable closure while holding this lock. - /// - /// - Parameter body: A sendable closure to invoke while holding this lock. - /// - Returns: The return value of `body`. - /// - Throws: Anything thrown by `body`. - /// - func withLock(_ body: @Sendable () throws -> R) rethrows -> R where R : Sendable { - try withLockUnchecked(body) - } - - /// Perform a closure while holding this lock. - /// This method does not enforce sendability requirement - /// on closure body and its return type. - /// The caller of this method is responsible for ensuring references - /// to non-sendables from closure uphold the Sendability contract. - /// - /// - Parameter body: A closure to invoke while holding this lock. - /// - Returns: The return value of `body`. - /// - Throws: Anything thrown by `body`. - /// - func withLockUnchecked(_ body: () throws -> R) rethrows -> R { - try __lock.withUnsafeMutablePointerToElements { lock in - os_unfair_lock_lock(lock) - defer { os_unfair_lock_unlock(lock) } - return try body() - } - } - - /// Attempt to acquire the lock if it is not already locked. - /// - /// - Returns: `true` if the lock was succesfully locked, and - /// `false` if the lock attempt failed. - @available(*, noasync, message: "Use async-safe scoped locking instead") - func lockIfAvailable() -> Bool { - __lock.withUnsafeMutablePointerToElements { os_unfair_lock_trylock($0) } - } - - /// Attempt to acquire the lock, if successful, perform a sendable closure while - /// holding the lock. - /// - /// - Parameter body: A sendable closure to invoke while holding this lock. - /// - Returns: If the lock is acquired, the result of `body`. - /// If the lock is not acquired, nil. - /// - Throws: Anything thrown by `body`. - /// - func withLockIfAvailable(_ body: @Sendable () throws -> R) rethrows -> R? where R : Sendable { - try withLockIfAvailableUnchecked(body) - } - - /// Attempt to acquire the lock, if successful, perform a closure while - /// holding the lock. - /// This method does not enforce sendability requirement - /// on closure body and its return type. - /// The caller of this method is responsible for ensuring references - /// to non-sendables from closure uphold the Sendability contract. - /// - /// - Parameter body: A closure to invoke while holding this lock. - /// - Returns: If the lock is acquired, the result of `body`. - /// If the lock is not acquired, nil. - /// - Throws: Anything thrown by `body`. - /// - func withLockIfAvailableUnchecked(_ body: () throws -> R) rethrows -> R? { - try __lock.withUnsafeMutablePointerToElements{ lock in - guard os_unfair_lock_trylock(lock) else { return nil } - defer { os_unfair_lock_unlock(lock) } - return try body() - } - } - -} - -public extension ManagedUnfairLock { - - /// Initialize an SwiftUnfairLock with a lock-protected sendable - /// `initialState`. - /// - Parameter initialState: An initial value to store that will be - /// protected under the lock. - init(initialState: State) where State:Sendable { - __lock = .create(minimumCapacity: 1) { buffer in - buffer.withUnsafeMutablePointerToElements{ $0.initialize(to: .init()) } - return initialState - } - } - -} - -@usableFromInline -internal protocol UnfairStateLock: Sendable { - - associatedtype State - - func withLock(_ body: @Sendable (inout State) throws -> R) rethrows -> R where R : Sendable - - func withLockUnchecked(_ body: (inout State) throws -> R) rethrows -> R - - func withLockIfAvailableUnchecked(_ body: (inout State) throws -> R) rethrows -> R? - - init(uncheckedState initialState: State) - - func withLockIfAvailable(_ body: @Sendable (inout State) throws -> R) rethrows -> R? where R: Sendable - -} - -@usableFromInline -internal protocol UnfairLockProtocol: Sendable { - - init() - - @_unavailableFromAsync(message: "Use async-safe scoped locking instead") - func lock() - - @_unavailableFromAsync(message: "Use async-safe scoped locking instead") - func unlock() - - func withLockUnchecked(_ body: () throws -> R) rethrows -> R - - func withLock(_ body: @Sendable () throws -> R) rethrows -> R where R : Sendable - -} - -@available(iOS 16.0, tvOS 16.0, macOS 13.0, macCatalyst 16.0, watchOS 9.0, *) -extension OSAllocatedUnfairLock: UnfairStateLock {} -@available(iOS 16.0, tvOS 16.0, macOS 13.0, macCatalyst 16.0, watchOS 9.0, *) -extension OSAllocatedUnfairLock: UnfairLockProtocol {} -extension ManagedUnfairLock: UnfairStateLock {} -extension ManagedUnfairLock: UnfairLockProtocol {} - -@usableFromInline -internal func createUncheckedStateLock(uncheckedState initialState:State) -> some UnfairStateLock { - if #available(iOS 16.0, tvOS 16.0, macCatalyst 16.0, watchOS 9.0, macOS 13.0, *) { - return OSAllocatedUnfairLock(uncheckedState: initialState) - } else { - return ManagedUnfairLock(uncheckedState: initialState) - } -} - -@usableFromInline -internal func createCheckedStateLock(checkedState initialState:State) -> some UnfairStateLock { - if #available(iOS 16.0, tvOS 16.0, macCatalyst 16.0, watchOS 9.0, macOS 13.0, *) { - return OSAllocatedUnfairLock(initialState: initialState) - } else { - return ManagedUnfairLock(initialState: initialState) - } -} - -@usableFromInline -internal func createUnfairLock() -> some UnfairLockProtocol { - if #available(iOS 16.0, tvOS 16.0, macCatalyst 16.0, watchOS 9.0, macOS 13.0, *) { - return OSAllocatedUnfairLock() - } else { - return ManagedUnfairLock() - } -} diff --git a/Sources/Tetra/Foundation/Mics.swift b/Sources/Tetra/Foundation/Mics.swift index 4b918b0..3130c3d 100644 --- a/Sources/Tetra/Foundation/Mics.swift +++ b/Sources/Tetra/Foundation/Mics.swift @@ -80,7 +80,7 @@ internal extension NSNumber { @inline(__always) @usableFromInline internal -func wrapToResult(_ block: () throws(Failure) -> T) -> Result { +func wrapToResult(_ block: () throws(Failure) -> T) -> Result { do { return .success(try block()) } catch { @@ -88,38 +88,12 @@ func wrapToResult(_ block: () throws(Failure) -> T) -> Result( - _ actor: isolated (any Actor)?, - _ iterator: inout Base -) async -> sending Result? { - do { - if let value = try await iterator.next(isolation: actor) { - return .success(value) - } - return nil -// if #available(macOS 15.0, iOS 18.0, watchOS 11.0, tvOS 18.0, visionOS 2.0, *) { -// let value = try await iterator.next(isolation: actor) -// if let value { -// return .success(value) -// } else { -// return nil -// } -// } else { -// let result = try await iterator.next() -// return result -// } - } catch { - return .failure(error) - } -} - @inline(__always) @usableFromInline -internal func wrapToResult(_ value:T, _ transform: (T) async throws(Failure) -> U) async -> sending Result { +internal func wrapToResult( + _ value: consuming T, _ transform: (consuming T) async throws(Failure) -> sending U +) async -> sending Result { do { let success = try await transform(value) return .success(success) diff --git a/Sources/Tetra/SwiftUI/Binding+Collection.swift b/Sources/Tetra/SwiftUI/Binding+Collection.swift index 8c0955b..a6357a1 100644 --- a/Sources/Tetra/SwiftUI/Binding+Collection.swift +++ b/Sources/Tetra/SwiftUI/Binding+Collection.swift @@ -9,7 +9,7 @@ // import Foundation -@preconcurrency import SwiftUI +import SwiftUI @available(watchOS, deprecated: 8.0, message: "use Binding itself as Collection") @available(macOS, deprecated: 12.0, message: "use Binding itself as Collection") diff --git a/Tests/TetraTests/AsyncFlatMapTests.swift b/Tests/TetraTests/AsyncFlatMapTests.swift index 9e0c802..9f38a66 100644 --- a/Tests/TetraTests/AsyncFlatMapTests.swift +++ b/Tests/TetraTests/AsyncFlatMapTests.swift @@ -8,6 +8,7 @@ import XCTest import Combine @testable import Tetra +@testable import BackPortAsyncSequence final class AsyncFlatMapTests: XCTestCase { @@ -23,20 +24,22 @@ final class AsyncFlatMapTests: XCTestCase { } ) .asyncFlatMap(maxTasks: .max(1)) { value in - AsyncStream{ continuation in + let base = AsyncTypedStream(base: AsyncStream{ continuation in sample.forEach{ continuation.yield($0) } continuation.finish() - }.map{ + }) + return BackPort.AsyncMapSequence(base, transform: { await Task.yield() return $0 - } + }) }.handleEvents( receiveSubscription: { XCTAssertEqual("\($0)", "AsyncFlatMap") } - ).mapError{ $0.unwrap() } + ) + // ensure downstream do not request unlimited .buffer(size: 1, prefetch: .keepFull, whenFull: .customError{ fatalError() }) .prefix(10) @@ -62,26 +65,25 @@ final class AsyncFlatMapTests: XCTestCase { } ) .asyncFlatMap(maxTasks: .max(2)) { value in - return AsyncStream{ continuation in + let base = AsyncTypedStream(base: AsyncStream{ continuation in sample.forEach{ continuation.yield($0 + value * 10) } continuation.finish() - }.map{ + }) + return BackPort.AsyncMapSequence(base, transform: { await Task.yield() return $0 - } + }) }.handleEvents( receiveSubscription: { XCTAssertEqual("\($0)", "AsyncFlatMap") } - ).mapError{ $0.unwrap() } + ) .sink { _ in completion.fulfill() } receiveValue: { value in - lock.withLock{ - array.append(value) - } + array.append(value) } wait(for: [completion]) @@ -99,11 +101,11 @@ final class AsyncFlatMapTests: XCTestCase { lock.withLock{ holder.bag = [] } - return AsyncStream{ + return AsyncTypedStream(base: AsyncStream{ $0.yield(value) $0.finish() - } - }.mapError{ $0.unwrap() } + }) + } .handleEvents( receiveCancel: { completion.fulfill() @@ -123,17 +125,20 @@ final class AsyncFlatMapTests: XCTestCase { let lock = NSRecursiveLock() lock.withLock { (0..<5).publisher - .asyncFlatMap(maxTasks: .unlimited) { value in - return AsyncStream{ +// .setFailureType(to: Error.self) + .asyncFlatMap(maxTasks: .unlimited) { value throws(Never) in + let stream = AsyncStream{ $0.yield(value) $0.finish() - }.map{ + } + let source = AsyncTypedStream(base: stream) + return BackPort.AsyncMapSequence(source) { lock.withLock{ holder.bag = [] } return $0 } - }.mapError{ $0.unwrap() } + } .handleEvents( receiveCancel: { completion.fulfill() @@ -153,13 +158,13 @@ final class AsyncFlatMapTests: XCTestCase { let lock = NSRecursiveLock() lock.withLock { (0..<5).publisher +// .setFailureType(to: Error.self) .asyncFlatMap(maxTasks: .unlimited) { @Sendable value in - return AsyncStream{ @Sendable in + let stream = AsyncStream{ @Sendable in $0.yield(value) $0.finish() } - }.mapError{ - $0.unwrap() + return AsyncTypedStream(base: stream) }.handleEvents( receiveCancel: { @Sendable in completion.fulfill() @@ -167,43 +172,33 @@ final class AsyncFlatMapTests: XCTestCase { ).sink { _ in XCTFail("should not reach here") } receiveValue: { _ in - lock.withLock{ - holder.bag = [] - } + holder.bag = [] }.store(in: &holder.bag) } wait(for: [completion], timeout: 0.2) } - @available(macOS 9999, *) func testThrowInTransformer() throws { let holder = UnsafeCancellableHolder() let completion = expectation(description: "cancellation") (0..<5).publisher - .asyncFlatMap(maxTasks: .max(1)) { value in + .setFailureType(to: CancellationError.self) + .asyncFlatMap(maxTasks: .max(1)) { value throws(CancellationError) in if value == 3 { throw CancellationError() } - return AsyncStream{ + let base = AsyncTypedStream(base: AsyncStream{ $0.yield(value) $0.finish() - } - }.mapError{ - switch $0 { - case .transform(let error): - return error - case .segment(let error): - XCTFail("should not throw during segment") - return error - } + }) + return AsyncMapErrorSequence(base: base, failure: CancellationError.self) }.sink { switch $0 { case .finished: break case .failure(let error): completion.fulfill() - XCTAssertTrue(error is CancellationError) } } receiveValue: { XCTAssertLessThan($0, 3) @@ -215,22 +210,18 @@ final class AsyncFlatMapTests: XCTestCase { let holder = UnsafeCancellableHolder() let completion = expectation(description: "cancellation") (0..<5).publisher - .asyncFlatMap(maxTasks: .max(1)) { value in - return AsyncStream{ + .setFailureType(to: CancellationError.self) + .asyncFlatMap(maxTasks: .max(1)) { value throws(CancellationError) in + let base = AsyncTypedStream(base: AsyncStream{ $0.yield(value) $0.finish() - }.map{ - if $0 == 3 { + }) + return BackPort.AsyncMapSequence(base, CancellationError.self, transform: { value2 throws(CancellationError) in + if value2 == 3 { throw CancellationError() } - return $0 - } - } - .mapError{ - switch $0 { - case .segment(let error): - return error - } + return value2 + }) } .sink { switch $0 { @@ -261,8 +252,7 @@ final class AsyncFlatMapTests: XCTestCase { } } let transformTask = withUnsafeCurrentTask{ $0 }?.hashValue - - return AsyncStream{ + let stream = AsyncStream{ await Task.yield() withUnsafeCurrentTask { XCTAssertEqual(transformTask, $0?.hashValue) @@ -273,8 +263,7 @@ final class AsyncFlatMapTests: XCTestCase { withUnsafeCurrentTask{$0?.cancel()} return value } - }.mapError{ - $0.unwrap() + return AsyncTypedStream(base: stream) }.sink { _ in completion.fulfill() } receiveValue: { @@ -285,4 +274,5 @@ final class AsyncFlatMapTests: XCTestCase { } + } diff --git a/Tests/TetraTests/AsyncSequencePublisherTests.swift b/Tests/TetraTests/AsyncSequencePublisherTests.swift index 19f1890..fb47717 100644 --- a/Tests/TetraTests/AsyncSequencePublisherTests.swift +++ b/Tests/TetraTests/AsyncSequencePublisherTests.swift @@ -9,6 +9,7 @@ import Foundation import XCTest @testable import Tetra import Combine +import BackPortAsyncSequence class AsyncSequencePublisherTests: XCTestCase { @@ -22,7 +23,8 @@ class AsyncSequencePublisherTests: XCTestCase { source.forEach{ continuation.yield($0) } continuation.finish() } - let cancellable = AsyncSequencePublisher(base: stream) + let cancellable = AsyncTypedStream(base: stream) + .tetra.publisher .catch{ _ in XCTFail() return Empty() @@ -59,7 +61,7 @@ class AsyncSequencePublisherTests: XCTestCase { } return value } - let pub = AsyncSequencePublisher(base: asyncSequence) + let pub = AsyncSequencePublisher(base: LegacyTypedAsyncSequence(base: asyncSequence)) .handleEvents( receiveCancel: { expect.fulfill() } ) @@ -88,7 +90,7 @@ class AsyncSequencePublisherTests: XCTestCase { } return value } - let cancellable = AsyncSequencePublisher(base: asyncSequence) + let cancellable = AsyncSequencePublisher(base: LegacyTypedAsyncSequence(base: asyncSequence)) .mapError{ $0 as! CancellationError } diff --git a/Tests/TetraTests/MapTaskTests.swift b/Tests/TetraTests/MapTaskTests.swift index 5e0fbf6..60e48a6 100644 --- a/Tests/TetraTests/MapTaskTests.swift +++ b/Tests/TetraTests/MapTaskTests.swift @@ -8,7 +8,7 @@ import XCTest @testable import Tetra import Combine - +import Testing final class MapTaskTests: XCTestCase { @@ -182,3 +182,7 @@ final class MapTaskTests: XCTestCase { } } + +func asdfasdf() { + +} diff --git a/Tests/TetraTests/MultiMapTaskTests.swift b/Tests/TetraTests/MultiMapTaskTests.swift index 8443123..a510531 100644 --- a/Tests/TetraTests/MultiMapTaskTests.swift +++ b/Tests/TetraTests/MultiMapTaskTests.swift @@ -43,16 +43,19 @@ final class MultiMapTaskTests: XCTestCase { let pub = MultiMapTask(maxTasks: .max(1), upstream: input.publisher) { value in if value == target { await withUnsafeContinuation { - lock.withLock { +// lock.withLock { holder.bag.removeAll() - } +// } $0.resume() } XCTAssertTrue(Task.isCancelled) } return value }.handleEvents( - receiveCancel: { expect.fulfill() } + receiveCancel: { + print("CAll") + expect.fulfill() + } ) lock.withLock { pub.sink { _ in @@ -63,7 +66,7 @@ final class MultiMapTaskTests: XCTestCase { } - wait(for: [expect], timeout: 0.5) + wait(for: [expect]) } func testSerialFailure() throws { diff --git a/Tests/TetraTests/TetraTests.swift b/Tests/TetraTests/TetraTests.swift index 4296f08..2eb2a41 100644 --- a/Tests/TetraTests/TetraTests.swift +++ b/Tests/TetraTests/TetraTests.swift @@ -9,6 +9,7 @@ import XCTest import os @testable import Tetra import Combine +import CriticalSection final class TetraTests: XCTestCase { diff --git a/Tests/TetraTests/TryMapTaskTests.swift b/Tests/TetraTests/TryMapTaskTests.swift index 1ac5d33..5058aae 100644 --- a/Tests/TetraTests/TryMapTaskTests.swift +++ b/Tests/TetraTests/TryMapTaskTests.swift @@ -39,13 +39,10 @@ final class TryMapTaskTests: XCTestCase { let target = try XCTUnwrap(input.randomElement()) let ref = UnsafeCancellableHolder() let expect = expectation(description: "task cancellation") - let lock = NSRecursiveLock() let pub = TryMapTask(upstream: input.publisher) { value in if value == target { await withUnsafeContinuation{ continuation in - lock.withLock { - ref.bag.removeAll() - } + ref.bag.removeAll() continuation.resume() } XCTAssertTrue(Task.isCancelled) @@ -56,13 +53,11 @@ final class TryMapTaskTests: XCTestCase { expect.fulfill() } ) - lock.withLock { - pub.sink { _ in - XCTFail() - } receiveValue: { - XCTAssertLessThanOrEqual($0, target) - }.store(in: &ref.bag) - } + pub.sink { _ in + XCTFail() + } receiveValue: { + XCTAssertLessThanOrEqual($0, target) + }.store(in: &ref.bag) wait(for: [expect], timeout: 0.5) } From 124caf871e0c7aecb678518c30d49a1166e2fbe1 Mon Sep 17 00:00:00 2001 From: park-byeong-gwan Date: Thu, 27 Jun 2024 15:40:18 +0900 Subject: [PATCH 35/63] commit to test for library use case --- Package.swift | 22 ++- .../AsyncCompactMapSequence.swift | 1 + .../AsyncThrowingStream.swift | 7 - Sources/BackPortAsyncSequence/TaskGroup.swift | 71 +++++++++ .../ThrowingTaskGroup.swift | 72 +++++++++ Sources/BackPortAsyncSequence/operators.swift | 150 ++++++++++++++++-- Sources/Namespace/TetraExtension.swift | 35 ++++ .../NamespaceExtension/TetraExtended.swift | 37 +++++ Sources/NamespaceExtension/conformance.swift | 27 ++++ .../Tetra/Combine/Combine+Concurrency.swift | 36 ++--- .../Tetra/Combine/DispatchTimePublisher.swift | 2 +- .../Tetra/Combine/ExperimentalMapTask.swift | 8 +- .../Tetra/Combine/Future+Concurrency.swift | 71 ++++++--- Sources/Tetra/Combine/MapTaskInner.swift | 44 +++++ .../Combine/Publishers+AsyncFlatMap.swift | 63 ++++---- .../Tetra/Combine/Publishers+MapTask.swift | 24 ++- .../Tetra/Combine/Publishers+TryMapTask.swift | 44 +---- Sources/Tetra/Combine/RunLoopScheduler.swift | 16 +- .../Combine/SubscriptionContinuation.swift | 106 ------------- .../Concurrency/AsyncSequencePublisher.swift | 37 +++-- .../CoreDataStack+Concurrency.swift | 5 +- .../Concurrency/Dispatch+Extension.swift | 2 +- .../Notification+AsyncSequence.swift | 2 +- .../Concurrency/TaskValueContinuation.swift | 5 +- .../TetraExtension+URLSession.swift | 2 +- Sources/Tetra/Foundation/EitherFailure.swift | 12 -- .../Tetra/Foundation/ManagedUnfairLock.swift | 1 - Sources/Tetra/Foundation/Mics.swift | 39 ----- .../Tetra/SwiftUI/AsyncImage+BackPort.swift | 1 + .../Tetra/SwiftUI/Binding+Collection.swift | 6 +- Sources/Tetra/TetraExtension.swift | 58 +------ Tests/TetraTests/AsyncFlatMapTests.swift | 15 +- .../AsyncSequencePublisherTests.swift | 3 +- .../NSManagedObjectContextTests.swift | 1 + .../TetraTests/URLSessionDownloadTests.swift | 1 + .../TetraTests/Utlitity/JsonSampleModel.swift | 4 +- 36 files changed, 630 insertions(+), 400 deletions(-) create mode 100644 Sources/BackPortAsyncSequence/TaskGroup.swift create mode 100644 Sources/BackPortAsyncSequence/ThrowingTaskGroup.swift create mode 100644 Sources/Namespace/TetraExtension.swift create mode 100644 Sources/NamespaceExtension/TetraExtended.swift create mode 100644 Sources/NamespaceExtension/conformance.swift create mode 100644 Sources/Tetra/Combine/MapTaskInner.swift delete mode 100644 Sources/Tetra/Combine/SubscriptionContinuation.swift delete mode 100644 Sources/Tetra/Foundation/EitherFailure.swift delete mode 100644 Sources/Tetra/Foundation/ManagedUnfairLock.swift diff --git a/Package.swift b/Package.swift index 83b8684..e25b66d 100644 --- a/Package.swift +++ b/Package.swift @@ -33,6 +33,21 @@ let package = Package( targets: [ // Targets are the basic building blocks of a package. A target can define a module or a test suite. // Targets can depend on other targets in this package, and on products in packages this package depends on. + .target( + name: "Namespace", + swiftSettings: [ + .swiftLanguageVersion(.v6) + ] + ), + .target( + name: "NamespaceExtension", + dependencies: [ + "Namespace", + ], + swiftSettings: [ + .swiftLanguageVersion(.v6) + ] + ), .target( name: "CriticalSection", dependencies: [ @@ -47,6 +62,9 @@ let package = Package( ), .target( name: "BackportDiscardingTaskGroup", + dependencies: [ + "Namespace", + ], swiftSettings: [ .enableUpcomingFeature("FullTypedThrows"), .enableExperimentalFeature("IsolatedAny"), @@ -60,6 +78,8 @@ let package = Package( "BackPortAsyncSequence", "CriticalSection", "BackportDiscardingTaskGroup", + "Namespace", + "NamespaceExtension" ], swiftSettings: [ .enableUpcomingFeature("FullTypedThrows"), @@ -69,7 +89,7 @@ let package = Package( ), .target( name: "BackPortAsyncSequence", - dependencies: [], + dependencies: [ "Namespace"], swiftSettings: [ .swiftLanguageVersion(.v6), ] diff --git a/Sources/BackPortAsyncSequence/AsyncCompactMapSequence.swift b/Sources/BackPortAsyncSequence/AsyncCompactMapSequence.swift index 7d7173e..7df0cee 100644 --- a/Sources/BackPortAsyncSequence/AsyncCompactMapSequence.swift +++ b/Sources/BackPortAsyncSequence/AsyncCompactMapSequence.swift @@ -5,6 +5,7 @@ // Created by 박병관 on 6/13/24. // + extension BackPort { diff --git a/Sources/BackPortAsyncSequence/AsyncThrowingStream.swift b/Sources/BackPortAsyncSequence/AsyncThrowingStream.swift index ee2ffca..db5df2d 100644 --- a/Sources/BackPortAsyncSequence/AsyncThrowingStream.swift +++ b/Sources/BackPortAsyncSequence/AsyncThrowingStream.swift @@ -74,11 +74,4 @@ extension AsyncTypedThrowingStream.Iterator: AsyncIteratorProtocol, TypedAsyncIt extension AsyncTypedThrowingStream: Sendable where Element: Sendable {} -extension AsyncThrowingStream { - - func bridge() -> some TypedAsyncSequence { - AsyncTypedThrowingStream(base: self) - } - -} diff --git a/Sources/BackPortAsyncSequence/TaskGroup.swift b/Sources/BackPortAsyncSequence/TaskGroup.swift new file mode 100644 index 0000000..49a44d8 --- /dev/null +++ b/Sources/BackPortAsyncSequence/TaskGroup.swift @@ -0,0 +1,71 @@ +// +// TaskGroup.swift +// +// +// Created by 박병관 on 6/28/24. +// +import Namespace + + +public struct TypedTaskGroup { + + @usableFromInline + let base:TaskGroup + + @inlinable + public init(base: TaskGroup) { + self.base = base + } + +} + +extension TypedTaskGroup: AsyncSequence, TypedAsyncSequence { + + public typealias Failure = Never + + @inlinable + public func makeAsyncIterator() -> Iterator { + Iterator(parent: base) + } + + public struct Iterator { + + @usableFromInline + var parent:TaskGroup + + @inlinable + internal init( + parent: TaskGroup + ) { + self.parent = parent + } + + } + +} + +extension TypedTaskGroup.Iterator: AsyncIteratorProtocol, TypedAsyncIteratorProtocol { + + public typealias Failure = Never + + @inlinable + public mutating func next(isolation actor: isolated (any Actor)? = #isolation) async -> Element? { + return await parent.next(isolation: actor) + } + + @_disfavoredOverload + @inlinable + public mutating func next() async -> Element? { + await next(isolation: nil) + } + +} + +extension TetraExtension{ + + @inlinable + public func bridge() -> TypedTaskGroup where Base == TaskGroup { + return .init(base: base) + } + +} diff --git a/Sources/BackPortAsyncSequence/ThrowingTaskGroup.swift b/Sources/BackPortAsyncSequence/ThrowingTaskGroup.swift new file mode 100644 index 0000000..1b7c394 --- /dev/null +++ b/Sources/BackPortAsyncSequence/ThrowingTaskGroup.swift @@ -0,0 +1,72 @@ +// +// ThrowingTaskGroup.swift +// +// +// Created by 박병관 on 6/28/24. +// +import Namespace + +public struct TypedThrowingTaskGroup { + + @usableFromInline + let base:ThrowingTaskGroup + + @inlinable + public init(base: ThrowingTaskGroup) { + self.base = base + } + +} + +extension TypedThrowingTaskGroup: AsyncSequence, TypedAsyncSequence { + + public typealias Failure = Failure + + + @inlinable + public func makeAsyncIterator() -> Iterator { + Iterator(parent: base) + } + + public struct Iterator { + + @usableFromInline + var parent:ThrowingTaskGroup + + @inlinable + internal init( + parent: ThrowingTaskGroup + ) { + self.parent = parent + } + + } + +} + +extension TypedThrowingTaskGroup.Iterator: AsyncIteratorProtocol, TypedAsyncIteratorProtocol { + + public typealias Failure = Failure + + @inlinable + public mutating func next(isolation actor: isolated (any Actor)? = #isolation) async throws(Failure) -> Element? { + return try await parent.nextResult()?.get() + } + + @_disfavoredOverload + @inlinable + public mutating func next() async throws(Failure) -> Element? { + try await next(isolation: nil) + } + +} + + +extension TetraExtension{ + + @inlinable + public func bridge() -> TypedThrowingTaskGroup where Base == ThrowingTaskGroup { + return .init(base: base) + } + +} diff --git a/Sources/BackPortAsyncSequence/operators.swift b/Sources/BackPortAsyncSequence/operators.swift index d1365fc..887a082 100644 --- a/Sources/BackPortAsyncSequence/operators.swift +++ b/Sources/BackPortAsyncSequence/operators.swift @@ -4,29 +4,153 @@ // // Created by 박병관 on 6/13/24. // +import Namespace + +public extension AsyncSequence { + + @inlinable + var tetra:TetraExtension { + .init(self) + } + +} -extension AsyncSequence where AsyncIterator: TypedAsyncIteratorProtocol { +public extension TetraExtension where Base:AsyncSequence, Base.AsyncIterator: TypedAsyncIteratorProtocol, Base.AsyncIterator.Err == Never { - func mapError( - _ mapError: @escaping @Sendable (AsyncIterator.Err) async throws(Err) -> Void - ) -> some TypedAsyncSequence { - AsyncMapErrorSequence(base: self, mapError: mapError) + @inlinable + func mapError( + _ failureType: Failure.Type = Failure.self + ) -> some TypedAsyncSequence { + AsyncMapErrorSequence(base: base, failure: failureType) } + @inlinable func filter( + _ isIncluded: @escaping @Sendable (Element) async throws(Failure) -> Bool + ) -> some TypedAsyncSequence + where Element == Base.Element { + mapError(Failure.self).tetra.filter(isIncluded) + } + @inlinable @_disfavoredOverload - func map2( - _ map: @escaping @Sendable (AsyncIterator.Element) async throws(Err) -> T - ) -> some TypedAsyncSequence where AsyncIterator.Err == Never { - return BackPort.AsyncMapSequence(self, transform: map) + func map( + _ map: @escaping @Sendable (Base.Element) async throws(Failure) -> T + ) -> some TypedAsyncSequence { + mapError().tetra.map(map) + } + + @inlinable + func flatMap2( + _ map: @escaping @Sendable (Base.AsyncIterator.Element) async throws(Failure) -> SegmentOfResults + ) -> some TypedAsyncSequence where + SegmentOfResults.AsyncIterator.Err == Failure, + SegmentOfResults.AsyncIterator: TypedAsyncIteratorProtocol, + SegmentOfResults.Element == T { + mapError().tetra.flatMap(map) + } +// + @inlinable + func flatMap0( + _ map: @escaping @Sendable (Base.AsyncIterator.Element) async throws(Failure) -> SegmentOfResults + ) -> some TypedAsyncSequence where + SegmentOfResults.AsyncIterator.Err == Never, + SegmentOfResults.AsyncIterator: TypedAsyncIteratorProtocol, + SegmentOfResults.Element == T { + mapError().tetra.flatMap{ value throws(Failure) in + let segment = try await map(value) + return segment.tetra.mapError(Failure.self) + } + } +// + +} + +public extension TetraExtension where Base:AsyncSequence, Base.AsyncIterator: TypedAsyncIteratorProtocol { + + + @inlinable + func mapError( + _ mapError: @escaping @Sendable (Base.AsyncIterator.Err) async throws(Failure) -> Void + ) -> some TypedAsyncSequence { + AsyncMapErrorSequence(base: base, mapError: mapError) } + - func map2( - _ map: @escaping @Sendable (AsyncIterator.Element) async throws(AsyncIterator.Err) -> T - ) -> some TypedAsyncSequence { - BackPort.AsyncMapSequence(self, transform: map) +// @preconcurrency @inlinable public func map(_ transform: @escaping @Sendable (Self.Element) async -> Transformed) -> AsyncMapSequence + @inlinable func filter( + _ isIncluded: @escaping @Sendable (Element) async throws(Failure) -> Bool + ) -> some TypedAsyncSequence + where Element == Base.Element, Failure == Base.AsyncIterator.Err { + BackPort.AsyncFilterSequence(base, isIncluded: isIncluded) + } + + @inlinable + func map( + _ map: @escaping @Sendable (Base.AsyncIterator.Element) async throws(Failure) -> T + ) -> some TypedAsyncSequence where Base.AsyncIterator.Err == Failure { + BackPort.AsyncMapSequence(base, transform: map) + } + + @inlinable + func flatMap( + _ map: @escaping @Sendable (Base.AsyncIterator.Element) async throws(Failure) -> SegmentOfResults + ) -> some TypedAsyncSequence where + Base.AsyncIterator.Err == Failure, + SegmentOfResults.AsyncIterator.Err == Failure, + SegmentOfResults.AsyncIterator: TypedAsyncIteratorProtocol, + SegmentOfResults.Element == T { + BackPort.AsyncFlatMapSequence(base, transform: map) } + @inlinable + func drop( + while predicate: @escaping @Sendable (Base.Element) async throws(Failure) -> Bool + ) -> some TypedAsyncSequence where Failure == Base.AsyncIterator.Err { + BackPort.AsyncDropWhileSequence(base, predicate: predicate) + } + + @inlinable + func prefix(while predicate: @escaping @Sendable (Base.Element) async throws(Failure) -> Bool) -> some TypedAsyncSequence where Base.AsyncIterator.Err == Failure { + BackPort.AsyncPrefixWhileSequence(base, predicate: predicate) + } + + @inlinable + func compactMap(_ transform: @escaping @Sendable (Base.Element) async throws(Failure) -> ElementOfResult?) -> some TypedAsyncSequence where Base.AsyncIterator.Err == Failure { + BackPort.AsyncCompactMapSequence(base, transform: transform) + } + + @inlinable func dropFirst(_ count: Int = 1) -> some TypedAsyncSequence { + BackPort.AsyncDropFirstSequence(base, dropping: count) + } + + @inlinable func prefix(_ count: Int = 1) -> some TypedAsyncSequence { + BackPort.AsyncPrefixSequence(base, count: count) + } + } + + +public extension TetraExtension { + + @inlinable func dropFirst(_ count: Int = 1) -> some TypedAsyncSequence where Base == BackPort.AsyncDropFirstSequence { + BackPort.AsyncDropFirstSequence(base, dropping: count + base.count) + } + + @inlinable func prefix(_ count: Int = 1) -> some TypedAsyncSequence where Base == BackPort.AsyncPrefixSequence { + BackPort.AsyncPrefixSequence(base, count: base.count + count) + } + + @inlinable + func bridge() -> AsyncTypedStream where Base == AsyncStream { + AsyncTypedStream(base: base) + } + + @inlinable + func bridge() -> AsyncTypedThrowingStream where Base == AsyncThrowingStream { + AsyncTypedThrowingStream(base: base) + } + +} + diff --git a/Sources/Namespace/TetraExtension.swift b/Sources/Namespace/TetraExtension.swift new file mode 100644 index 0000000..689cbd2 --- /dev/null +++ b/Sources/Namespace/TetraExtension.swift @@ -0,0 +1,35 @@ +// +// TetraExtension.swift +// +// +// Created by 박병관 on 1/1/24. +// + +import Foundation + +public struct TetraExtension { + + @usableFromInline + package var value:Base + + @inlinable + public var base:Base { + get { value } + } + + @inlinable + public init(base: Base) { + self.value = base + } + + @inlinable + public init(_ base: Base) { + self.value = base + } + + +} + + +extension TetraExtension: Sendable where Base:Sendable {} + diff --git a/Sources/NamespaceExtension/TetraExtended.swift b/Sources/NamespaceExtension/TetraExtended.swift new file mode 100644 index 0000000..4e2ab52 --- /dev/null +++ b/Sources/NamespaceExtension/TetraExtended.swift @@ -0,0 +1,37 @@ +// +// TetraExtended.swift +// +// +// Created by 박병관 on 6/27/24. +// +public import Namespace + +public protocol TetraExtended { + /// Type being extended. + associatedtype Base + + /// Static Tetra extension point. + @inlinable + static var tetra: TetraExtension.Type { get set } + /// Instance Tetra extension point. + @inlinable + var tetra: TetraExtension { get set } +} + +extension TetraExtended where Base == Self { + + /// Static Tetra extension point. + @inlinable + public static var tetra: TetraExtension.Type { + get { TetraExtension.self } + set {} + } + + /// Instance Tetra extension point. + @inlinable + public var tetra: TetraExtension { + get { TetraExtension(self) } + set {} + } + +} diff --git a/Sources/NamespaceExtension/conformance.swift b/Sources/NamespaceExtension/conformance.swift new file mode 100644 index 0000000..dfa42d1 --- /dev/null +++ b/Sources/NamespaceExtension/conformance.swift @@ -0,0 +1,27 @@ +// +// conformance.swift +// +// +// Created by 박병관 on 6/27/24. +// +import CoreData +import Foundation +import Combine +public import Namespace + +extension NSPersistentContainer: TetraExtended {} +extension NSPersistentStoreCoordinator: TetraExtended {} +extension NSManagedObjectContext: TetraExtended {} +extension NotificationCenter: TetraExtended {} +extension DispatchSource: TetraExtended {} +extension URLSession: TetraExtended {} +extension Task: TetraExtended {} + +public extension Publisher { + + @inlinable + var tetra:TetraExtension { + .init(self) + } + +} diff --git a/Sources/Tetra/Combine/Combine+Concurrency.swift b/Sources/Tetra/Combine/Combine+Concurrency.swift index 722d848..8042148 100644 --- a/Sources/Tetra/Combine/Combine+Concurrency.swift +++ b/Sources/Tetra/Combine/Combine+Concurrency.swift @@ -8,14 +8,9 @@ import Foundation import Combine internal import BackPortAsyncSequence +import Namespace + -public extension Publisher { - @inlinable - var tetra:TetraExtension { - .init(self) - } - -} public extension TetraExtension where Base: Publisher { @@ -30,25 +25,28 @@ public extension Publisher { @inlinable func mapTask( + priority: TaskPriority? = nil, transform: @escaping @isolated(any) @Sendable (Output) async -> sending T ) -> some Publisher where Output:Sendable { - MapTask(upstream: self, transform: transform) + MapTask(priority: priority, upstream: self, transform: transform) } @inlinable func tryMapTask( + priority: TaskPriority? = nil, transform: @escaping @isolated(any) @Sendable (Output) async throws -> sending T ) -> some Publisher where Output:Sendable { - TryMapTask(upstream: self, transform: transform) + TryMapTask(priority: priority, upstream: self, transform: transform) } @_spi(Experimental) @inlinable func multiMapTask( + priority: TaskPriority? = nil, maxTasks: Subscribers.Demand = .max(1), transform: @escaping @Sendable @isolated(any) (Output) async throws(Failure) -> sending T ) -> some Publisher where Output: Sendable { - MultiMapTask(maxTasks: maxTasks, upstream: self, transform: transform) + MultiMapTask(priority: priority, maxTasks: maxTasks, upstream: self, transform: transform) } @@ -57,19 +55,19 @@ public extension Publisher { internal extension Publisher { -// -// func asyncFlatMap( -// maxTasks: Subscribers.Demand = .unlimited, -// transform: @escaping @Sendable @isolated(any) (Output) async throws(any Error) -> sending Segment -// ) -> AsyncFlatMap> where Output:Sendable { -// return AsyncFlatMap(maxTasks: maxTasks, upstream: self, transform: transform) -// } - + func asyncFlatMap( maxTasks: Subscribers.Demand = .unlimited, + priority: TaskPriority? = nil, transform: @escaping @Sendable @isolated(any) (Output) async throws(Failure) -> sending Segment ) -> AsyncFlatMap where Output:Sendable, Segment.Err == Failure { - return AsyncFlatMap(maxTasks: maxTasks, upstream: self, transform: transform) + return AsyncFlatMap( + priority: priority, + maxTasks: maxTasks, + upstream: self, + transform: transform + ) } } + diff --git a/Sources/Tetra/Combine/DispatchTimePublisher.swift b/Sources/Tetra/Combine/DispatchTimePublisher.swift index 2675abc..a5c0fed 100644 --- a/Sources/Tetra/Combine/DispatchTimePublisher.swift +++ b/Sources/Tetra/Combine/DispatchTimePublisher.swift @@ -9,8 +9,8 @@ import Foundation import Dispatch import Combine internal import CriticalSection +import Namespace -extension DispatchSource: TetraExtended {} public extension TetraExtension where Base:DispatchSource { diff --git a/Sources/Tetra/Combine/ExperimentalMapTask.swift b/Sources/Tetra/Combine/ExperimentalMapTask.swift index 9a67599..11a0bf2 100644 --- a/Sources/Tetra/Combine/ExperimentalMapTask.swift +++ b/Sources/Tetra/Combine/ExperimentalMapTask.swift @@ -132,7 +132,7 @@ extension MultiMapTask { break case .success(let success): let flag = group.addTaskUnlessCancelled(priority: nil) { - let result = await wrapToResult(success, transform) + let result = await wrapToResult(consume success, transform) switch result { case .failure(let error): await barrier.markDone() @@ -175,10 +175,8 @@ extension MultiMapTask { } return (old, taskEffect) } - if let subscriber { - subscriber.receive(completion: completion) - } - taskEffect?.run() + (consume subscriber)?.receive(completion: completion) + (consume taskEffect)?.run() } diff --git a/Sources/Tetra/Combine/Future+Concurrency.swift b/Sources/Tetra/Combine/Future+Concurrency.swift index 261d6ce..c162572 100644 --- a/Sources/Tetra/Combine/Future+Concurrency.swift +++ b/Sources/Tetra/Combine/Future+Concurrency.swift @@ -7,36 +7,59 @@ import Foundation import Combine +import Namespace - -extension TetraExtension { - +extension TetraExtension where Base: _CombineFuterProtocol { + @inlinable - public func next() async throws(Failure) -> Output where Base == Combine.Future { - let result: Result = await withCheckedContinuation { continuation in - base.subscribe(AnySubscriber( - receiveSubscription: { - $0.request(.max(1)) - }, - receiveValue: { (value: sending Output) in - continuation.resume(returning: .success(value)) - return .none - }, - receiveCompletion: { - if case let .failure(error) = $0 { - continuation.resume(returning: .failure(error)) + public var value:Base.Output { + get async throws(Base.Failure) { + let future = base._tetraFuture + // subscriber inferface is guaranteed to be called serially, so we can use variable without lock happily :) + var subscription: (any Subscription)? = nil + defer { + withExtendedLifetime(subscription, { }) + } + let result: Result = await withCheckedContinuation { continuation in + future.subscribe(AnySubscriber( + receiveSubscription: { + $0.request(.max(1)) + subscription = $0 + }, + receiveValue: { (value: sending Base.Output) in + continuation.resume(returning: .success(value)) + return .none + }, + receiveCompletion: { + if case let .failure(error) = $0 { + continuation.resume(returning: .failure(error)) + } + subscription = nil } - } - )) - } - switch result { - case .success(let success): - return success - case .failure(let failure): - throw failure + )) + } + switch result { + case .success(let success): + return success + case .failure(let failure): + throw failure + } + } } +} + +public protocol _CombineFuterProtocol: Publisher { + + @inlinable + var _tetraFuture: Combine.Future { get } + +} + +extension Combine.Future: _CombineFuterProtocol { + @inlinable + public var _tetraFuture: Future { self } } diff --git a/Sources/Tetra/Combine/MapTaskInner.swift b/Sources/Tetra/Combine/MapTaskInner.swift new file mode 100644 index 0000000..05a3887 --- /dev/null +++ b/Sources/Tetra/Combine/MapTaskInner.swift @@ -0,0 +1,44 @@ +// +// MapTaskInner.swift +// +// +// Created by 박병관 on 6/28/24. +// +import Combine + +struct MapTaskInner: Subscription, Subscriber, CustomStringConvertible, CustomPlaygroundDisplayConvertible { + + var description: String + + var playgroundDescription: Any { description } + + var downstream:S + var upstream:Subscription? = nil + var combineIdentifier: CombineIdentifier { downstream.combineIdentifier } + + func receive(_ input: S.Input) -> Subscribers.Demand { + downstream.receive(input) + } + + func receive(subscription: any Subscription) { + let newSubscription = Self( + description: description, + downstream: downstream, + upstream: subscription + ) + downstream.receive(subscription: newSubscription) + } + + func receive(completion: Subscribers.Completion) { + downstream.receive(completion: completion) + } + + func request(_ demand: Subscribers.Demand) { + upstream?.request(demand) + } + + func cancel() { + upstream?.cancel() + } + +} diff --git a/Sources/Tetra/Combine/Publishers+AsyncFlatMap.swift b/Sources/Tetra/Combine/Publishers+AsyncFlatMap.swift index c07779d..bcba29b 100644 --- a/Sources/Tetra/Combine/Publishers+AsyncFlatMap.swift +++ b/Sources/Tetra/Combine/Publishers+AsyncFlatMap.swift @@ -17,13 +17,18 @@ struct AsyncFlatMap: Publisher where typealias Failure = Upstream.Failure typealias Transform = @Sendable (Upstream.Output) async throws(Failure) -> sending Segment var priority: TaskPriority? = nil + let taskExecutor: (any Executor)? var maxTasks:Subscribers.Demand let upstream:Upstream let transform:Transform func receive(subscriber: S) where S : Subscriber, Failure == S.Failure, Segment.Element == S.Input { let processor = Inner(maxTasks: maxTasks, subscriber: subscriber, transform: transform) - let task = Task(priority: priority, operation: processor.run) + let task = if #available(macOS 15.0, iOS 18.0, watchOS 11.0, tvOS 18.0, visionOS 2.0, *), let executor = taskExecutor as? (any TaskExecutor) { + Task(executorPreference: executor, priority: priority, operation: processor.run) + } else { + Task(priority: priority, operation: processor.run) + } processor.resumeCondition(task) upstream.subscribe(processor) } @@ -39,6 +44,7 @@ struct AsyncFlatMap: Publisher where self.maxTasks = maxTasks self.upstream = upstream self.transform = transform + self.taskExecutor = nil } @usableFromInline @@ -52,14 +58,16 @@ struct AsyncFlatMap: Publisher where self.maxTasks = maxTasks self.upstream = upstream self.transform = { (value) throws(Failure) in - .init(base: try await transform(value)) + try await .init(base: transform(value)) } + self.taskExecutor = nil } @available(macOS 15.0, iOS 18.0, watchOS 11.0, tvOS 18.0, visionOS 2.0, *) @usableFromInline init( priority: TaskPriority? = nil, + taskExecutor: (any TaskExecutor)? = nil, maxTasks: Subscribers.Demand, upstream: Upstream, typedTransform: @escaping @isolated(any) @Sendable (Upstream.Output) async throws(Failure) -> sending Source @@ -68,8 +76,9 @@ struct AsyncFlatMap: Publisher where self.maxTasks = maxTasks self.upstream = upstream self.transform = { (value) throws(Failure) in - .init(base: try await typedTransform(value)) + try await .init(base: typedTransform(value)) } + self.taskExecutor = taskExecutor } @@ -139,7 +148,7 @@ extension AsyncFlatMap { await localTask(isolation: barrier, group: &group) } } - send(completion: .finished) + send(completion: .finished, shouldCancel: false) } @@ -178,7 +187,7 @@ extension AsyncFlatMap { let effect = $0.upstreamSubscription.transition(.resume(subscription)) return (effect, requestValue) } - effect?.run() + (consume effect)?.run() if requestValue && maxTasks > .none { subscription.request(maxTasks) } @@ -191,13 +200,13 @@ extension AsyncFlatMap { } func cancel() { - send(completion: nil) + send(completion: nil, shouldCancel: true) } // almost uncontented call private func handleDownStream( isolation actor: isolated some Actor, - event: Result?, EitherFailure> + event: Result?, Failure> ) async { switch event { case .success(.none): @@ -227,7 +236,7 @@ extension AsyncFlatMap { ) return case .failure(let failure): - send(completion: .failure(failure)) + send(completion: .failure(failure), shouldCancel: true) return } return @@ -235,16 +244,10 @@ extension AsyncFlatMap { // almost uncontented call private func send( - completion: Subscribers.Completion>? + completion: Subscribers.Completion?, + shouldCancel:Bool ) { valueSource.continuation.finish() - let shouldCancel:Bool - switch completion { - case .none, .failure(.second(_)): - shouldCancel = true - case .failure(.first(_)), .finished: - shouldCancel = false - } let (subscriber, effect, interruption, taskEffect) = lock.withLockUnchecked{ let old = $0.subscriber $0.subscriber = nil @@ -261,17 +264,13 @@ extension AsyncFlatMap { } return (old, effect, interruption, taskEffect) } - effect?.run() - interruption?.run() - switch completion { - case .finished: - subscriber?.receive(completion: .finished) - case .failure(.first(let error)), .failure(.second(let error)): - subscriber?.receive(completion: .failure(error)) - case nil: - break + // ensure compiler it is good to destroy the objects + (consume effect)?.run() + (consume interruption)?.run() + (consume taskEffect)?.run() + if let completion { + (consume subscriber)?.receive(completion: completion) } - taskEffect?.run() } // almost uncontented call @@ -379,7 +378,7 @@ extension AsyncFlatMap { await handleDownStream(isolation: barrier, event: .success(.none)) return false case .failure(let error): - await handleDownStream(isolation: barrier, event: .failure(.second(error))) + await handleDownStream(isolation: barrier, event: .failure(error)) return false case .success(let value): await handleDownStream(isolation: barrier, event: .success(.init(value: value))) @@ -398,10 +397,10 @@ extension AsyncFlatMap { } switch result { case .failure(let failure): - await handleDownStream( - isolation: barrier, - event: .failure(.first(failure)) - ) + let block = { (actor: isolated (any Actor)?) in + send(completion: .failure(failure), shouldCancel: false) + } + await block(barrier) return case .success(let value): let isSuccess = group.addTaskUnlessCancelled(priority: nil) { @@ -412,7 +411,7 @@ extension AsyncFlatMap { await barrier.markDone() await handleDownStream( isolation: barrier, - event: .failure(.second(failure)) + event: .failure(failure) ) return case .success(let source): diff --git a/Sources/Tetra/Combine/Publishers+MapTask.swift b/Sources/Tetra/Combine/Publishers+MapTask.swift index 93d2c42..6acb9ec 100644 --- a/Sources/Tetra/Combine/Publishers+MapTask.swift +++ b/Sources/Tetra/Combine/Publishers+MapTask.swift @@ -67,14 +67,23 @@ public struct MapTask: Publisher where Upstream.Outp public func receive(subscriber: S) where S : Subscriber, Upstream.Failure == S.Failure, Output == S.Input { - let processor = Inner(subscriber: subscriber, transform: transform) - let task = Task(priority: priority, operation: processor.run) - processor.resumeCondition(task) - upstream.subscribe(processor) +// let processor = Inner(subscriber: subscriber, transform: transform) +// let task = Task(priority: priority, operation: processor.run) +// processor.resumeCondition(task) +// upstream.subscribe(processor) + MultiMapTask( + priority: priority, + maxTasks: .max(1), + upstream: upstream, + transform: { [transform] value throws(Failure) in + try await transform(consume value).get() + }) + .subscribe(MapTaskInner(description: "MapTask", downstream: subscriber)) } } + extension MapTask: Sendable where Upstream: Sendable {} extension MapTask { @@ -121,11 +130,12 @@ extension MapTask { } return (old, effect, taskEffect) } - effect?.run() + // tell compiler we hope to remove these objects as soon as possible + (consume effect)?.run() if let completion { - subscriber?.receive(completion: completion) + (consume subscriber)?.receive(completion: completion) } - taskEffect?.run() + (consume taskEffect)?.run() } private func send(_ value:Output) throws { diff --git a/Sources/Tetra/Combine/Publishers+TryMapTask.swift b/Sources/Tetra/Combine/Publishers+TryMapTask.swift index 921d58f..00b2e19 100644 --- a/Sources/Tetra/Combine/Publishers+TryMapTask.swift +++ b/Sources/Tetra/Combine/Publishers+TryMapTask.swift @@ -60,44 +60,16 @@ public struct TryMapTask: Publisher where Upstream.O maxTasks: .max(1), upstream: upstream.mapError{ $0 as any Error }, transform: transform - ).subscribe(TryMapTaskInner(downstream: subscriber, upstream: nil)) + ).subscribe(MapTaskInner( + description: "TryMapTask", + downstream: subscriber, + upstream: nil + )) } } -struct TryMapTaskInner: Subscription, Subscriber, CustomStringConvertible, CustomPlaygroundDisplayConvertible { - - var description: String { "TryMapTask" } - - var playgroundDescription: Any { description } - - var downstream:S - var upstream:Subscription? = nil - var combineIdentifier: CombineIdentifier { downstream.combineIdentifier } - - func receive(_ input: S.Input) -> Subscribers.Demand { - downstream.receive(input) - } - - func receive(subscription: any Subscription) { - let newSubscription = Self(downstream: downstream, upstream: subscription) - downstream.receive(subscription: newSubscription) - } - - func receive(completion: Subscribers.Completion) { - downstream.receive(completion: completion) - } - - func request(_ demand: Subscribers.Demand) { - upstream?.request(demand) - } - - func cancel() { - upstream?.cancel() - } - -} extension TryMapTask: Sendable where Upstream: Sendable {} @@ -145,11 +117,11 @@ extension TryMapTask { } return (old, effect, taskEffect) } - effect?.run() + (consume effect)?.run() if let completion { - subscriber?.receive(completion: completion) + (consume subscriber)?.receive(completion: completion) } - taskEffect?.run() + (consume taskEffect)?.run() } private func send(_ value:Output) throws { diff --git a/Sources/Tetra/Combine/RunLoopScheduler.swift b/Sources/Tetra/Combine/RunLoopScheduler.swift index aa09698..93ca6b8 100644 --- a/Sources/Tetra/Combine/RunLoopScheduler.swift +++ b/Sources/Tetra/Combine/RunLoopScheduler.swift @@ -202,21 +202,19 @@ public final class RunLoopScheduler: Scheduler, @unchecked Sendable, Hashable { nonisolated public var minimumTolerance: SchedulerTimeType.Stride { 0.0 } - public func scheduleTask(_ block: @escaping () throws -> T) async rethrows -> T { - let result:Result = await withUnsafeContinuation{ continuation in + public func scheduleTask(_ block: @escaping () throws(Failure) -> T) async throws(Failure) -> T { + let result:Result = await withUnsafeContinuation{ continuation in CFRunLoopPerformBlock(cfRunLoop, CFRunLoopMode.commonModes.rawValue) { - continuation.resume(returning: .init(catching: { try block() })) + let result = Result { () throws(Failure) in + return try block() + } + continuation.resume(returning: result) } if CFRunLoopIsWaiting(cfRunLoop) { CFRunLoopWakeUp(cfRunLoop) } } - switch result { - case .success(let success): - return success - case .failure: - try result._rethrowOrFail() - } + return try result.get() } @usableFromInline diff --git a/Sources/Tetra/Combine/SubscriptionContinuation.swift b/Sources/Tetra/Combine/SubscriptionContinuation.swift deleted file mode 100644 index 8192068..0000000 --- a/Sources/Tetra/Combine/SubscriptionContinuation.swift +++ /dev/null @@ -1,106 +0,0 @@ -// -// SubscriptionContinuation.swift -// -// -// Created by pbk on 2023/01/29. -// - -import Foundation -@preconcurrency import Combine - -@usableFromInline -internal enum SubscriptionContinuation { - - case waiting - case cached(any Subscription) - case suspending(UnsafeContinuation) - case finished - - enum Event { - case resume(any Subscription) - case suspend(UnsafeContinuation) - case cancel - } - - enum Effect { - case drop(any Subscription) - case resume(UnsafeContinuation, any Subscription) - case cancel(UnsafeContinuation) - - consuming func run() { - switch consume self { - case .drop(let subscription): - subscription.cancel() - case .resume(let unsafeContinuation, let subscription): - unsafeContinuation.resume(returning: subscription) - case .cancel(let unsafeContinuation): - unsafeContinuation.resume(returning: nil) - } - } - } - - mutating func transition(_ event:consuming Event) -> Effect? { - switch consume event { - case .resume(let subscription): - return resume(subscription) - case .suspend(let unsafeContinuation): - return suspend(unsafeContinuation) - case .cancel: - return cancel() - } - } - - - private mutating func resume(_ subscription: any Subscription) -> Effect? { - switch self { - case .waiting: - self = .cached(subscription) - return nil - case .cached(let oldValue): - self = .cached(subscription) - assertionFailure("received subscption more than once") - return .drop(oldValue) - case .suspending(let unsafeContinuation): - self = .finished - return .resume(unsafeContinuation, subscription) - case .finished: - return nil - } - } - - private mutating func suspend(_ continuation: UnsafeContinuation) -> Effect? { - switch self { - case .waiting: - self = .suspending(continuation) - return nil - case .cached(let subscription): - self = .finished - return .resume(continuation, subscription) - case .suspending(let oldValue): - self = .suspending(continuation) - assertionFailure("received continuation more than once") - - return .cancel(oldValue) - case .finished: - - return .cancel(continuation) - } - } - - private mutating func cancel() -> Effect? { - switch self { - case .waiting: - self = .finished - return nil - case .cached(let subscription): - self = .finished - return .drop(subscription) - case .suspending(let unsafeContinuation): - self = .finished - return .cancel(unsafeContinuation) - case .finished: - return nil - } - } - -} diff --git a/Sources/Tetra/Concurrency/AsyncSequencePublisher.swift b/Sources/Tetra/Concurrency/AsyncSequencePublisher.swift index 3bc39d0..da1e99b 100644 --- a/Sources/Tetra/Concurrency/AsyncSequencePublisher.swift +++ b/Sources/Tetra/Concurrency/AsyncSequencePublisher.swift @@ -9,38 +9,49 @@ import Foundation @preconcurrency import Combine public import BackPortAsyncSequence internal import CriticalSection +import Namespace -public extension AsyncSequence { +public extension TetraExtension where Base: AsyncSequence { - @inlinable - var tetra:TetraExtension { - .init(self) + @available(*, deprecated, renamed: "toPublisher()", message: "use toPublisher() which provides task priority and isolation") + var publisher:some Publisher { + toPublisher() } -} - -public extension TetraExtension where Base: AsyncSequence { @_disfavoredOverload @inlinable - var publisher:some Publisher { - AsyncSequencePublisher(base: LegacyTypedAsyncSequence(base: base)) + func toPublisher( + barrier: (any Actor)? = #isolation, + priority: TaskPriority? = nil + ) -> some Publisher { + AsyncSequencePublisher( + base: LegacyTypedAsyncSequence(base: base), + barrier: barrier, + priority: priority + ) } } -public extension TetraExtension where Base: AsyncSequence, Base.AsyncIterator: TypedAsyncIteratorProtocol { +public extension TetraExtension where Base: TypedAsyncSequence, Base.AsyncIterator: TypedAsyncIteratorProtocol { @inlinable - var publisher:some Publisher { - AsyncSequencePublisher(base: base) + func toPublisher( + barrier: (any Actor)? = #isolation, + priority: TaskPriority? = nil + ) -> some Publisher where Element == Base.Element, Failure == Base.AsyncIterator.Err { + AsyncSequencePublisher( + base: base, + barrier: barrier, + priority: priority + ) } } public struct AsyncSequencePublisher: Publisher where Base.AsyncIterator: TypedAsyncIteratorProtocol { - public typealias Output = Base.AsyncIterator.Element public typealias Failure = Base.AsyncIterator.Err diff --git a/Sources/Tetra/Concurrency/CoreDataStack+Concurrency.swift b/Sources/Tetra/Concurrency/CoreDataStack+Concurrency.swift index 879577d..aba1a83 100644 --- a/Sources/Tetra/Concurrency/CoreDataStack+Concurrency.swift +++ b/Sources/Tetra/Concurrency/CoreDataStack+Concurrency.swift @@ -10,10 +10,9 @@ import _Concurrency #if canImport(CoreData) import CoreData +import Namespace + -extension NSPersistentContainer: TetraExtended {} -extension NSPersistentStoreCoordinator: TetraExtended {} -extension NSManagedObjectContext: TetraExtended {} extension TetraExtension where Base: NSPersistentStoreCoordinator { diff --git a/Sources/Tetra/Concurrency/Dispatch+Extension.swift b/Sources/Tetra/Concurrency/Dispatch+Extension.swift index e9bc907..66b4e04 100644 --- a/Sources/Tetra/Concurrency/Dispatch+Extension.swift +++ b/Sources/Tetra/Concurrency/Dispatch+Extension.swift @@ -8,8 +8,8 @@ @preconcurrency import Foundation import Dispatch internal import CriticalSection +import Namespace -extension Task: TetraExtended {} public extension TetraExtension where Base == Task { diff --git a/Sources/Tetra/Concurrency/Notification+AsyncSequence.swift b/Sources/Tetra/Concurrency/Notification+AsyncSequence.swift index a6cab53..159162e 100644 --- a/Sources/Tetra/Concurrency/Notification+AsyncSequence.swift +++ b/Sources/Tetra/Concurrency/Notification+AsyncSequence.swift @@ -9,10 +9,10 @@ import Foundation import _Concurrency internal import BackPortAsyncSequence +import Namespace public import CriticalSection -extension NotificationCenter: TetraExtended {} extension TetraExtension where Base: NotificationCenter { diff --git a/Sources/Tetra/Concurrency/TaskValueContinuation.swift b/Sources/Tetra/Concurrency/TaskValueContinuation.swift index c8622ae..a9a5541 100644 --- a/Sources/Tetra/Concurrency/TaskValueContinuation.swift +++ b/Sources/Tetra/Concurrency/TaskValueContinuation.swift @@ -53,17 +53,18 @@ enum TaskValueContinuation: Sendable { } } + borrowing func shouldMutate(_ event: Event) -> Bool { switch self { case .waiting: return true - case .suspending(let unsafeContinuation): + case .suspending(_): if case .suspend(_) = event { return false } else { return true } - case .cached(let task): + case .cached(_): if case .resume(_) = event { return false } else { diff --git a/Sources/Tetra/Concurrency/TetraExtension+URLSession.swift b/Sources/Tetra/Concurrency/TetraExtension+URLSession.swift index d0abd8a..3b5c166 100644 --- a/Sources/Tetra/Concurrency/TetraExtension+URLSession.swift +++ b/Sources/Tetra/Concurrency/TetraExtension+URLSession.swift @@ -6,8 +6,8 @@ // import Foundation +import Namespace -extension URLSession: TetraExtended {} @available(iOS 13.0, tvOS 13.0, macCatalyst 13.0, macOS 10.15, watchOS 6.0, *) public extension TetraExtension where Base: URLSession { diff --git a/Sources/Tetra/Foundation/EitherFailure.swift b/Sources/Tetra/Foundation/EitherFailure.swift deleted file mode 100644 index a222b47..0000000 --- a/Sources/Tetra/Foundation/EitherFailure.swift +++ /dev/null @@ -1,12 +0,0 @@ -// -// EitherFailure.swift -// -// -// Created by 박병관 on 6/22/24. -// - -enum EitherFailure:Error { - - case first(First) - case second(Second) -} diff --git a/Sources/Tetra/Foundation/ManagedUnfairLock.swift b/Sources/Tetra/Foundation/ManagedUnfairLock.swift deleted file mode 100644 index 8b13789..0000000 --- a/Sources/Tetra/Foundation/ManagedUnfairLock.swift +++ /dev/null @@ -1 +0,0 @@ - diff --git a/Sources/Tetra/Foundation/Mics.swift b/Sources/Tetra/Foundation/Mics.swift index 3130c3d..4344c82 100644 --- a/Sources/Tetra/Foundation/Mics.swift +++ b/Sources/Tetra/Foundation/Mics.swift @@ -9,45 +9,6 @@ import Foundation import Combine import os -internal enum SubscriptionStatus { - case awaitingSubscription - case subscribed(any Subscription) - case terminal - - var subscription:Subscription? { - guard case .subscribed(let subscription) = self else { - return nil - } - return subscription - } - -} - - - -@rethrows -@usableFromInline -internal protocol _ErrorMechanism { - associatedtype Output - func get() throws -> Output -} - -extension _ErrorMechanism { - // rethrow an error only in the cases where it is known to be reachable - - @usableFromInline - internal func _rethrowOrFail() rethrows -> Never { - _ = try _rethrowGet() - fatalError("materialized error without being in a throwing context") - } - - @usableFromInline - internal func _rethrowGet() rethrows -> Output { - return try get() - } -} - -extension Result: _ErrorMechanism { } diff --git a/Sources/Tetra/SwiftUI/AsyncImage+BackPort.swift b/Sources/Tetra/SwiftUI/AsyncImage+BackPort.swift index 0293d01..444b138 100644 --- a/Sources/Tetra/SwiftUI/AsyncImage+BackPort.swift +++ b/Sources/Tetra/SwiftUI/AsyncImage+BackPort.swift @@ -8,6 +8,7 @@ import Foundation import Combine +import Namespace #if canImport(SwiftUI) diff --git a/Sources/Tetra/SwiftUI/Binding+Collection.swift b/Sources/Tetra/SwiftUI/Binding+Collection.swift index a6357a1..e745c78 100644 --- a/Sources/Tetra/SwiftUI/Binding+Collection.swift +++ b/Sources/Tetra/SwiftUI/Binding+Collection.swift @@ -29,7 +29,7 @@ public extension Binding where Value: MutableCollection { @available(macCatalyst, deprecated: 15.0, message: "use Binding itself as Collection") @available(tvOS, deprecated: 15.0, message: "use Binding itself as Collection") @available(iOS, deprecated: 15.0, message: "use Binding itself as Collection") -public struct BindingCollection: Collection, Sendable { +public struct BindingCollection: Collection { @usableFromInline @Binding var collection:T @@ -45,9 +45,9 @@ public struct BindingCollection: Collection, Sendable { } else { nonisolated(unsafe) let index = consume position - return .init { + return .init { [binding] in binding.wrappedValue[index] - } set: { newValue, transaction in + } set: { [binding] newValue, transaction in withTransaction(transaction) { binding.wrappedValue[index] = newValue } diff --git a/Sources/Tetra/TetraExtension.swift b/Sources/Tetra/TetraExtension.swift index 2a3c8b3..c3b9726 100644 --- a/Sources/Tetra/TetraExtension.swift +++ b/Sources/Tetra/TetraExtension.swift @@ -6,57 +6,7 @@ // import Foundation - -public struct TetraExtension { - - @usableFromInline - internal var value:Base - @inlinable - public var base:Base { - get { value } - } - - @inlinable - public init(base: Base) { - self.value = base - } - - @inlinable - public init(_ base: Base) { - self.value = base - } - - -} - - -extension TetraExtension: Sendable where Base:Sendable {} - -public protocol TetraExtended { - /// Type being extended. - associatedtype Base - - /// Static Tetra extension point. - @inlinable - static var tetra: TetraExtension.Type { get set } - /// Instance Tetra extension point. - @inlinable - var tetra: TetraExtension { get set } -} - -extension TetraExtended { - /// Static Tetra extension point. - @inlinable - public static var tetra: TetraExtension.Type { - get { TetraExtension.self } - set {} - } - - /// Instance Tetra extension point. - @inlinable - public var tetra: TetraExtension { - get { TetraExtension(self) } - set {} - } -} - +//public import Namespace +//public import NamespaceExtension +//public import BackPortAsyncSequence +//public import CriticalSection diff --git a/Tests/TetraTests/AsyncFlatMapTests.swift b/Tests/TetraTests/AsyncFlatMapTests.swift index 9f38a66..42b76c6 100644 --- a/Tests/TetraTests/AsyncFlatMapTests.swift +++ b/Tests/TetraTests/AsyncFlatMapTests.swift @@ -24,17 +24,18 @@ final class AsyncFlatMapTests: XCTestCase { } ) .asyncFlatMap(maxTasks: .max(1)) { value in - let base = AsyncTypedStream(base: AsyncStream{ continuation in + return AsyncStream{ continuation in sample.forEach{ continuation.yield($0) } continuation.finish() - }) - return BackPort.AsyncMapSequence(base, transform: { - await Task.yield() - return $0 - }) - }.handleEvents( + }.tetra.bridge() + .tetra.map{ + await Task.yield() + return $0 + } + } + .handleEvents( receiveSubscription: { XCTAssertEqual("\($0)", "AsyncFlatMap") } diff --git a/Tests/TetraTests/AsyncSequencePublisherTests.swift b/Tests/TetraTests/AsyncSequencePublisherTests.swift index fb47717..d51c809 100644 --- a/Tests/TetraTests/AsyncSequencePublisherTests.swift +++ b/Tests/TetraTests/AsyncSequencePublisherTests.swift @@ -10,6 +10,7 @@ import XCTest @testable import Tetra import Combine import BackPortAsyncSequence +import Namespace class AsyncSequencePublisherTests: XCTestCase { @@ -24,7 +25,7 @@ class AsyncSequencePublisherTests: XCTestCase { continuation.finish() } let cancellable = AsyncTypedStream(base: stream) - .tetra.publisher + .tetra.toPublisher() .catch{ _ in XCTFail() return Empty() diff --git a/Tests/TetraTests/NSManagedObjectContextTests.swift b/Tests/TetraTests/NSManagedObjectContextTests.swift index 20f5e5c..7c0504a 100644 --- a/Tests/TetraTests/NSManagedObjectContextTests.swift +++ b/Tests/TetraTests/NSManagedObjectContextTests.swift @@ -9,6 +9,7 @@ import XCTest #if canImport(CoreData) import CoreData @testable import Tetra +internal import NamespaceExtension final class NSManagedObjectContextTests: XCTestCase { diff --git a/Tests/TetraTests/URLSessionDownloadTests.swift b/Tests/TetraTests/URLSessionDownloadTests.swift index a460ec3..d648531 100644 --- a/Tests/TetraTests/URLSessionDownloadTests.swift +++ b/Tests/TetraTests/URLSessionDownloadTests.swift @@ -7,6 +7,7 @@ import XCTest @testable import Tetra +import Namespace final class URLSessionDownloadTests: XCTestCase { diff --git a/Tests/TetraTests/Utlitity/JsonSampleModel.swift b/Tests/TetraTests/Utlitity/JsonSampleModel.swift index 9d46951..bad2a26 100644 --- a/Tests/TetraTests/Utlitity/JsonSampleModel.swift +++ b/Tests/TetraTests/Utlitity/JsonSampleModel.swift @@ -12,7 +12,7 @@ struct JsonSample1Model: Codable, Hashable { var imaging:[LocationItem] var labs:[LocationItem] - + var medications:[Medical] struct LocationItem: Codable, Hashable { @@ -34,7 +34,7 @@ struct JsonSample1Model: Codable, Hashable { struct MetaData: Codable, Hashable { - var does:String + var dose:String var name:String var pillCount:String var refills:String From 9d018b535a1542270711e201ec80f76a629ad17f Mon Sep 17 00:00:00 2001 From: Byeong Gwan Date: Fri, 28 Jun 2024 19:13:48 +0900 Subject: [PATCH 36/63] Update README.md fix MulitmapTask readme description --- README.md | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index c48a538..dbcbe9a 100644 --- a/README.md +++ b/README.md @@ -71,13 +71,16 @@ import Combine let cancellable = (0..<20).publisher .setFailureType(to: URLError.self) - .multiMapTask(maxTasks: .unlimited) { _ in + .multiMapTask(maxTasks: .unlimited) { _ throws(URLError) in + // Underlying Task is cancelled if subscription is cancelled before task completes. do { let (data, response) = try await URLSession.shared.data(from: URL(string: "https://google.com")!) - return .success(data) as Result + // below unsafe cancel is no-op, throw appropriate error to interrupt combine pipeline + // withUnsafeCurrentTask { $0?.cancel() } + return data } catch { - return .failure(error as! URLError) as Result + throw (error as! URLError) } }.sink { completion in @@ -101,8 +104,7 @@ import Tetra continuation.yield(1) continuation.finish() // Underlying AsyncIterator and Task receive task cancellation if subscription is cancelled. - }.tetra.publisher - .catch{ _ in Empty().setFailureType(to: Never.self) } + }.tetra.bridge.tetra.publisher .sink { number in } From 404281dbb4bffc96545d45b33b7007345527d6e1 Mon Sep 17 00:00:00 2001 From: park-byeong-gwan Date: Sun, 30 Jun 2024 03:38:41 +0900 Subject: [PATCH 37/63] add TaskLocal test failing case for request about issue --- Package.swift | 17 +- Sources/CriticalSection/PriorityRunLoop.swift | 524 ++++++++++++++++++ Tests/RunLoopExecutorTest/Tests.swift | 60 ++ 3 files changed, 597 insertions(+), 4 deletions(-) create mode 100644 Sources/CriticalSection/PriorityRunLoop.swift create mode 100644 Tests/RunLoopExecutorTest/Tests.swift diff --git a/Package.swift b/Package.swift index e25b66d..1cf9a9f 100644 --- a/Package.swift +++ b/Package.swift @@ -25,10 +25,10 @@ let package = Package( // .package(url: /* package url */, from: "1.0.0"), .package(url: "https://github.com/apple/swift-collections.git", .upToNextMajor(from: "1.1.0")), .package( - url: "https://github.com/apple/swift-atomics.git", - .upToNextMajor(from: "1.2.0") // or `.upToNextMinor + url: "https://github.com/apple/swift-atomics.git", + .upToNextMajor(from: "1.2.0") // or `.upToNextMinor ), - + ], targets: [ // Targets are the basic building blocks of a package. A target can define a module or a test suite. @@ -48,10 +48,18 @@ let package = Package( .swiftLanguageVersion(.v6) ] ), + .testTarget(name: "RunLoopExecutorTest", + + dependencies: [ + "CriticalSection" + ] + ), .target( name: "CriticalSection", dependencies: [ - .product(name: "Atomics", package: "swift-atomics") + .product(name: "Atomics", package: "swift-atomics"), + .product(name: "HeapModule", package: "swift-collections"), + ], swiftSettings: [ .swiftLanguageVersion(.v6), @@ -60,6 +68,7 @@ let package = Package( .enableExperimentalFeature("BuiltinModule") ] ), + .target( name: "BackportDiscardingTaskGroup", dependencies: [ diff --git a/Sources/CriticalSection/PriorityRunLoop.swift b/Sources/CriticalSection/PriorityRunLoop.swift new file mode 100644 index 0000000..b2cbece --- /dev/null +++ b/Sources/CriticalSection/PriorityRunLoop.swift @@ -0,0 +1,524 @@ +// +// PriorityRunLoop.swift +// +// +// Created by 박병관 on 6/29/24. +// + +import Atomics +import HeapModule +import CoreFoundation + +extension LockFreeQueue: @unchecked Sendable where Element: Sendable {} +final class LockFreeQueue { + + final class Node: AtomicReference { + let next: ManagedAtomic + var value: Element? + + init(value: consuming Element?, next: Node?) { + self.value = value + self.next = ManagedAtomic(next) + } + + deinit { + var values = 0 + // Prevent stack overflow when reclaiming a long queue + var node = self.next.exchange(nil, ordering: .relaxed) + while node != nil && isKnownUniquelyReferenced(&node) { + let next = node!.next.exchange(nil, ordering: .relaxed) + withExtendedLifetime(node) { + values += 1 + } + node = next + } + if values > 0 { + print(values) + } + } + } + + let head: ManagedAtomic + let tail: ManagedAtomic + + // Used to distinguish removed nodes from active nodes with a nil `next`. + let marker = Node(value: nil, next: nil) + private let counter = UnsafeAtomic.create(0) + + let sanityCheck = ManagedAtomic(false) + + init() { + let dummy = Node(value: nil, next: nil) + self.head = ManagedAtomic(dummy) + self.tail = ManagedAtomic(dummy) + + } + + deinit { + counter.destroy() + + } + + func enqueue(_ newValue: consuming Element) { + if sanityCheck.load(ordering: .acquiring) { + preconditionFailure("queue is want's to be closed") + } + let new = Node(value: newValue, next: nil) + var tail = self.tail.load(ordering: .acquiring) + while true { + let next = tail.next.load(ordering: .acquiring) + if tail === marker || next === marker { + // The node we loaded has been unlinked by a dequeue on another thread. + // Try again. + tail = self.tail.load(ordering: .acquiring) + DispatchQueue.global().async { + print("enqueue","contention", "1") + } + continue + } + if let next = next { + // Assist competing threads by nudging `self.tail` forward a step. + let (exchanged, original) = self.tail.compareExchange( + expected: tail, + desired: next, + ordering: .acquiringAndReleasing) + tail = (exchanged ? next : original) + DispatchQueue.global().async { + print("enqueue","contention", "2") + } + continue + } + let (exchanged, current) = tail.next.compareExchange( + expected: nil, + desired: new, + ordering: .acquiringAndReleasing + ) + if exchanged { + _ = self.tail.compareExchange(expected: tail, desired: new, ordering: .releasing) + counter.wrappingIncrement(ordering: .releasing) + return + } + DispatchQueue.global().async { + print("enqueue","contention", "3") + } + tail = current! + } + } + + func dequeue() -> Element? { + while true { + let head = self.head.load(ordering: .acquiring) + let next = head.next.load(ordering: .acquiring) + if next === marker { + DispatchQueue.global().async { + print("dequeue","contention", "1") + } +// print("dequeue", "contention", 1) + continue + } + guard let n = next else { return nil } + let tail = self.tail.load(ordering: .acquiring) + if head === tail { + // Nudge `tail` forward a step to make sure it doesn't fall off the + // list when we unlink this node. + _ = self.tail.compareExchange(expected: tail, desired: n, ordering: .acquiringAndReleasing) + } + if self.head.compareExchange(expected: head, desired: n, ordering: .releasing).exchanged { + var result:Element? = nil + swap(&result, &n.value) + // To prevent threads that are suspended in `enqueue`/`dequeue` from + // holding onto arbitrarily long chains of removed nodes, we unlink + // removed nodes by replacing their `next` value with the special + // `marker`. + head.next.store(marker, ordering: .releasing) + counter.wrappingDecrement(ordering: .releasing) + return result + } + DispatchQueue.global().async { + print("dequeue","contention", "2") + } +// print("dequeue", "contention", 2) + + } + } + + func estimatedLength() -> Int { + counter.load(ordering: .acquiring) + } + + + + +} +import Foundation + +@available(macOS 14.0, iOS 17.0, watchOS 10.0, tvOS 17.0, *) +struct ExecutorJobContext: ~Copyable { + + let job:ExecutorJob + let executor:UnownedSerialExecutor + + consuming func consume() -> ExecutorJob { + return job + } + +} + +struct C333 { + + + weak var c:T? = nil +} + + +@available(macOS 14.0, iOS 17.0, watchOS 10.0, tvOS 17.0, *) +package +final class RunLoopPriorityExecutor: SerialExecutor { + + struct ExecutorContext: Sendable { + + let queue:LockFreeQueue + nonisolated(unsafe) + let source:CFRunLoopSource + + } +// static + @TaskLocal + static let myLocal: ExecutorContext? = nil + + private let queue:LockFreeQueue + private let sourceRef:RunLoopSourceRef + private let owned:Bool + + nonisolated(unsafe) + private let runloop:RunLoop + + internal + init() { + + let existing = Self.myLocal + self.queue = existing?.queue ?? .init() + self.runloop = .current + self.owned = true + if let source = existing?.source { + var context = CFRunLoopSourceContext() + CFRunLoopSourceGetContext(source, &context) + let info = context.info! + self.sourceRef = Unmanaged.fromOpaque(info).takeUnretainedValue() + } else { + self.sourceRef = .init(ref: queue, nested: false) + } + } + + internal + init(nested:()) { + let existing = Self.myLocal + self.queue = existing?.queue ?? .init() + self.runloop = .current + self.owned = false + self.sourceRef = .init(null: ()) + +// if runloop == .main { +// self.sourceRef = .init(null: ()) +// } else { +// self.sourceRef = .init(ref: queue, nested: true) +// CFRunLoopAddSource(runloop.getCFRunLoop(), sourceRef.source, .commonModes) +// } + + } + + private init( + queue:LockFreeQueue, + runLoop:RunLoop + ) { + self.queue = queue + self.runloop = runLoop + self.owned = true + self.sourceRef = .init(null: ()) + } + + package + func enqueue(_ job: consuming ExecutorJob) { + if runloop == .main { + MainActor.shared.enqueue(UnownedJob(job)) + return + } + if !owned { + let jobRef = UnownedJob(job) + let executorRef = asUnownedSerialExecutor() + runloop.perform { + jobRef.runSynchronously(on: executorRef) + } + return + } + queue.enqueue(.init(job: job, executor: asUnownedSerialExecutor())) + CFRunLoopSourceSignal(sourceRef.source) + CFRunLoopWakeUp(runloop.getCFRunLoop()) + } + + package + func checkIsolated() { + precondition(runloop == .current) + } + + var inRunLoop: Bool { + runloop == .current + } + + package + func isSameExclusiveExecutionContext(other: RunLoopPriorityExecutor) -> Bool { + return runloop == other.runloop + } + + package + func asUnownedSerialExecutor() -> UnownedSerialExecutor { + if runloop == .main { + return MainActor.sharedUnownedExecutor + } else { + return .init(complexEquality: self) + } + } + + func makeContext() -> ExecutorContext { + .init(queue: queue, source: sourceRef.source) + } + + // This is the main of runloop thread + //ExecutorContext contains JobQueue and RunLoopSource + static func controlRunLoop(context:ExecutorContext) { + guard Self.myLocal == nil, + RunLoop.current.currentMode == nil, + RunLoop.current != RunLoop.main + else { return } + defer { + while true { + let (exchanged, _ ) = context.queue.sanityCheck.compareExchange(expected: false, desired: true, ordering: .releasing) + if exchanged { + break + } + } + // last safety check + Self.$myLocal.withValue(context) { + Self.processEvents() + } + } + Self.$myLocal.withValue(context) { + CFRunLoopAddSource(CFRunLoopGetCurrent(), context.source, .commonModes) + while CFRunLoopSourceIsValid(context.source) { + let passed = RunLoop.current.run(mode: .default, before: .distantFuture) + if !passed { + break + } + } + } + + } + + static func pump( + queue:LockFreeQueue + ) { + let estimatedCap = queue.estimatedLength() + var buffer = Heap(minimumCapacity: estimatedCap) + buffer.reserveCapacity(estimatedCap) + print("pump and nil", Self.myLocal == nil) + var id:UInt64 = .max + while let block = queue.dequeue() { + defer { id -= 1 } + let ref = JobBlock(id: id, job: block) + buffer.insert(ref) + } + while let job = buffer.popMax() { + job.jobImp.runSynchronously(on: job.executor) + } + + } + + static func processEvents() { + if let queue = Self.myLocal?.queue { + pump(queue: queue) + } + + } + +} + +import os + +@available(macOS 14.0, iOS 17.0, watchOS 10.0, tvOS 17.0, *) +final class RunLoopSourceRef: @unchecked Sendable { + + private(set) var source:CFRunLoopSource! + let queue:LockFreeQueue! + + init(null:()) { + self.source = nil + self.queue = nil + } + + init(ref: LockFreeQueue, nested:Bool) { + if nested { + queue = ref + } else { + queue = nil + } + var context = CFRunLoopSourceContext() + context.version = 0 + context.info = Unmanaged.passUnretained(self).toOpaque() + if nested { + context.perform = { + let ref: RunLoopSourceRef = Unmanaged.fromOpaque($0!).takeUnretainedValue() + RunLoopPriorityExecutor.pump(queue: ref.queue) + } + context.cancel = { info, runLoop, mode in + let ref: RunLoopSourceRef = Unmanaged.fromOpaque(info!).takeUnretainedValue() + if let runLoop { + CFRunLoopStop(runLoop) + } + CFRunLoopPerformBlock(runLoop, CFRunLoopMode.commonModes.rawValue) { + RunLoopPriorityExecutor.pump(queue: ref.queue) + } + } + } else { + context.cancel = { info, runLoop, mode in + if let runLoop { + CFRunLoopStop(runLoop) + } + } + context.perform = { _ in + RunLoopPriorityExecutor.processEvents() + } + } + + context.copyDescription = { _ in + let description = "RunLoopPriorityExecutor-Source" + return .passRetained(description as CFString) + } + self.source = CFRunLoopSourceCreate(nil, 0, &context) + } + + + deinit { + if let source { + CFRunLoopSourceInvalidate(source) + } + } + +} + + +/// Transform current Thread as the RunLoop Executor, and run the runLoop +/// +/// +/// Actual behavior depends on the current RunLoop state. +/// +/// 1) called from existing `RunLoopPriorityExecutor` thread. +/// new executor is connected to the cached component and return, executor lifetime is shared with previous executor. +/// This does not runs runloop. RunLoop is deactivated when this, and all previous executor is dead. +/// 2) called from `MainThread` +/// create dummy executor and return. Dummy executor dispatch all the jobs to the `MainActor` +/// 3) called from active runloop Thread. (someone is already controlling the RunLoop) +/// create unoptimized executor and return. This executor does not controls the runLoop. Existing RunLoop owner has the resposibility to keep runLoop alive, otherwise enqued Job would leak. +/// This executor does not support JobPriority. And simply create NSObject and schedule the block +/// +/// 4) called from fresh Thread (no one is running runLoop) +/// create optimized executor, call `setupHandle` than controls the RunLoop of current Thread. +/// This function does not return, until executor is dead. So, `setupHandle` is the entrypoint of using the executor. +/// This executor does recognize priority +/// - Parameter setupHandle: called right before runLoop runs, runloop is active until executor is dead. this block is called exactly once. +@available(macOS 14.0, iOS 17.0, watchOS 10.0, tvOS 17.0, *) +package +func executeRunloop( + setupHandle: (RunLoopPriorityExecutor) -> Void +) { + // if nested -> already running runloop 1) we are controlling the runloop 2) somebody is controlling runloop + // 1) our executor is already controlling thread and we are calling it at the same thread + // -> do not perform any nested run + // 2) somebody is already taking control of this thread + // if mainloop -> no-op redirect to mainactor + // if no-runloop -> controll it! + guard !Thread.isMainThread else { + // we don't run main runLoop since MainActor is the way togo + // `RunLoopPriorityExecutor` redirect every thing back to the MainActor + setupHandle(RunLoopPriorityExecutor(nested: ())) + return + } + let existing = RunLoopPriorityExecutor.myLocal + if let existing, CFRunLoopContainsSource(CFRunLoopGetCurrent(), existing.source, .commonModes) { + // nested runloop which we are taking full control or main runloop + // connect to the existing task-queue and return + setupHandle(RunLoopPriorityExecutor.init()) + return + } + + + // complex case + if RunLoop.current.currentMode != nil { + // we are in the active runloop which someone else is taking the full control + // nested runloop is not an ideal case + + // configure runloop source and attach it maybe? + // but in that case tasklocal is not visible in same thread ... + // lets fall back to unoptimized way, stashing every job as Clousre block, ignoring priority + // this RunLoopExecutor never controls the runloop + let executor = RunLoopPriorityExecutor(nested: ()) + setupHandle(executor) + return + } + let context:RunLoopPriorityExecutor.ExecutorContext + do { + let executor = RunLoopPriorityExecutor() + context = executor.makeContext() + setupHandle(consume executor) + } + RunLoopPriorityExecutor.controlRunLoop(context: context) +} + + + + +struct JobBlock: Hashable, Comparable { + + let id:UInt64 + let priority:UInt8 + let jobImp:UnownedJob + let executor:UnownedSerialExecutor + + @available(macOS 14.0, *) + init(id: UInt64, job: consuming ExecutorJobContext) { + self.id = id + self.priority = job.job.priority.rawValue + self.executor = job.executor + self.jobImp = UnownedJob(job.job) + + } + + init(id: UInt64, jobRef: UnownedJob, executor:UnownedSerialExecutor) { + self.id = id + self.priority = 0 + self.jobImp = jobRef + self.executor = executor + } + + func hash(into hasher: inout Hasher) { + hasher.combine(priority) + } + + static func == (lhs: Self, rhs: Self) -> Bool { + lhs.id == rhs.id + } + + static func < (lhs: Self, rhs: Self) -> Bool { + if lhs.priority == rhs.priority { + return lhs.id < rhs.id + } + return lhs.priority < rhs.priority + } + + static func > (lhs:Self, rhs:Self) -> Bool { + if lhs.priority == rhs.priority { + return lhs.id > rhs.id + } + return lhs.priority > rhs.priority + } + +} diff --git a/Tests/RunLoopExecutorTest/Tests.swift b/Tests/RunLoopExecutorTest/Tests.swift new file mode 100644 index 0000000..b567989 --- /dev/null +++ b/Tests/RunLoopExecutorTest/Tests.swift @@ -0,0 +1,60 @@ +// +// Tests.swift +// +// +// Created by 박병관 on 6/30/24. +// + +@testable import CriticalSection +import Testing +import Foundation + +@Suite +struct Tests { + + + @Test + func evaluate() async { + if #available(macOS 14.0, *) { + let myActor = await RunLoopActor() + let block = { (act: isolated RunLoopActor) in + + #expect(RunLoopPriorityExecutor.myLocal != nil) + + } + await block(myActor) + } else { + // Fallback on earlier versions + } + } + +} + +@available(macOS 14.0, iOS 17.0, watchOS 10.0, tvOS 17.0, *) + +actor RunLoopActor { + + let executor:RunLoopPriorityExecutor + + init() async { + + let ref:RunLoopPriorityExecutor = await withUnsafeContinuation { continuation in + + let th = Thread{ + executeRunloop { + continuation.resume(returning: $0) + } + + } + th.qualityOfService = .default + th.start() + } + + self.executor = ref + } + nonisolated var unownedExecutor: UnownedSerialExecutor { + executor.asUnownedSerialExecutor() + } + + +} From 632ae77e369dab2bee290be0209753ce95651ac0 Mon Sep 17 00:00:00 2001 From: park-byeong-gwan Date: Fri, 5 Jul 2024 22:44:11 +0900 Subject: [PATCH 38/63] implement Serial Priority based RunLoopExecutor Source --- Package.swift | 12 +- Sources/CriticalSection/PriorityRunLoop.swift | 524 ------------------ Sources/Tetra/Concurrency/JobBlock.swift | 66 +++ .../Tetra/Concurrency/PriorityRunLoop.swift | 346 ++++++++++++ Tests/RunLoopExecutorTest/Tests.swift | 60 -- 5 files changed, 415 insertions(+), 593 deletions(-) delete mode 100644 Sources/CriticalSection/PriorityRunLoop.swift create mode 100644 Sources/Tetra/Concurrency/JobBlock.swift create mode 100644 Sources/Tetra/Concurrency/PriorityRunLoop.swift delete mode 100644 Tests/RunLoopExecutorTest/Tests.swift diff --git a/Package.swift b/Package.swift index 1cf9a9f..e225e50 100644 --- a/Package.swift +++ b/Package.swift @@ -48,27 +48,19 @@ let package = Package( .swiftLanguageVersion(.v6) ] ), - .testTarget(name: "RunLoopExecutorTest", - - dependencies: [ - "CriticalSection" - ] - ), .target( name: "CriticalSection", dependencies: [ .product(name: "Atomics", package: "swift-atomics"), - .product(name: "HeapModule", package: "swift-collections"), ], swiftSettings: [ .swiftLanguageVersion(.v6), .enableExperimentalFeature("StaticExclusiveOnly"), .enableExperimentalFeature("RawLayout"), - .enableExperimentalFeature("BuiltinModule") + .enableExperimentalFeature("BuiltinModule"), ] ), - .target( name: "BackportDiscardingTaskGroup", dependencies: [ @@ -84,6 +76,8 @@ let package = Package( name: "Tetra", dependencies: [ .product(name: "DequeModule", package: "swift-collections"), + .product(name: "HeapModule", package: "swift-collections"), + "BackPortAsyncSequence", "CriticalSection", "BackportDiscardingTaskGroup", diff --git a/Sources/CriticalSection/PriorityRunLoop.swift b/Sources/CriticalSection/PriorityRunLoop.swift deleted file mode 100644 index b2cbece..0000000 --- a/Sources/CriticalSection/PriorityRunLoop.swift +++ /dev/null @@ -1,524 +0,0 @@ -// -// PriorityRunLoop.swift -// -// -// Created by 박병관 on 6/29/24. -// - -import Atomics -import HeapModule -import CoreFoundation - -extension LockFreeQueue: @unchecked Sendable where Element: Sendable {} -final class LockFreeQueue { - - final class Node: AtomicReference { - let next: ManagedAtomic - var value: Element? - - init(value: consuming Element?, next: Node?) { - self.value = value - self.next = ManagedAtomic(next) - } - - deinit { - var values = 0 - // Prevent stack overflow when reclaiming a long queue - var node = self.next.exchange(nil, ordering: .relaxed) - while node != nil && isKnownUniquelyReferenced(&node) { - let next = node!.next.exchange(nil, ordering: .relaxed) - withExtendedLifetime(node) { - values += 1 - } - node = next - } - if values > 0 { - print(values) - } - } - } - - let head: ManagedAtomic - let tail: ManagedAtomic - - // Used to distinguish removed nodes from active nodes with a nil `next`. - let marker = Node(value: nil, next: nil) - private let counter = UnsafeAtomic.create(0) - - let sanityCheck = ManagedAtomic(false) - - init() { - let dummy = Node(value: nil, next: nil) - self.head = ManagedAtomic(dummy) - self.tail = ManagedAtomic(dummy) - - } - - deinit { - counter.destroy() - - } - - func enqueue(_ newValue: consuming Element) { - if sanityCheck.load(ordering: .acquiring) { - preconditionFailure("queue is want's to be closed") - } - let new = Node(value: newValue, next: nil) - var tail = self.tail.load(ordering: .acquiring) - while true { - let next = tail.next.load(ordering: .acquiring) - if tail === marker || next === marker { - // The node we loaded has been unlinked by a dequeue on another thread. - // Try again. - tail = self.tail.load(ordering: .acquiring) - DispatchQueue.global().async { - print("enqueue","contention", "1") - } - continue - } - if let next = next { - // Assist competing threads by nudging `self.tail` forward a step. - let (exchanged, original) = self.tail.compareExchange( - expected: tail, - desired: next, - ordering: .acquiringAndReleasing) - tail = (exchanged ? next : original) - DispatchQueue.global().async { - print("enqueue","contention", "2") - } - continue - } - let (exchanged, current) = tail.next.compareExchange( - expected: nil, - desired: new, - ordering: .acquiringAndReleasing - ) - if exchanged { - _ = self.tail.compareExchange(expected: tail, desired: new, ordering: .releasing) - counter.wrappingIncrement(ordering: .releasing) - return - } - DispatchQueue.global().async { - print("enqueue","contention", "3") - } - tail = current! - } - } - - func dequeue() -> Element? { - while true { - let head = self.head.load(ordering: .acquiring) - let next = head.next.load(ordering: .acquiring) - if next === marker { - DispatchQueue.global().async { - print("dequeue","contention", "1") - } -// print("dequeue", "contention", 1) - continue - } - guard let n = next else { return nil } - let tail = self.tail.load(ordering: .acquiring) - if head === tail { - // Nudge `tail` forward a step to make sure it doesn't fall off the - // list when we unlink this node. - _ = self.tail.compareExchange(expected: tail, desired: n, ordering: .acquiringAndReleasing) - } - if self.head.compareExchange(expected: head, desired: n, ordering: .releasing).exchanged { - var result:Element? = nil - swap(&result, &n.value) - // To prevent threads that are suspended in `enqueue`/`dequeue` from - // holding onto arbitrarily long chains of removed nodes, we unlink - // removed nodes by replacing their `next` value with the special - // `marker`. - head.next.store(marker, ordering: .releasing) - counter.wrappingDecrement(ordering: .releasing) - return result - } - DispatchQueue.global().async { - print("dequeue","contention", "2") - } -// print("dequeue", "contention", 2) - - } - } - - func estimatedLength() -> Int { - counter.load(ordering: .acquiring) - } - - - - -} -import Foundation - -@available(macOS 14.0, iOS 17.0, watchOS 10.0, tvOS 17.0, *) -struct ExecutorJobContext: ~Copyable { - - let job:ExecutorJob - let executor:UnownedSerialExecutor - - consuming func consume() -> ExecutorJob { - return job - } - -} - -struct C333 { - - - weak var c:T? = nil -} - - -@available(macOS 14.0, iOS 17.0, watchOS 10.0, tvOS 17.0, *) -package -final class RunLoopPriorityExecutor: SerialExecutor { - - struct ExecutorContext: Sendable { - - let queue:LockFreeQueue - nonisolated(unsafe) - let source:CFRunLoopSource - - } -// static - @TaskLocal - static let myLocal: ExecutorContext? = nil - - private let queue:LockFreeQueue - private let sourceRef:RunLoopSourceRef - private let owned:Bool - - nonisolated(unsafe) - private let runloop:RunLoop - - internal - init() { - - let existing = Self.myLocal - self.queue = existing?.queue ?? .init() - self.runloop = .current - self.owned = true - if let source = existing?.source { - var context = CFRunLoopSourceContext() - CFRunLoopSourceGetContext(source, &context) - let info = context.info! - self.sourceRef = Unmanaged.fromOpaque(info).takeUnretainedValue() - } else { - self.sourceRef = .init(ref: queue, nested: false) - } - } - - internal - init(nested:()) { - let existing = Self.myLocal - self.queue = existing?.queue ?? .init() - self.runloop = .current - self.owned = false - self.sourceRef = .init(null: ()) - -// if runloop == .main { -// self.sourceRef = .init(null: ()) -// } else { -// self.sourceRef = .init(ref: queue, nested: true) -// CFRunLoopAddSource(runloop.getCFRunLoop(), sourceRef.source, .commonModes) -// } - - } - - private init( - queue:LockFreeQueue, - runLoop:RunLoop - ) { - self.queue = queue - self.runloop = runLoop - self.owned = true - self.sourceRef = .init(null: ()) - } - - package - func enqueue(_ job: consuming ExecutorJob) { - if runloop == .main { - MainActor.shared.enqueue(UnownedJob(job)) - return - } - if !owned { - let jobRef = UnownedJob(job) - let executorRef = asUnownedSerialExecutor() - runloop.perform { - jobRef.runSynchronously(on: executorRef) - } - return - } - queue.enqueue(.init(job: job, executor: asUnownedSerialExecutor())) - CFRunLoopSourceSignal(sourceRef.source) - CFRunLoopWakeUp(runloop.getCFRunLoop()) - } - - package - func checkIsolated() { - precondition(runloop == .current) - } - - var inRunLoop: Bool { - runloop == .current - } - - package - func isSameExclusiveExecutionContext(other: RunLoopPriorityExecutor) -> Bool { - return runloop == other.runloop - } - - package - func asUnownedSerialExecutor() -> UnownedSerialExecutor { - if runloop == .main { - return MainActor.sharedUnownedExecutor - } else { - return .init(complexEquality: self) - } - } - - func makeContext() -> ExecutorContext { - .init(queue: queue, source: sourceRef.source) - } - - // This is the main of runloop thread - //ExecutorContext contains JobQueue and RunLoopSource - static func controlRunLoop(context:ExecutorContext) { - guard Self.myLocal == nil, - RunLoop.current.currentMode == nil, - RunLoop.current != RunLoop.main - else { return } - defer { - while true { - let (exchanged, _ ) = context.queue.sanityCheck.compareExchange(expected: false, desired: true, ordering: .releasing) - if exchanged { - break - } - } - // last safety check - Self.$myLocal.withValue(context) { - Self.processEvents() - } - } - Self.$myLocal.withValue(context) { - CFRunLoopAddSource(CFRunLoopGetCurrent(), context.source, .commonModes) - while CFRunLoopSourceIsValid(context.source) { - let passed = RunLoop.current.run(mode: .default, before: .distantFuture) - if !passed { - break - } - } - } - - } - - static func pump( - queue:LockFreeQueue - ) { - let estimatedCap = queue.estimatedLength() - var buffer = Heap(minimumCapacity: estimatedCap) - buffer.reserveCapacity(estimatedCap) - print("pump and nil", Self.myLocal == nil) - var id:UInt64 = .max - while let block = queue.dequeue() { - defer { id -= 1 } - let ref = JobBlock(id: id, job: block) - buffer.insert(ref) - } - while let job = buffer.popMax() { - job.jobImp.runSynchronously(on: job.executor) - } - - } - - static func processEvents() { - if let queue = Self.myLocal?.queue { - pump(queue: queue) - } - - } - -} - -import os - -@available(macOS 14.0, iOS 17.0, watchOS 10.0, tvOS 17.0, *) -final class RunLoopSourceRef: @unchecked Sendable { - - private(set) var source:CFRunLoopSource! - let queue:LockFreeQueue! - - init(null:()) { - self.source = nil - self.queue = nil - } - - init(ref: LockFreeQueue, nested:Bool) { - if nested { - queue = ref - } else { - queue = nil - } - var context = CFRunLoopSourceContext() - context.version = 0 - context.info = Unmanaged.passUnretained(self).toOpaque() - if nested { - context.perform = { - let ref: RunLoopSourceRef = Unmanaged.fromOpaque($0!).takeUnretainedValue() - RunLoopPriorityExecutor.pump(queue: ref.queue) - } - context.cancel = { info, runLoop, mode in - let ref: RunLoopSourceRef = Unmanaged.fromOpaque(info!).takeUnretainedValue() - if let runLoop { - CFRunLoopStop(runLoop) - } - CFRunLoopPerformBlock(runLoop, CFRunLoopMode.commonModes.rawValue) { - RunLoopPriorityExecutor.pump(queue: ref.queue) - } - } - } else { - context.cancel = { info, runLoop, mode in - if let runLoop { - CFRunLoopStop(runLoop) - } - } - context.perform = { _ in - RunLoopPriorityExecutor.processEvents() - } - } - - context.copyDescription = { _ in - let description = "RunLoopPriorityExecutor-Source" - return .passRetained(description as CFString) - } - self.source = CFRunLoopSourceCreate(nil, 0, &context) - } - - - deinit { - if let source { - CFRunLoopSourceInvalidate(source) - } - } - -} - - -/// Transform current Thread as the RunLoop Executor, and run the runLoop -/// -/// -/// Actual behavior depends on the current RunLoop state. -/// -/// 1) called from existing `RunLoopPriorityExecutor` thread. -/// new executor is connected to the cached component and return, executor lifetime is shared with previous executor. -/// This does not runs runloop. RunLoop is deactivated when this, and all previous executor is dead. -/// 2) called from `MainThread` -/// create dummy executor and return. Dummy executor dispatch all the jobs to the `MainActor` -/// 3) called from active runloop Thread. (someone is already controlling the RunLoop) -/// create unoptimized executor and return. This executor does not controls the runLoop. Existing RunLoop owner has the resposibility to keep runLoop alive, otherwise enqued Job would leak. -/// This executor does not support JobPriority. And simply create NSObject and schedule the block -/// -/// 4) called from fresh Thread (no one is running runLoop) -/// create optimized executor, call `setupHandle` than controls the RunLoop of current Thread. -/// This function does not return, until executor is dead. So, `setupHandle` is the entrypoint of using the executor. -/// This executor does recognize priority -/// - Parameter setupHandle: called right before runLoop runs, runloop is active until executor is dead. this block is called exactly once. -@available(macOS 14.0, iOS 17.0, watchOS 10.0, tvOS 17.0, *) -package -func executeRunloop( - setupHandle: (RunLoopPriorityExecutor) -> Void -) { - // if nested -> already running runloop 1) we are controlling the runloop 2) somebody is controlling runloop - // 1) our executor is already controlling thread and we are calling it at the same thread - // -> do not perform any nested run - // 2) somebody is already taking control of this thread - // if mainloop -> no-op redirect to mainactor - // if no-runloop -> controll it! - guard !Thread.isMainThread else { - // we don't run main runLoop since MainActor is the way togo - // `RunLoopPriorityExecutor` redirect every thing back to the MainActor - setupHandle(RunLoopPriorityExecutor(nested: ())) - return - } - let existing = RunLoopPriorityExecutor.myLocal - if let existing, CFRunLoopContainsSource(CFRunLoopGetCurrent(), existing.source, .commonModes) { - // nested runloop which we are taking full control or main runloop - // connect to the existing task-queue and return - setupHandle(RunLoopPriorityExecutor.init()) - return - } - - - // complex case - if RunLoop.current.currentMode != nil { - // we are in the active runloop which someone else is taking the full control - // nested runloop is not an ideal case - - // configure runloop source and attach it maybe? - // but in that case tasklocal is not visible in same thread ... - // lets fall back to unoptimized way, stashing every job as Clousre block, ignoring priority - // this RunLoopExecutor never controls the runloop - let executor = RunLoopPriorityExecutor(nested: ()) - setupHandle(executor) - return - } - let context:RunLoopPriorityExecutor.ExecutorContext - do { - let executor = RunLoopPriorityExecutor() - context = executor.makeContext() - setupHandle(consume executor) - } - RunLoopPriorityExecutor.controlRunLoop(context: context) -} - - - - -struct JobBlock: Hashable, Comparable { - - let id:UInt64 - let priority:UInt8 - let jobImp:UnownedJob - let executor:UnownedSerialExecutor - - @available(macOS 14.0, *) - init(id: UInt64, job: consuming ExecutorJobContext) { - self.id = id - self.priority = job.job.priority.rawValue - self.executor = job.executor - self.jobImp = UnownedJob(job.job) - - } - - init(id: UInt64, jobRef: UnownedJob, executor:UnownedSerialExecutor) { - self.id = id - self.priority = 0 - self.jobImp = jobRef - self.executor = executor - } - - func hash(into hasher: inout Hasher) { - hasher.combine(priority) - } - - static func == (lhs: Self, rhs: Self) -> Bool { - lhs.id == rhs.id - } - - static func < (lhs: Self, rhs: Self) -> Bool { - if lhs.priority == rhs.priority { - return lhs.id < rhs.id - } - return lhs.priority < rhs.priority - } - - static func > (lhs:Self, rhs:Self) -> Bool { - if lhs.priority == rhs.priority { - return lhs.id > rhs.id - } - return lhs.priority > rhs.priority - } - -} diff --git a/Sources/Tetra/Concurrency/JobBlock.swift b/Sources/Tetra/Concurrency/JobBlock.swift new file mode 100644 index 0000000..d89ccdf --- /dev/null +++ b/Sources/Tetra/Concurrency/JobBlock.swift @@ -0,0 +1,66 @@ +// +// JobBlock.swift +// +// +// Created by 박병관 on 6/30/24. +// + +@usableFromInline +struct JobBlock: Hashable, Comparable, Sendable { + + @usableFromInline + let id:Int + @usableFromInline + let jobImp:UnownedJob + + @usableFromInline + var priority:UInt8 { + if #available(macOS 14.0, iOS 17.0, watchOS 10.0, tvOS 17.0, *) { + jobImp.priority.rawValue + } else { + 0 + } + } + + @usableFromInline + @available(macOS 14.0, *) + init(id: Int, job: consuming ExecutorJob) { + self.id = id + self.jobImp = UnownedJob(job) + + } + + @usableFromInline + init(id: Int, jobRef: UnownedJob) { + self.id = id + self.jobImp = jobRef + } + + @usableFromInline + func hash(into hasher: inout Hasher) { + hasher.combine(priority) + } + + @usableFromInline + static func == (lhs: Self, rhs: Self) -> Bool { + lhs.id == rhs.id + } + + @usableFromInline + static func < (lhs: Self, rhs: Self) -> Bool { + if lhs.priority == rhs.priority { + return lhs.id < rhs.id + } + return lhs.priority < rhs.priority + } + + @usableFromInline + static func > (lhs:Self, rhs:Self) -> Bool { + if lhs.priority == rhs.priority { + return lhs.id > rhs.id + } + return lhs.priority > rhs.priority + } + +} + diff --git a/Sources/Tetra/Concurrency/PriorityRunLoop.swift b/Sources/Tetra/Concurrency/PriorityRunLoop.swift new file mode 100644 index 0000000..163691e --- /dev/null +++ b/Sources/Tetra/Concurrency/PriorityRunLoop.swift @@ -0,0 +1,346 @@ +// +// PriorityRunLoop2.swift +// +// +// Created by 박병관 on 6/30/24. +// + +import HeapModule +import Foundation +import CoreFoundation +public import CriticalSection + +public final class RunLoopPriorityExecutor { + + @usableFromInline + internal let heaps: some UnfairStateLock> = createCheckedStateLock(checkedState: .init()) + + // cache for faster comparsion, RunLoop comparsion trigger creating extra RunLoop + // I'm not sure storing pthread_t as bitpattern is a good idea + @usableFromInline + let threadId:Int + + @usableFromInline + nonisolated(unsafe) + internal let looper:RunLoop + + @usableFromInline + internal var cfRef:CFRunLoop { + looper.getCFRunLoop() + } + + @usableFromInline + nonisolated(unsafe) + internal let source:CFRunLoopSource + + + @inlinable + internal init() { + + let (buffer, source) = Self.sharedSource + self.looper = .current + self.threadId = .init(bitPattern: pthread_self()) + self.source = source + buffer.header = .init(value: self) + if Thread.isMainThread { + CFRunLoopSourceInvalidate(source) + return + } else { + CFRunLoopAddSource(cfRef, source, .commonModes) + } + } + + @inlinable + deinit { + let key = ObjectIdentifier(looper) + let _ = Self.cache.withLockUnchecked{ + $0.removeValue(forKey: key) + } + CFRunLoopSourceInvalidate(source) + if inRunLoop, looper.currentMode != nil { + var arrays = CFRunLoopCopyAllModes(looper.getCFRunLoop()) as! [CFString] + arrays.append(CFRunLoopMode.commonModes.rawValue) + let timer = CFRunLoopTimerCreate(nil, CFAbsoluteTimeGetCurrent(), 0, 0, 0, nil, nil) + arrays.forEach{ + CFRunLoopAddTimer(cfRef, timer, .init($0)) + } + } else { + CFRunLoopWakeUp(looper.getCFRunLoop()) + } + + } + + @inlinable + nonisolated + public var inRunLoop: Bool { + let this = pthread_t(bitPattern: threadId) + let current = pthread_self() + let check = pthread_equal(this, current) + + return check != 0 + } + + // if you access the runLoop while not isolated, it will trigger assert + @inlinable + public var runLoop: RunLoop { + assert(inRunLoop, "can not access \(#function) outside of isolation") + return looper + } + + @usableFromInline + nonisolated + internal func schedule(_ job: consuming UnownedJob) { + if CFEqual(cfRef, CFRunLoopGetMain()) { + MainActor.shared.enqueue(job) + return + } + heaps.withLock{ [job] in + // lastest has the lower id which results lower priority + let id = -$0.count + $0.insert(.init(id: id, jobRef: consume job)) + } + CFRunLoopSourceSignal(source) + if !inRunLoop { + CFRunLoopWakeUp(cfRef) + } + } + + @usableFromInline + internal func evaluateCommonModes() -> [RunLoop.Mode] { + var arrys = [String]() + withUnsafeMutablePointer(to: &arrys) { ptr in + var context = CFRunLoopSourceContext() + context.info = .init(ptr) + context.schedule = { info, _ , mode in + let arrayPtr = info!.assumingMemoryBound(to: [String].self) + arrayPtr.pointee.append(mode!.rawValue as String) + } + let emptySource = CFRunLoopSourceCreate(nil, 0, &context)! + CFRunLoopAddSource(cfRef, emptySource, .commonModes) + CFRunLoopSourceInvalidate(source) + } + return arrys.map{ RunLoop.Mode($0) } + } + + @inlinable + nonisolated + public func add(_ mode:RunLoop.Mode) { + if CFEqual(cfRef, CFRunLoopGetMain()) { + return + } + CFRunLoopAddSource(cfRef, source, .init(mode.rawValue as CFString)) + } + + // you can not remove common mode + @inlinable + nonisolated + public func remove(_ mode:RunLoop.Mode) { + if CFEqual(cfRef, CFRunLoopGetMain()) { + return + } + if mode == .common || mode == .default || evaluateCommonModes().contains(mode) { + return + } + CFRunLoopRemoveSource(cfRef, source, .init(mode.rawValue as CFString)) + } + +} + +extension RunLoopPriorityExecutor: SerialExecutor { + + @inlinable + nonisolated + public func isSameExclusiveExecutionContext(other: borrowing RunLoopPriorityExecutor) -> Bool { + return CFEqual(cfRef, other.cfRef) + } + + @inlinable + nonisolated + public func asUnownedSerialExecutor() -> UnownedSerialExecutor { + if CFEqual(cfRef, CFRunLoopGetMain()) { + return MainActor.sharedUnownedExecutor + } else if #available(macOS 14.0, iOS 17.0, watchOS 10.0, tvOS 17.0, *) { + return .init(complexEquality: self) + } else { + return .init(ordinary: self) + } + } + + @inlinable + nonisolated + public func checkIsolated() { + precondition(CFEqual(cfRef, CFRunLoopGetCurrent()), "Unexpected isolation context, expected to be executing on \(cfRef)") + } + + @available(macOS 14.0, iOS 17.0, watchOS 10.0, tvOS 17.0, *) + @inlinable + nonisolated + public func enqueue(_ job: consuming ExecutorJob) { + schedule(.init(job)) + } + + @inlinable + nonisolated + public func enqueue(_ job: UnownedJob) { + schedule(job) + } + +} + +// MARK: Concurrency TaskExecutor +@available(macOS 15.0, iOS 18.0, watchOS 11.0, tvOS 18.0, visionOS 2.0, *) +extension RunLoopPriorityExecutor: TaskExecutor { } + + +extension RunLoopPriorityExecutor { + + @usableFromInline + struct Boxed { + @usableFromInline + unowned let value:RunLoopPriorityExecutor? + + @usableFromInline + init(value: RunLoopPriorityExecutor?) { + self.value = value + } + + } + + // we rarely access this state, only when creating and deinitialzing + // so it is reasonable to use global lock rather than thread local + // since thread local keeps stored reference alive and + // user can call this method from thread pool( Concurrency, libdispatch) + @usableFromInline + static let cache: some UnfairStateLock<[ObjectIdentifier: Boxed]> = createUncheckedStateLock(uncheckedState: [:]) + + @usableFromInline + static var sharedSource:(ManagedBuffer, CFRunLoopSource) { + let buffer = ManagedBuffer.create(minimumCapacity: 0) { _ in + return .init(value: nil) + } + var context = CFRunLoopSourceContext() + context.version = 0 + context.info = Unmanaged.passUnretained(buffer).toOpaque() + context.retain = { + let ptr = Unmanaged.fromOpaque($0!).retain().toOpaque() + return .init(ptr) + } + context.release = { + Unmanaged.fromOpaque($0!).release() + } + context.perform = { + var heap:Heap + let execute:(@Sendable (consuming UnownedJob) ->Void) + do { + let boxed = Unmanaged>.fromOpaque($0!) + .takeUnretainedValue() + guard let ref = boxed.header.value else { return } + heap = ref.heaps.withLock{ + + var old = Heap() + swap(&$0, &old) + return old + } + let serial = ref.asUnownedSerialExecutor() + if #available(macOS 15.0, iOS 18.0, watchOS 11.0, tvOS 18.0, visionOS 2.0, *) { + let task = ref.asUnownedTaskExecutor() + execute = { [serial, task] in + $0.runSynchronously(isolatedTo: serial, taskExecutor: task) + } + } else { + execute = { [serial] in + $0.runSynchronously(on: serial) + } + } + } + while let block = heap.popMax() { + execute(block.jobImp) + } + } + + context.copyDescription = { null in + let description = "RunLoopPExecutor" + return .passRetained(description as CFString) + } + return (buffer, CFRunLoopSourceCreate(nil, 0, &context)) + } +} + +extension RunLoopPriorityExecutor { + + + /// Transform current Thread as the RunLoop Executor, and run the runLoop + /// + /// + /// Actual behavior depends on the current RunLoop state. + /// + /// 1) called from existing `RunLoopPriorityExecutor` thread. + /// existing Executor is returned + /// This does not runs runloop. RunLoop is deactivated when this, and all previous executor is dead. + /// 2) called from `MainThread` + /// create dummy executor and return. Dummy executor dispatch all the jobs to the `MainActor` + /// 3) called from active runloop Thread. (someone is already controlling the RunLoop) + /// create executor and return. This executor does not controls the runLoop. Existing RunLoop owner has the resposibility to keep runLoop alive, otherwise enqued Job would leak. + /// + /// 4) called from fresh Thread (no one is running runLoop) + /// create optimized executor, call `setupHandle` than controls the RunLoop of current Thread. + /// This function does not return, until executor is dead. So,`setupHandle` is the entrypoint of using the executor. + /// - Parameter setupHandle: called right before runLoop runs, runloop is active until executor is dead. this block is called exactly once. + /// - Important: when using it with existing active runLoop, keep runloop alive until Executor is gracefully deinitialized + @inlinable + public static func getOrCreate(_ block: (consuming Self) -> Void) { + if Thread.isMainThread { + let executor = Self() + (consume block)(executor) + return + } + let runLoop = RunLoop.current + let key = ObjectIdentifier(runLoop) + if let existing = cache.withLockUnchecked({ + $0[key]?.value + }) { + (consume block)(existing as! Self) + return + } + let source:CFRunLoopSource + do { + let executor = Self() + Self.cache.withLockUnchecked{ + $0[key] = .init(value: executor) + } + source = executor.source + (consume block)(executor) + } + while CFRunLoopSourceIsValid(source), RunLoop.current.run(mode: .default, before: .distantFuture) { + + } + } + + +} + + + + + + +/* + + Executor -> contains runLoop and Context + Context has JobQueue + One or More Executor can reference the same Context And RunLoop + if Executor reference the same RunLoop than the Context also must be smae + Context has connection to RunLoop Source and Observer + Context owns the Source and Observer, + Source has no info about the context + Observer has a weak reference to the Context + when Context is destroyed (no executor is alived), it stops the Source, but do not destroy RunLoop Observer + RunLoopObserver it self checks the weak reference of Context and do its clean up + + thread_local -> store + + + + + */ + diff --git a/Tests/RunLoopExecutorTest/Tests.swift b/Tests/RunLoopExecutorTest/Tests.swift deleted file mode 100644 index b567989..0000000 --- a/Tests/RunLoopExecutorTest/Tests.swift +++ /dev/null @@ -1,60 +0,0 @@ -// -// Tests.swift -// -// -// Created by 박병관 on 6/30/24. -// - -@testable import CriticalSection -import Testing -import Foundation - -@Suite -struct Tests { - - - @Test - func evaluate() async { - if #available(macOS 14.0, *) { - let myActor = await RunLoopActor() - let block = { (act: isolated RunLoopActor) in - - #expect(RunLoopPriorityExecutor.myLocal != nil) - - } - await block(myActor) - } else { - // Fallback on earlier versions - } - } - -} - -@available(macOS 14.0, iOS 17.0, watchOS 10.0, tvOS 17.0, *) - -actor RunLoopActor { - - let executor:RunLoopPriorityExecutor - - init() async { - - let ref:RunLoopPriorityExecutor = await withUnsafeContinuation { continuation in - - let th = Thread{ - executeRunloop { - continuation.resume(returning: $0) - } - - } - th.qualityOfService = .default - th.start() - } - - self.executor = ref - } - nonisolated var unownedExecutor: UnownedSerialExecutor { - executor.asUnownedSerialExecutor() - } - - -} From d0b4e52117760de5c6ccff7c7e50011d1054f600 Mon Sep 17 00:00:00 2001 From: park-byeong-gwan Date: Sat, 6 Jul 2024 22:58:52 +0900 Subject: [PATCH 39/63] implement qos elevation of PriorityRunLoop. remove legacy runloop schedular --- Sources/Tetra/Combine/RunLoopScheduler.swift | 304 ---------------- Sources/Tetra/Concurrency/JobBlock.swift | 3 + .../Tetra/Concurrency/PriorityRunLoop.swift | 332 ++++++++++-------- .../Tetra/Concurrency/RunLoopExecutor.swift | 150 -------- Sources/Tetra/Concurrency/TaskQos.swift | 103 ++++++ .../Tetra/Foundation/RunLoopSourceBlock.swift | 85 +++++ Tests/TetraTests/RunLoopSchedulerTests.swift | 75 ---- 7 files changed, 385 insertions(+), 667 deletions(-) delete mode 100644 Sources/Tetra/Combine/RunLoopScheduler.swift delete mode 100644 Sources/Tetra/Concurrency/RunLoopExecutor.swift create mode 100644 Sources/Tetra/Concurrency/TaskQos.swift create mode 100644 Sources/Tetra/Foundation/RunLoopSourceBlock.swift delete mode 100644 Tests/TetraTests/RunLoopSchedulerTests.swift diff --git a/Sources/Tetra/Combine/RunLoopScheduler.swift b/Sources/Tetra/Combine/RunLoopScheduler.swift deleted file mode 100644 index 93ca6b8..0000000 --- a/Sources/Tetra/Combine/RunLoopScheduler.swift +++ /dev/null @@ -1,304 +0,0 @@ -// -// RunLoopScheduler.swift -// -// -// Created by pbk on 2022/12/10. -// - -@preconcurrency import Foundation -import Dispatch -import os -import Combine - -/** - RunLoopScheduler suitable for background runLoop - - this class runs RunLoop indefinitely in default Mode, until deinitialized. - - #1 Nested RunLoop - - It's not a good idea to create nested RunLoop inside RunLoopScheduler but if you do need that, keep strong reference to the Scheduler. - - - - important: Memory leaks found in instrument from this class are not acually leaked and they will be released as soon as `RunLoopScheduler`'s `Thread` terminate. - */ -public final class RunLoopScheduler: Scheduler, @unchecked Sendable, Hashable { - - public static func == (lhs: RunLoopScheduler, rhs: RunLoopScheduler) -> Bool { - lhs.cfRunLoop == rhs.cfRunLoop && lhs.source == rhs.source - } - - public func hash(into hasher: inout Hasher) { - hasher.combine(cfRunLoop) - hasher.combine(source) - } - - - public typealias SchedulerTimeType = RunLoop.SchedulerTimeType - public typealias SchedulerOptions = Never - - private let source:CFRunLoopSource - nonisolated - public let cfRunLoop:CFRunLoop - - nonisolated - public let config:Configuration - - deinit { - CFRunLoopSourceInvalidate(source) - CFRunLoopWakeUp(cfRunLoop) - } - - public init(async: Void = (), config: Configuration = .init()) async { - var nullContext = CFRunLoopSourceContext() - nullContext.version = 0 - nullContext.cancel = { _, runLoop, _ in - guard let runLoop else { return } - CFRunLoopStop(runLoop) - } - nullContext.copyDescription = { _ in - .passRetained("RunLoopScheduler Default Source" as CFString) - } - let emptySource = CFRunLoopSourceCreate(nil, 0, &nullContext).unsafelyUnwrapped - let runLoop = await withUnsafeContinuation{ continuation in - let runner = RunLoopRunner(emptySource) { - continuation.resume(returning: $0) - } - let workerThread = Thread(target: runner, selector: #selector(runner.main), object: nil) - workerThread.qualityOfService = config.qos - workerThread.start() - } - self.cfRunLoop = runLoop - self.source = emptySource - self.config = config - } - - /** - Create Scheduler in sync - - Create new Thread and run the CFRunLoop of that Thread. This initializer blocks the current thread until the Scheduler is ready. - */ - public init(sync: Void = (), config: Configuration = .init()) { - let reference = UnsafeMutablePointer.allocate(capacity: 1) - defer { reference.deallocate() } - var nullContext = CFRunLoopSourceContext() - nullContext.cancel = { _, runLoop, _ in - guard let runLoop else { return } - CFRunLoopStop(runLoop) - } - nullContext.copyDescription = { _ in - .passRetained("RunLoopScheduler Default Source" as CFString) - } - nullContext.version = 0 - let emptySource = CFRunLoopSourceCreate(nil, 0, &nullContext).unsafelyUnwrapped - - let condition = NSCondition() - let runner = RunLoopRunner(emptySource) { - reference.initialize(to: $0) - condition.withLock { - condition.signal() - } - } - let workerThread = Thread(target: runner, selector: #selector(runner.main), object: nil) - workerThread.qualityOfService = config.qos - let runLoop = condition.withLock { - workerThread.start() - condition.wait() - return reference.move() - } - self.cfRunLoop = runLoop - self.source = emptySource - self.config = config - } - - @usableFromInline - struct Block: @unchecked Sendable { - - let block: () -> Void - - @usableFromInline - func callAsFunction() { - block() - } - - @usableFromInline - init(block: @escaping () -> Void) { - self.block = block - } - - } - - @inlinable - nonisolated - public func schedule( - after date: SchedulerTimeType, - interval: SchedulerTimeType.Stride, - tolerance: SchedulerTimeType.Stride, - options: SchedulerOptions?, - _ action: @escaping () -> Void - ) -> Cancellable { - let timer:Timer - let block = Block(block: action) - if config.keepAliveUntilFinish { - let observer = createRetainToken() - timer = .init(fire: date.date, interval: interval.timeInterval, repeats: true) { _ in - CFRunLoopObserverInvalidate(observer) - block() - } - } else { - timer = .init(fire: date.date, interval: interval.timeInterval, repeats: true) { _ in block() } - } - timer.tolerance = tolerance.timeInterval - let cfTimer = timer as CFRunLoopTimer - CFRunLoopAddTimer(cfRunLoop, cfTimer, .commonModes) - return AnyCancellable{ - CFRunLoopTimerInvalidate(cfTimer) - } - } - - @inlinable - nonisolated - public func schedule( - after date: SchedulerTimeType, - tolerance: SchedulerTimeType.Stride, - options: SchedulerOptions?, - _ action: @escaping () -> Void - ) { - let timer:Timer - let block = Block(block: action) - if config.keepAliveUntilFinish { - let observer = createRetainToken() - timer = .init(fire: date.date, interval: 0, repeats: false) { _ in - CFRunLoopObserverInvalidate(observer) - block() - } - } else { - timer = .init(fire: date.date, interval: 0, repeats: false) { _ in block() } - } - timer.tolerance = tolerance.timeInterval - CFRunLoopAddTimer(cfRunLoop, timer as CFRunLoopTimer, .commonModes) - } - - @inlinable - nonisolated - public func schedule(options: SchedulerOptions?, _ action: @escaping () -> Void) { - if config.keepAliveUntilFinish { - let observer = createRetainToken() - CFRunLoopPerformBlock(cfRunLoop, CFRunLoopMode.commonModes.rawValue) { - CFRunLoopObserverInvalidate(observer) - action() - } - } else { - CFRunLoopPerformBlock(cfRunLoop, CFRunLoopMode.commonModes.rawValue, action) - } - if CFRunLoopIsWaiting(cfRunLoop) { - CFRunLoopWakeUp(cfRunLoop) - } - } - - nonisolated - public var now: SchedulerTimeType { .init(Date()) } - - nonisolated - public var minimumTolerance: SchedulerTimeType.Stride { 0.0 } - - public func scheduleTask(_ block: @escaping () throws(Failure) -> T) async throws(Failure) -> T { - let result:Result = await withUnsafeContinuation{ continuation in - CFRunLoopPerformBlock(cfRunLoop, CFRunLoopMode.commonModes.rawValue) { - let result = Result { () throws(Failure) in - return try block() - } - continuation.resume(returning: result) - } - if CFRunLoopIsWaiting(cfRunLoop) { - CFRunLoopWakeUp(cfRunLoop) - } - } - return try result.get() - } - - @usableFromInline - internal func createRetainToken() -> CFRunLoopObserver { - var context = CFRunLoopObserverContext(version: 0, info: Unmanaged.passUnretained(self).toOpaque()) { - UnsafeRawPointer(Unmanaged.fromOpaque($0.unsafelyUnwrapped).retain().toOpaque()) - } release: { - Unmanaged.fromOpaque($0.unsafelyUnwrapped).release() - } copyDescription: { - .passRetained(String(describing: Unmanaged.fromOpaque($0.unsafelyUnwrapped).takeUnretainedValue()) as CFString) - } - - return CFRunLoopObserverCreate(nil, 0, false, 0, nil, &context) - } - - private struct RunnerParameter { - let source:CFRunLoopSource - let completion:(CFRunLoop) -> () - } - - private final class RunLoopRunner { - - private var parameter:RunnerParameter? - - init(_ source:CFRunLoopSource, completionHandler: @escaping (CFRunLoop) -> ()) { - self.parameter = .init(source: source, completion: completionHandler) - } - - @objc - func main() { - let emptySource:CFRunLoopSource - if let parameter { - emptySource = parameter.source - parameter.completion(CFRunLoopGetCurrent()) - self.parameter = nil - } else { - return - } - Thread.setThreadPriority(0) - let interrupter = createNestedLoopInterrupter(emptySource) - CFRunLoopAddSource(CFRunLoopGetCurrent(), emptySource, .defaultMode) - CFRunLoopAddObserver(CFRunLoopGetCurrent(), interrupter, .commonModes) - defer { CFRunLoopObserverInvalidate(interrupter) } - while - CFRunLoopSourceIsValid(emptySource), - RunLoop.current.run(mode: .default, before: .distantFuture) - { } - } - - } - - @usableFromInline - static func createNestedLoopInterrupter(_ emptySource:CFRunLoopSource) -> CFRunLoopObserver { - var context = CFRunLoopObserverContext(version: 0, info: Unmanaged.passUnretained(emptySource).toOpaque()) { .init(Unmanaged.fromOpaque($0.unsafelyUnwrapped).retain().toOpaque()) - } release: { Unmanaged.fromOpaque($0.unsafelyUnwrapped).release() - } copyDescription: { _ in - .passRetained("RunLoopScheduler.NestedRunLoop.Interrupter" as CFString) - } - return CFRunLoopObserverCreate(nil, CFRunLoopActivity.exit.union([.beforeTimers, .beforeSources, .beforeWaiting]).rawValue, true, 0, { _, _, ref in - let source = Unmanaged.fromOpaque(ref.unsafelyUnwrapped).takeUnretainedValue() - if CFRunLoopSourceIsValid(source) { - return - } else { - CFRunLoopStop(CFRunLoopGetCurrent()) - } - }, &context) - } - -} - - -public extension RunLoopScheduler { - - struct Configuration: Hashable, Sendable { - - public var qos:QualityOfService = .default - /** whether to keep the scheduler alive until submitted tasks are finished */ - public var keepAliveUntilFinish = true - - @inlinable - public init(qos: QualityOfService = .default, keepAliveUntilFinish: Bool = true) { - self.qos = qos - self.keepAliveUntilFinish = keepAliveUntilFinish - } - } - -} diff --git a/Sources/Tetra/Concurrency/JobBlock.swift b/Sources/Tetra/Concurrency/JobBlock.swift index d89ccdf..6890d86 100644 --- a/Sources/Tetra/Concurrency/JobBlock.swift +++ b/Sources/Tetra/Concurrency/JobBlock.swift @@ -4,6 +4,7 @@ // // Created by 박병관 on 6/30/24. // +import Darwin @usableFromInline struct JobBlock: Hashable, Comparable, Sendable { @@ -12,6 +13,8 @@ struct JobBlock: Hashable, Comparable, Sendable { let id:Int @usableFromInline let jobImp:UnownedJob + nonisolated(unsafe) + var token:pthread_override_t? @usableFromInline var priority:UInt8 { diff --git a/Sources/Tetra/Concurrency/PriorityRunLoop.swift b/Sources/Tetra/Concurrency/PriorityRunLoop.swift index 163691e..9f846c7 100644 --- a/Sources/Tetra/Concurrency/PriorityRunLoop.swift +++ b/Sources/Tetra/Concurrency/PriorityRunLoop.swift @@ -10,138 +10,236 @@ import Foundation import CoreFoundation public import CriticalSection -public final class RunLoopPriorityExecutor { +@usableFromInline +struct RunLoopPriorityQueue: ~Copyable, Sendable { @usableFromInline internal let heaps: some UnfairStateLock> = createCheckedStateLock(checkedState: .init()) - - // cache for faster comparsion, RunLoop comparsion trigger creating extra RunLoop - // I'm not sure storing pthread_t as bitpattern is a good idea + @usableFromInline - let threadId:Int + nonisolated(unsafe) + internal let runLoop:CFRunLoop @usableFromInline nonisolated(unsafe) - internal let looper:RunLoop - - @usableFromInline - internal var cfRef:CFRunLoop { - looper.getCFRunLoop() - } + internal let source:CFRunLoopSource @usableFromInline + internal let isMain:Bool + nonisolated(unsafe) - internal let source:CFRunLoopSource + internal let thread:pthread_t - - @inlinable - internal init() { - - let (buffer, source) = Self.sharedSource - self.looper = .current - self.threadId = .init(bitPattern: pthread_self()) - self.source = source - buffer.header = .init(value: self) - if Thread.isMainThread { + @usableFromInline + init( + runLoop: RunLoop, + threadId: pthread_t, + execute: @escaping (consuming UnownedJob) -> Void + ) { + self.runLoop = runLoop.getCFRunLoop() + self.thread = threadId + self.isMain = CFEqual(runLoop, CFRunLoopGetMain()) + if isMain { + self.source = CFRunLoopSourceCreate(nil, 0, nil) CFRunLoopSourceInvalidate(source) - return } else { - CFRunLoopAddSource(cfRef, source, .commonModes) + self.source = RunLoopSourceCreateWithHandler { [heaps] in + + guard $0 == .perform else { return } + + var queue = heaps.withLock{ + var next = Heap() + swap(&next, &$0) + return next + } + while let block = queue.popMax() { + defer { + if let ref = block.token { + pthread_override_qos_class_end_np(ref) + } + } + let job = block.jobImp + execute(job) + } + } + CFRunLoopAddSource(self.runLoop, source, .commonModes) } } @inlinable deinit { - let key = ObjectIdentifier(looper) - let _ = Self.cache.withLockUnchecked{ - $0.removeValue(forKey: key) + if isMain { + return } CFRunLoopSourceInvalidate(source) - if inRunLoop, looper.currentMode != nil { - var arrays = CFRunLoopCopyAllModes(looper.getCFRunLoop()) as! [CFString] + if CFRunLoopCopyCurrentMode(runLoop) != nil{ + var arrays = CFRunLoopCopyAllModes(runLoop) as! [CFString] arrays.append(CFRunLoopMode.commonModes.rawValue) let timer = CFRunLoopTimerCreate(nil, CFAbsoluteTimeGetCurrent(), 0, 0, 0, nil, nil) arrays.forEach{ - CFRunLoopAddTimer(cfRef, timer, .init($0)) + CFRunLoopAddTimer(runLoop, timer, .init($0)) } } else { - CFRunLoopWakeUp(looper.getCFRunLoop()) + CFRunLoopWakeUp(runLoop) + } + heaps.withLock{ + precondition($0.count == 0) } - } - @inlinable - nonisolated - public var inRunLoop: Bool { - let this = pthread_t(bitPattern: threadId) - let current = pthread_self() - let check = pthread_equal(this, current) - - return check != 0 + @usableFromInline + internal func evaluateCommonModes() -> [CFRunLoopMode] { + var arrys = [CFRunLoopMode]() + withUnsafeMutablePointer(to: &arrys) { ptr in + var context = CFRunLoopSourceContext() + context.info = .init(ptr) + context.schedule = { info, _ , mode in + let arrayPtr = info!.assumingMemoryBound(to: [CFRunLoopMode].self) + arrayPtr.pointee.append(mode!) + } + let emptySource = CFRunLoopSourceCreate(nil, 0, &context)! + CFRunLoopAddSource(runLoop, emptySource, .commonModes) + CFRunLoopSourceInvalidate(source) + } + return arrys } - // if you access the runLoop while not isolated, it will trigger assert - @inlinable - public var runLoop: RunLoop { - assert(inRunLoop, "can not access \(#function) outside of isolation") - return looper - } @usableFromInline nonisolated internal func schedule(_ job: consuming UnownedJob) { - if CFEqual(cfRef, CFRunLoopGetMain()) { + if isMain { MainActor.shared.enqueue(job) return } - heaps.withLock{ [job] in + + let override: pthread_override_t? + + if #available(macOS 14.0, iOS 17.0, watchOS 10.0, tvOS 17.0, *) { + let qos = job.priority.evaluateQos() + if qos.qosClass == .unspecified { + override = nil + } else { + override = pthread_override_qos_class_start_np(thread, qos.qosClass.rawValue, Int32(qos.relativePriority)) + } + } else { + override = nil + } + + + heaps.withLockUnchecked{ [job] in // lastest has the lower id which results lower priority let id = -$0.count - $0.insert(.init(id: id, jobRef: consume job)) + var item = JobBlock(id: id, jobRef: consume job) + item.token = override + $0.insert(item) } CFRunLoopSourceSignal(source) - if !inRunLoop { - CFRunLoopWakeUp(cfRef) + } + + @inlinable + nonisolated + internal func add(_ mode:CFRunLoopMode) { + if isMain { + return } + CFRunLoopAddSource(runLoop, source, mode) } + // you can not remove common mode + @inlinable + nonisolated + internal func remove(_ mode:CFRunLoopMode) { + if isMain { + return + } + if mode == .commonModes || mode == .defaultMode || evaluateCommonModes().contains(mode) { + return + } + CFRunLoopRemoveSource(runLoop, source, mode) + } + +} + + +public final class RunLoopPriorityExecutor { + + + // cache for faster comparsion, RunLoop comparsion trigger creating extra RunLoop + // I'm not sure storing pthread_t as bitpattern is a good idea @usableFromInline - internal func evaluateCommonModes() -> [RunLoop.Mode] { - var arrys = [String]() - withUnsafeMutablePointer(to: &arrys) { ptr in - var context = CFRunLoopSourceContext() - context.info = .init(ptr) - context.schedule = { info, _ , mode in - let arrayPtr = info!.assumingMemoryBound(to: [String].self) - arrayPtr.pointee.append(mode!.rawValue as String) + let threadId:Int + + @usableFromInline + internal let queue:RunLoopPriorityQueue + + @usableFromInline + nonisolated(unsafe) + internal let _runLoop:RunLoop + + @inlinable + internal init() { + self._runLoop = .current + var serialRef: UnownedSerialExecutor! = nil + self.threadId = .init(bitPattern: pthread_self()) + if #available(macOS 15.0, iOS 18.0, watchOS 11.0, tvOS 18.0, visionOS 2.0, *) { + var taskRef:UnownedTaskExecutor! = nil + self.queue = RunLoopPriorityQueue(runLoop: .current, threadId: pthread_self()) { + $0.runSynchronously(isolatedTo: serialRef, taskExecutor: taskRef) } - let emptySource = CFRunLoopSourceCreate(nil, 0, &context)! - CFRunLoopAddSource(cfRef, emptySource, .commonModes) - CFRunLoopSourceInvalidate(source) + taskRef = asUnownedTaskExecutor() + } else { + self.queue = RunLoopPriorityQueue(runLoop: .current, threadId: pthread_self()) { + $0.runSynchronously(on: serialRef) + } + } + serialRef = asUnownedSerialExecutor() + + } + + @inlinable + deinit { + let key = ObjectIdentifier(queue.runLoop) + let _ = Self.cache.withLockUnchecked{ + $0.removeValue(forKey: key) } - return arrys.map{ RunLoop.Mode($0) } + + + } + + @inlinable + nonisolated + public var inRunLoop: Bool { + let this = pthread_t(bitPattern: threadId) + let current = pthread_self() + let check = pthread_equal(this, current) + + return check != 0 } + // if you access the runLoop while not isolated, it will trigger assert + @inlinable + public var runLoop: RunLoop { + assert(inRunLoop, "can not access \(#function) outside of isolation") + return _runLoop + } + @inlinable nonisolated public func add(_ mode:RunLoop.Mode) { - if CFEqual(cfRef, CFRunLoopGetMain()) { - return - } - CFRunLoopAddSource(cfRef, source, .init(mode.rawValue as CFString)) + queue.add(.init(mode.rawValue as CFString)) } // you can not remove common mode @inlinable nonisolated public func remove(_ mode:RunLoop.Mode) { - if CFEqual(cfRef, CFRunLoopGetMain()) { - return - } - if mode == .common || mode == .default || evaluateCommonModes().contains(mode) { - return - } - CFRunLoopRemoveSource(cfRef, source, .init(mode.rawValue as CFString)) + queue.remove(.init(mode.rawValue as CFString)) + } + + @usableFromInline + internal var source:CFRunLoopSource { + queue.source } } @@ -151,13 +249,13 @@ extension RunLoopPriorityExecutor: SerialExecutor { @inlinable nonisolated public func isSameExclusiveExecutionContext(other: borrowing RunLoopPriorityExecutor) -> Bool { - return CFEqual(cfRef, other.cfRef) + return CFEqual(queue.runLoop, other.queue.runLoop) } @inlinable nonisolated public func asUnownedSerialExecutor() -> UnownedSerialExecutor { - if CFEqual(cfRef, CFRunLoopGetMain()) { + if queue.isMain { return MainActor.sharedUnownedExecutor } else if #available(macOS 14.0, iOS 17.0, watchOS 10.0, tvOS 17.0, *) { return .init(complexEquality: self) @@ -169,20 +267,27 @@ extension RunLoopPriorityExecutor: SerialExecutor { @inlinable nonisolated public func checkIsolated() { - precondition(CFEqual(cfRef, CFRunLoopGetCurrent()), "Unexpected isolation context, expected to be executing on \(cfRef)") + precondition(CFEqual(queue.runLoop, CFRunLoopGetCurrent()), "Unexpected isolation context, expected to be executing on \(runLoop)") } @available(macOS 14.0, iOS 17.0, watchOS 10.0, tvOS 17.0, *) @inlinable nonisolated public func enqueue(_ job: consuming ExecutorJob) { - schedule(.init(job)) + queue.schedule(.init(job)) + + if !inRunLoop { + CFRunLoopWakeUp(queue.runLoop) + } } @inlinable nonisolated public func enqueue(_ job: UnownedJob) { - schedule(job) + queue.schedule(job) + if !inRunLoop { + CFRunLoopWakeUp(queue.runLoop) + } } } @@ -211,59 +316,9 @@ extension RunLoopPriorityExecutor { // since thread local keeps stored reference alive and // user can call this method from thread pool( Concurrency, libdispatch) @usableFromInline - static let cache: some UnfairStateLock<[ObjectIdentifier: Boxed]> = createUncheckedStateLock(uncheckedState: [:]) + static let cache: some UnfairStateLock<[ObjectIdentifier: Unmanaged]> = createUncheckedStateLock(uncheckedState: [:]) - @usableFromInline - static var sharedSource:(ManagedBuffer, CFRunLoopSource) { - let buffer = ManagedBuffer.create(minimumCapacity: 0) { _ in - return .init(value: nil) - } - var context = CFRunLoopSourceContext() - context.version = 0 - context.info = Unmanaged.passUnretained(buffer).toOpaque() - context.retain = { - let ptr = Unmanaged.fromOpaque($0!).retain().toOpaque() - return .init(ptr) - } - context.release = { - Unmanaged.fromOpaque($0!).release() - } - context.perform = { - var heap:Heap - let execute:(@Sendable (consuming UnownedJob) ->Void) - do { - let boxed = Unmanaged>.fromOpaque($0!) - .takeUnretainedValue() - guard let ref = boxed.header.value else { return } - heap = ref.heaps.withLock{ - - var old = Heap() - swap(&$0, &old) - return old - } - let serial = ref.asUnownedSerialExecutor() - if #available(macOS 15.0, iOS 18.0, watchOS 11.0, tvOS 18.0, visionOS 2.0, *) { - let task = ref.asUnownedTaskExecutor() - execute = { [serial, task] in - $0.runSynchronously(isolatedTo: serial, taskExecutor: task) - } - } else { - execute = { [serial] in - $0.runSynchronously(on: serial) - } - } - } - while let block = heap.popMax() { - execute(block.jobImp) - } - } - - context.copyDescription = { null in - let description = "RunLoopPExecutor" - return .passRetained(description as CFString) - } - return (buffer, CFRunLoopSourceCreate(nil, 0, &context)) - } + } extension RunLoopPriorityExecutor { @@ -288,25 +343,25 @@ extension RunLoopPriorityExecutor { /// - Parameter setupHandle: called right before runLoop runs, runloop is active until executor is dead. this block is called exactly once. /// - Important: when using it with existing active runLoop, keep runloop alive until Executor is gracefully deinitialized @inlinable - public static func getOrCreate(_ block: (consuming Self) -> Void) { + public static func getOrCreate(_ block: (consuming RunLoopPriorityExecutor) -> Void) { if Thread.isMainThread { let executor = Self() (consume block)(executor) return } - let runLoop = RunLoop.current + let runLoop = RunLoop.current.getCFRunLoop() let key = ObjectIdentifier(runLoop) - if let existing = cache.withLockUnchecked({ - $0[key]?.value + if let existing = cache.withLock({ + $0[key]?.takeUnretainedValue() }) { - (consume block)(existing as! Self) + (consume block)(existing) return } let source:CFRunLoopSource do { let executor = Self() - Self.cache.withLockUnchecked{ - $0[key] = .init(value: executor) + Self.cache.withLock{ + $0[key] = .passUnretained(executor) } source = executor.source (consume block)(executor) @@ -344,3 +399,4 @@ extension RunLoopPriorityExecutor { */ + diff --git a/Sources/Tetra/Concurrency/RunLoopExecutor.swift b/Sources/Tetra/Concurrency/RunLoopExecutor.swift deleted file mode 100644 index 885cfcf..0000000 --- a/Sources/Tetra/Concurrency/RunLoopExecutor.swift +++ /dev/null @@ -1,150 +0,0 @@ -// -// RunLoopExecutor.swift -// -// -// Created by 박병관 on 8/20/23. -// - -@preconcurrency import Foundation - -internal struct RunLoopRunner: ~Copyable, @unchecked Sendable { - - internal let thread:Thread - private let source:CFRunLoopSource - - fileprivate init(qos:QualityOfService = .default) { - var context = CFRunLoopSourceContext() - context.version = 0 - self.source = CFRunLoopSourceCreate(nil, 0, &context) - self.thread = Thread { [source] in - runInCurrent(source: source) - } - thread.name = "RunLoopSerialExecutor" - thread.qualityOfService = qos - thread.threadPriority = 0.0 - thread.start() - } - - deinit { - let source = source - submit { - CFRunLoopSourceInvalidate(source) - CFRunLoopStop(CFRunLoopGetCurrent()) - } - } - - internal func submit(_ block: @Sendable @escaping () -> Void) { - let job = Timer(timeInterval: 0, repeats: false) { _ in - block() - } - job.perform( - #selector(job.fire), - on: thread, - with: nil, - waitUntilDone: false, - modes: [RunLoop.Mode.common.rawValue] - ) - } - - -} - -@available(macOS 14.0, iOS 17.0, watchOS 10.0, tvOS 17.0, visionOS 1.0, *) -public final class RunLoopExecutor: SerialExecutor { - - internal let runner:RunLoopRunner - - public init(qos: QualityOfService = .default) { - runner = .init(qos: qos) - } - - public func enqueue(_ job: consuming ExecutorJob) { - let ref = UnownedJob(job) - let executor = asUnownedSerialExecutor() - runner.submit { - ref.runSynchronously(on: executor) - } - } - - - public func isSameExclusiveExecutionContext(other: RunLoopExecutor) -> Bool { - return runner.thread == other.runner.thread - } - - public func checkIsolated() { - precondition(runner.thread == Thread.current, "Expected \(runner.thread) but found \(Thread.current)") - } - - -} - -@inlinable -internal func runInCurrent(source:CFRunLoopSource) { - guard RunLoop.current.currentMode == nil else { return } - CFRunLoopAddSource(CFRunLoopGetCurrent(), source, CFRunLoopMode.defaultMode) - defer { CFRunLoopSourceInvalidate(source) } - while CFRunLoopContainsSource(CFRunLoopGetCurrent(), source, CFRunLoopMode.defaultMode) { - let processed = autoreleasepool { - RunLoop.current.run(mode: .default, before: .distantFuture) - } - if !processed { - break - } - } -} - -@available(macOS, deprecated: 14.0, renamed: "RunLoopExecutor") -@available(iOS, deprecated: 17.0, renamed: "RunLoopExecutor", message: "LegacyRunLoopExecutor is deprecated by MoveOnly Types use RunLoopExecutor instead") -@available(watchOS, deprecated: 10.0, renamed: "RunLoopExecutor", message: "LegacyRunLoopExecutor is deprecated by MoveOnly Types use RunLoopExecutor instead") -@available(tvOS, deprecated: 17.0, renamed: "RunLoopExecutor", message: "LegacyRunLoopExecutor is deprecated by MoveOnly Types use RunLoopExecutor instead") -@available(visionOS, deprecated: 1.0, renamed: "RunLoopExecutor", message: "LegacyRunLoopExecutor is deprecated by MoveOnly Types use RunLoopExecutor instead") -public final class LegacyRunLoopExecutor: SerialExecutor { - - internal let runner:RunLoopRunner - - public init(qos:QualityOfService = .default) { - runner = .init(qos: qos) - } - - #if os(macOS) || os(tvOS) || os(watchOS) || os(iOS) - public func enqueue(_ job: UnownedJob) { - let executor = asUnownedSerialExecutor() - runner.submit { - job.runSynchronously(on: executor) - } - } - #else - public func enqueue(_ job: consuming ExecutorJob) { - let ref = UnownedJob(job) - let executor = self.asUnownedSerialExecutor() - runner.submit { - ref.runSynchronously(on: executor) - } - } - #endif - - @available(macOS 14.0, iOS 17.0, watchOS 10.0, tvOS 17.0, visionOS 1.0, *) - public func isSameExclusiveExecutionContext(other: LegacyRunLoopExecutor) -> Bool { - return runner.thread == other.runner.thread - } - #if os(macOS) || os(tvOS) || os(watchOS) || os(iOS) - public func asUnownedSerialExecutor() -> UnownedSerialExecutor { - return .init(ordinary: self) - } - #endif - - public func checkIsolated() { - precondition(runner.thread == Thread.current, "Expected \(runner.thread) but found \(Thread.current)") - } -} - - -@inlinable -public func newRunLoopExecutor(qos: QualityOfService) -> some SerialExecutor { - if #available(macOS 14.0, iOS 17.0, watchOS 10.0, tvOS 17.0, visionOS 1.0, *) { - return RunLoopExecutor(qos: qos) - } else { - return LegacyRunLoopExecutor(qos: qos) - } -} - diff --git a/Sources/Tetra/Concurrency/TaskQos.swift b/Sources/Tetra/Concurrency/TaskQos.swift new file mode 100644 index 0000000..acb7c62 --- /dev/null +++ b/Sources/Tetra/Concurrency/TaskQos.swift @@ -0,0 +1,103 @@ +// +// TaskQos.swift +// +// +// Created by 박병관 on 7/6/24. +// +import Dispatch + +@available(macOS 14.0, iOS 17.0, watchOS 10.0, tvOS 17.0, *) +extension JobPriority { + + func evaluateQos() -> DispatchQoS { + if rawValue == DispatchQoS.unspecified.qosClass.rawValue.rawValue { + return .unspecified + } + var value = DispatchQoS.QoSClass.userInteractive + while true { + let diff = Int(rawValue) - Int(value.rawValue.rawValue) + // same + if diff == 0 { + return .init(qosClass: value, relativePriority: 0) + } + // current priority is higher than QOS + // check for upgrade + if diff > 0 { + if value == .userInteractive { + return .userInteractive + } + // try to upgrade + if let upgrade = value.up { + value = upgrade + continue + } else { + return .unspecified + } + } + // check for downgrade + // relativePriority can only have negative ~ 0 priority + // current priority is less or equal to next lower level + if value == .background { + // fallthrough + } else if let downgrade = value.down { + if rawValue <= downgrade.rawValue.rawValue { + value = downgrade + continue + } + // fallthrough + } else { + return .unspecified + } + // current priority is greater or equal to next lower level, but lower than current qos + // use relativePriority! + let relativePriority = max(Int(QOS_MIN_RELATIVE_PRIORITY), diff) + return .init(qosClass: value, relativePriority: relativePriority) + } + return .unspecified + } + + +} + +extension DispatchQoS.QoSClass { + + var down:Self? { + switch self { + case .background: + return nil + case .utility: + return .background + case .default: + return .utility + case .userInitiated: + return .default + case .userInteractive: + return .userInitiated + case .unspecified: + return nil + @unknown default: + return nil + } + } + + var up:Self? { + switch self { + case .background: + return .utility + case .utility: + return .default + case .default: + return .userInitiated + case .userInitiated: + return .userInteractive + case .userInteractive: + return nil + case .unspecified: + return nil + @unknown default: + return nil + } + } +} + + diff --git a/Sources/Tetra/Foundation/RunLoopSourceBlock.swift b/Sources/Tetra/Foundation/RunLoopSourceBlock.swift new file mode 100644 index 0000000..02abbab --- /dev/null +++ b/Sources/Tetra/Foundation/RunLoopSourceBlock.swift @@ -0,0 +1,85 @@ +// +// RunLoopSourceBlock.swift +// +// +// Created by 박병관 on 7/6/24. +// +import CoreFoundation + +@usableFromInline +enum RunLoopSourceEvent: Hashable { + + case perform + case schedule(CFRunLoop, CFRunLoopMode) + case cancel(CFRunLoop, CFRunLoopMode) +} + + +/// This creates Block based version 0 CFRunLoopSource +/// - Parameters: +/// - allocator: pass `nil` unless you have a reason for it +/// - order: pass `0` unless you have a reason for it +/// - block: RunLoopSource Event handler, this callbacked called in serial, and never called concurrently +/// - Returns: this block based `CFRunLoopSource` +@usableFromInline +func RunLoopSourceCreateWithHandler( + _ allocator: CFAllocator? = nil, + _ order: CFIndex = 0, + _ block: @escaping (RunLoopSourceEvent) -> () +) -> CFRunLoopSource { + typealias BlockSourceType = @convention(block) (CFRunLoop?, CFRunLoopMode?, UnsafePointer?) -> Void + let cBlock: BlockSourceType = { + switch $2?.pointee { + case .none: + block(.perform) + case .some(true): + block(.schedule($0!, $1!)) + case .some(false): + block(.cancel($0!, $1!)) + } + } + let ref = cBlock as AnyObject + let info = Unmanaged.passUnretained(ref).toOpaque() + var sourceContext = CFRunLoopSourceContext() + sourceContext.perform = { info in + let ref = Unmanaged.fromOpaque(info!).takeUnretainedValue() + + let block = unsafeBitCast(ref, to: BlockSourceType.self) + block(nil, nil, nil) + } + sourceContext.schedule = { info, runLoop, mode in + let ref = Unmanaged.fromOpaque(info!).takeUnretainedValue() + let block = unsafeBitCast(ref, to: BlockSourceType.self) + withUnsafePointer(to: true) { + block(runLoop, mode, $0) + } + } + sourceContext.cancel = { info, runLoop, mode in + let ref = Unmanaged.fromOpaque(info!).takeUnretainedValue() + let block = unsafeBitCast(ref, to: BlockSourceType.self) + withUnsafePointer(to: false) { + block(runLoop, mode, $0) + } + } + sourceContext.hash = nil + sourceContext.equal = nil + sourceContext.copyDescription = { info in + if let info { + let address = UInt(bitPattern: info) + let hex = String(address, radix: 16) + return .passRetained("RunLoopSourceCreateWithHandler (0x\(hex))" as CFString) + } else { + return .passRetained("RunLoopSourceCreateWithHandler" as CFString) + } + } + sourceContext.version = 0 + sourceContext.release = { + Unmanaged.fromOpaque($0!).release() + } + sourceContext.retain = { + .init(Unmanaged.fromOpaque($0!).retain().toOpaque()) + } + sourceContext.info = info + let source = CFRunLoopSourceCreate(allocator, order, &sourceContext)! + return source +} diff --git a/Tests/TetraTests/RunLoopSchedulerTests.swift b/Tests/TetraTests/RunLoopSchedulerTests.swift deleted file mode 100644 index 3bf9796..0000000 --- a/Tests/TetraTests/RunLoopSchedulerTests.swift +++ /dev/null @@ -1,75 +0,0 @@ -// -// RunLoopSchedulerTests.swift -// -// -// Created by pbk on 2023/01/27. -// - -import XCTest -@testable import Tetra - -final class RunLoopSchedulerTests: XCTestCase { - - func testBlockingInitializerPerformance() { - measure { - let _ = RunLoopScheduler(sync: ()) - } - } - - func testRunLoopBasic() async { - let scheduler = await RunLoopScheduler(async: (), config: .init(qos: .background)) - await withUnsafeContinuation{ continuation in - scheduler.schedule { - XCTAssertEqual(CFRunLoopGetCurrent(), scheduler.cfRunLoop) - continuation.resume() - } - } - await withUnsafeContinuation{ continuation in - let date = Date().addingTimeInterval(0.5) - scheduler.schedule(after: .init(date)) { - XCTAssertEqual( - date.timeIntervalSinceReferenceDate, - Date().timeIntervalSinceReferenceDate, - accuracy: 0.05 - ) - XCTAssertEqual(CFRunLoopGetCurrent(), scheduler.cfRunLoop) - - continuation.resume() - } - } - } - - func testRunLoopNotificationQueue() async { - let scheduler = await RunLoopScheduler(async: (), config: .init(qos: .background)) - let name = Notification.Name(UUID().uuidString) - let object = NSObject() - let expect1 = expectation(forNotification: name, object: object, notificationCenter: .default) { notification in - XCTAssertTrue(notification.userInfo?["A"] as? String == "B") - return true - } - - scheduler.schedule { - NotificationQueue.default - .enqueue(.init(name: name, object: object, userInfo: ["A":"B"]), postingStyle: .whenIdle, coalesceMask: [.onName, .onSender], forModes: nil) - NotificationQueue.default - .enqueue(.init(name: name, object: object, userInfo: ["A":"1"]), postingStyle: .whenIdle, coalesceMask: [.onName, .onSender], forModes: nil) - NotificationQueue.default - .enqueue(.init(name: name, object: object, userInfo: ["A":"2"]), postingStyle: .whenIdle, coalesceMask: [.onName, .onSender], forModes: nil) - NotificationQueue.default - .enqueue(.init(name: name, object: object, userInfo: ["A":"3"]), postingStyle: .whenIdle, coalesceMask: [.onName, .onSender], forModes: nil) - - } - await fulfillment(of: [expect1], timeout: 1) - let expect2 = expectation(forNotification: name, object: object, notificationCenter: .default) { - XCTAssertTrue($0.userInfo?["A"] as? String == "C") - return true - } - - scheduler.schedule { - NotificationQueue.default - .enqueue(.init(name: name, object: object, userInfo: ["A":"C"]), postingStyle: .whenIdle, coalesceMask: [.onName, .onSender], forModes: nil) - } - await fulfillment(of: [expect2], timeout: 1) - } - -} From a54923b06a667fb20261230e64e4048020eeff45 Mon Sep 17 00:00:00 2001 From: park-byeong-gwan Date: Sun, 7 Jul 2024 02:33:02 +0900 Subject: [PATCH 40/63] compact qos algorithnm --- Sources/Tetra/Concurrency/JobBlock.swift | 2 + .../Tetra/Concurrency/PriorityRunLoop.swift | 70 ++++++--- Sources/Tetra/Concurrency/TaskQos.swift | 141 ++++++++---------- .../Tetra/Foundation/RunLoopSourceBlock.swift | 75 +++++++++- 4 files changed, 191 insertions(+), 97 deletions(-) diff --git a/Sources/Tetra/Concurrency/JobBlock.swift b/Sources/Tetra/Concurrency/JobBlock.swift index 6890d86..4a67a19 100644 --- a/Sources/Tetra/Concurrency/JobBlock.swift +++ b/Sources/Tetra/Concurrency/JobBlock.swift @@ -5,7 +5,9 @@ // Created by 박병관 on 6/30/24. // import Darwin +import Dispatch +// three word is max size to use stack allocation @usableFromInline struct JobBlock: Hashable, Comparable, Sendable { diff --git a/Sources/Tetra/Concurrency/PriorityRunLoop.swift b/Sources/Tetra/Concurrency/PriorityRunLoop.swift index 9f846c7..f0a3357 100644 --- a/Sources/Tetra/Concurrency/PriorityRunLoop.swift +++ b/Sources/Tetra/Concurrency/PriorityRunLoop.swift @@ -30,6 +30,7 @@ struct RunLoopPriorityQueue: ~Copyable, Sendable { nonisolated(unsafe) internal let thread:pthread_t + @usableFromInline init( runLoop: RunLoop, @@ -38,11 +39,12 @@ struct RunLoopPriorityQueue: ~Copyable, Sendable { ) { self.runLoop = runLoop.getCFRunLoop() self.thread = threadId - self.isMain = CFEqual(runLoop, CFRunLoopGetMain()) + self.isMain = CFEqual(runLoop.getCFRunLoop(), CFRunLoopGetMain()) if isMain { self.source = CFRunLoopSourceCreate(nil, 0, nil) CFRunLoopSourceInvalidate(source) } else { + self.source = RunLoopSourceCreateWithHandler { [heaps] in guard $0 == .perform else { return } @@ -52,14 +54,38 @@ struct RunLoopPriorityQueue: ~Copyable, Sendable { swap(&next, &$0) return next } - while let block = queue.popMax() { - defer { - if let ref = block.token { - pthread_override_qos_class_end_np(ref) - } + let currentQos:DispatchQoS + do { + let thread_qos = qos_class_self() + var priority:Int32 = 0 + pthread_get_qos_class_np(pthread_self(), nil, &priority) + currentQos = .init(qosClass: .init(rawValue: thread_qos)!, relativePriority: Int(priority)) + } + var qos: DispatchQoS = currentQos + + defer { + if qos.qosClass != currentQos.qosClass { + let result = pthread_set_qos_class_self_np(currentQos.qosClass.rawValue, Int32(currentQos.relativePriority)) + assert(result == 0, "\(result)") } + } + + var jobPriority = currentQos.evaluateTaskPriority() + while let block = queue.popMax() { let job = block.jobImp - execute(job) + defer { execute(job) } + if #available(macOS 14.0, iOS 17.0, watchOS 10.0, tvOS 17.0, *), let newTaskPriority = TaskPriority(job.priority), jobPriority != newTaskPriority { + let newQos = newTaskPriority.evaluateQos() + jobPriority = newTaskPriority + let result = pthread_set_qos_class_self_np(newQos.qosClass.rawValue, Int32(newQos.relativePriority)) + qos = newQos + assert(result == 0, "\(result)") + + } + if let ref = block.token { + let result = pthread_override_qos_class_end_np(ref) + assert(result == 0, "\(result) pthread_override_qos_class_end_np failed") + } } } CFRunLoopAddSource(self.runLoop, source, .commonModes) @@ -112,27 +138,33 @@ struct RunLoopPriorityQueue: ~Copyable, Sendable { MainActor.shared.enqueue(job) return } - - let override: pthread_override_t? - + let qos:DispatchQoS? if #available(macOS 14.0, iOS 17.0, watchOS 10.0, tvOS 17.0, *) { - let qos = job.priority.evaluateQos() - if qos.qosClass == .unspecified { - override = nil + let value = TaskPriority(job.priority)?.evaluateQos() + if value?.qosClass != .unspecified { + qos = value } else { - override = pthread_override_qos_class_start_np(thread, qos.qosClass.rawValue, Int32(qos.relativePriority)) + qos = nil } } else { - override = nil + qos = nil } - - + let threadId = thread + var qos_class = QOS_CLASS_UNSPECIFIED + pthread_get_qos_class_np(threadId, &qos_class, nil) +// print(DispatchQoS.QoSClass(rawValue: qos_class)!) heaps.withLockUnchecked{ [job] in // lastest has the lower id which results lower priority let id = -$0.count - var item = JobBlock(id: id, jobRef: consume job) - item.token = override + let item = JobBlock(id: id, jobRef: consume job) $0.insert(item) + if #available(macOS 14.0, iOS 17.0, watchOS 10.0, tvOS 17.0, *), var max = $0.popMax() { + if max.token == nil, let qos, qos.qosClass.rawValue != qos_class, item.priority == max.priority { + let override:pthread_override_t? = pthread_override_qos_class_start_np(threadId, qos.qosClass.rawValue, Int32(qos.relativePriority)) + max.token = override + } + $0.insert(max) + } } CFRunLoopSourceSignal(source) } diff --git a/Sources/Tetra/Concurrency/TaskQos.swift b/Sources/Tetra/Concurrency/TaskQos.swift index acb7c62..3408094 100644 --- a/Sources/Tetra/Concurrency/TaskQos.swift +++ b/Sources/Tetra/Concurrency/TaskQos.swift @@ -6,98 +6,87 @@ // import Dispatch -@available(macOS 14.0, iOS 17.0, watchOS 10.0, tvOS 17.0, *) -extension JobPriority { +extension TaskPriority { func evaluateQos() -> DispatchQoS { - if rawValue == DispatchQoS.unspecified.qosClass.rawValue.rawValue { + let userInteractive = TaskPriority.userInteractive + switch self { + case userInteractive: + return .userInteractive + case .high: + return .userInitiated + case .medium: + return .default + case .low: + return .utility + case .background: + return .background + case .unspecified: return .unspecified + default: + break + } + if self > userInteractive { + return .userInteractive + } + let calculate = { (basis: TaskPriority) in + let diff = basis.rawValue - self.rawValue + return max(-Int(diff), Int(QOS_MIN_RELATIVE_PRIORITY)) + } + if self < .background { + return .init(qosClass: .background, relativePriority: calculate(.background)) + } + if (TaskPriority.userInitiated.. 0 { - if value == .userInteractive { - return .userInteractive - } - // try to upgrade - if let upgrade = value.up { - value = upgrade - continue - } else { - return .unspecified - } - } - // check for downgrade - // relativePriority can only have negative ~ 0 priority - // current priority is less or equal to next lower level - if value == .background { - // fallthrough - } else if let downgrade = value.down { - if rawValue <= downgrade.rawValue.rawValue { - value = downgrade - continue - } - // fallthrough - } else { - return .unspecified - } - // current priority is greater or equal to next lower level, but lower than current qos - // use relativePriority! - let relativePriority = max(Int(QOS_MIN_RELATIVE_PRIORITY), diff) - return .init(qosClass: value, relativePriority: relativePriority) + if (TaskPriority.medium.. TaskPriority? { + let evaluate = { (base:TaskPriority) in + let rawValue = Int8(bitPattern: base.rawValue) + Int8(relativePriority) + return TaskPriority(rawValue: UInt8(bitPattern: rawValue)) } - } - - var up:Self? { - switch self { - case .background: - return .utility - case .utility: - return .default - case .default: - return .userInitiated - case .userInitiated: - return .userInteractive + switch qosClass { case .userInteractive: - return nil - case .unspecified: - return nil - @unknown default: + return evaluate(.userInteractive) + case .userInitiated: + return evaluate(.high) + case .default: + return evaluate(.medium) + case .utility: + return evaluate(.low) + case .background: + return evaluate(.background) + default: return nil } + } + } +@available(macOS 14.0, iOS 17.0, watchOS 10.0, tvOS 17.0, *) +extension JobPriority { + + func evaluateQos() -> DispatchQoS { + return TaskPriority(rawValue: rawValue).evaluateQos() + } + + +} + diff --git a/Sources/Tetra/Foundation/RunLoopSourceBlock.swift b/Sources/Tetra/Foundation/RunLoopSourceBlock.swift index 02abbab..0000dc8 100644 --- a/Sources/Tetra/Foundation/RunLoopSourceBlock.swift +++ b/Sources/Tetra/Foundation/RunLoopSourceBlock.swift @@ -8,18 +8,24 @@ import CoreFoundation @usableFromInline enum RunLoopSourceEvent: Hashable { - + /// called when Source Fire, check current Threads RunLoop more detail infomation. e.g current Mode, runLoop case perform + /// called when Source is added to the RunLoop case schedule(CFRunLoop, CFRunLoopMode) + /// called when Source is removed from the RunLoop case cancel(CFRunLoop, CFRunLoopMode) } /// This creates Block based version 0 CFRunLoopSource +/// +/// schedule and cancel event can be invoke concurrenlty +/// +/// /// - Parameters: /// - allocator: pass `nil` unless you have a reason for it /// - order: pass `0` unless you have a reason for it -/// - block: RunLoopSource Event handler, this callbacked called in serial, and never called concurrently +/// - block: RunLoopSource Event handler, /// - Returns: this block based `CFRunLoopSource` @usableFromInline func RunLoopSourceCreateWithHandler( @@ -83,3 +89,68 @@ func RunLoopSourceCreateWithHandler( let source = CFRunLoopSourceCreate(allocator, order, &sourceContext)! return source } + +@usableFromInline +func RunLoopSourceCreateWithHandler2( + _ allocator: CFAllocator? = nil, + _ order: CFIndex = 0, + _ perform: @escaping () -> (), + _ schedule: ( (CFRunLoop, CFRunLoopMode) -> Void)? = nil, + _ cancel: ( (CFRunLoop, CFRunLoopMode) -> Void)? = nil +) -> CFRunLoopSource { + typealias BlockSourceType = @convention(block) (CFRunLoop?, CFRunLoopMode?, UnsafePointer?) -> Void + let cBlock: BlockSourceType = { + switch $2?.pointee { + case .none: + perform() + case .some(true): + schedule?($0!, $1!) + case .some(false): + cancel?($0!, $1!) + } + } + let ref = cBlock as AnyObject + let info = Unmanaged.passUnretained(ref).toOpaque() + var sourceContext = CFRunLoopSourceContext() + sourceContext.perform = { info in + let ref = Unmanaged.fromOpaque(info!).takeUnretainedValue() + + let block = unsafeBitCast(ref, to: BlockSourceType.self) + block(nil, nil, nil) + } + sourceContext.schedule = { info, runLoop, mode in + let ref = Unmanaged.fromOpaque(info!).takeUnretainedValue() + let block = unsafeBitCast(ref, to: BlockSourceType.self) + withUnsafePointer(to: true) { + block(runLoop, mode, $0) + } + } + sourceContext.cancel = { info, runLoop, mode in + let ref = Unmanaged.fromOpaque(info!).takeUnretainedValue() + let block = unsafeBitCast(ref, to: BlockSourceType.self) + withUnsafePointer(to: false) { + block(runLoop, mode, $0) + } + } + sourceContext.hash = nil + sourceContext.equal = nil + sourceContext.copyDescription = { info in + if let info { + let address = UInt(bitPattern: info) + let hex = String(address, radix: 16) + return .passRetained("RunLoopSourceCreateWithHandler (0x\(hex))" as CFString) + } else { + return .passRetained("RunLoopSourceCreateWithHandler" as CFString) + } + } + sourceContext.version = 0 + sourceContext.release = { + Unmanaged.fromOpaque($0!).release() + } + sourceContext.retain = { + .init(Unmanaged.fromOpaque($0!).retain().toOpaque()) + } + sourceContext.info = info + let source = CFRunLoopSourceCreate(allocator, order, &sourceContext)! + return source +} From 4e8910a473c936099977de5ce77b812503362cfe Mon Sep 17 00:00:00 2001 From: park-byeong-gwan Date: Mon, 8 Jul 2024 18:25:07 +0900 Subject: [PATCH 41/63] fix available annotation --- Sources/Tetra/Concurrency/JobBlock.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Sources/Tetra/Concurrency/JobBlock.swift b/Sources/Tetra/Concurrency/JobBlock.swift index 4a67a19..b04755a 100644 --- a/Sources/Tetra/Concurrency/JobBlock.swift +++ b/Sources/Tetra/Concurrency/JobBlock.swift @@ -28,7 +28,7 @@ struct JobBlock: Hashable, Comparable, Sendable { } @usableFromInline - @available(macOS 14.0, *) + @available(macOS 14.0, iOS 17.0, watchOS 10.0, tvOS 17.0, *) init(id: Int, job: consuming ExecutorJob) { self.id = id self.jobImp = UnownedJob(job) From a2dde3c4d4ffc51ac853ade4e614859ae8b16da4 Mon Sep 17 00:00:00 2001 From: park-byeong-gwan Date: Mon, 15 Jul 2024 17:34:36 +0900 Subject: [PATCH 42/63] =?UTF-8?q?Swift=206=20Concurrency=20check=20?= =?UTF-8?q?=EC=9E=84=EC=8B=9C=20=EB=8C=80=EC=9D=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../AsyncCompactMapSequence.swift | 2 +- .../AsyncDropWhileSequence.swift | 5 +- .../AsyncFilterSequence.swift | 2 +- .../AsyncFlatMapSequence.swift | 2 +- .../AsyncMapSequence.swift | 86 ++++++++++++++++++- .../AsyncPrefixWhileSequence.swift | 2 +- .../BackPortAsyncSequence/AsyncStream.swift | 6 +- .../AsyncThrowingStream.swift | 7 +- .../LegacyTypedAsyncSequence.swift | 13 ++- .../ThrowingTaskGroup.swift | 2 +- Sources/BackPortAsyncSequence/operators.swift | 15 ++++ .../CompatDiscardingTaskGroup.swift | 15 ++-- .../TaskGroup.swift | 24 +++--- .../ThrowingTaskGroup.swift | 32 +++---- .../conformance.swift | 14 ++- .../Tetra/Combine/AsyncSubscriberState.swift | 2 +- .../Tetra/Combine/ExperimentalMapTask.swift | 5 +- .../Combine/Publishers+AsyncFlatMap.swift | 5 +- .../Notification+AsyncSequence.swift | 7 +- 19 files changed, 187 insertions(+), 59 deletions(-) diff --git a/Sources/BackPortAsyncSequence/AsyncCompactMapSequence.swift b/Sources/BackPortAsyncSequence/AsyncCompactMapSequence.swift index 7df0cee..b707d3a 100644 --- a/Sources/BackPortAsyncSequence/AsyncCompactMapSequence.swift +++ b/Sources/BackPortAsyncSequence/AsyncCompactMapSequence.swift @@ -94,7 +94,7 @@ extension BackPort.AsyncCompactMapSequence.Iterator: AsyncIteratorProtocol, Type return nil } do { - if let transformed = try await transform(element) { + if let transformed = try await transform(Suppress(base: element).base) { return transformed } } catch { diff --git a/Sources/BackPortAsyncSequence/AsyncDropWhileSequence.swift b/Sources/BackPortAsyncSequence/AsyncDropWhileSequence.swift index 123b6c9..c3742f8 100644 --- a/Sources/BackPortAsyncSequence/AsyncDropWhileSequence.swift +++ b/Sources/BackPortAsyncSequence/AsyncDropWhileSequence.swift @@ -45,7 +45,7 @@ extension BackPort.AsyncDropWhileSequence: AsyncSequence, TypedAsyncSequence { var baseIterator: Base.AsyncIterator @usableFromInline - let predicate: ((Base.Element) async throws(Failure) -> Bool) + let predicate: (( Base.Element) async throws(Failure) -> Bool) @usableFromInline var finished = false @@ -94,7 +94,8 @@ extension BackPort.AsyncDropWhileSequence.Iterator: AsyncIteratorProtocol, Typed return nil } do { - if try await predicate(element) == false { + + if try await predicate(Suppress(base: element).base) == false { doneDropping = true return element } diff --git a/Sources/BackPortAsyncSequence/AsyncFilterSequence.swift b/Sources/BackPortAsyncSequence/AsyncFilterSequence.swift index 5831951..744537c 100644 --- a/Sources/BackPortAsyncSequence/AsyncFilterSequence.swift +++ b/Sources/BackPortAsyncSequence/AsyncFilterSequence.swift @@ -79,7 +79,7 @@ extension BackPort.AsyncFilterSequence.Iterator: AsyncIteratorProtocol, TypedAsy return nil } do { - if try await isIncluded(element) { + if try await isIncluded(Suppress(base: element).base) { return element } } catch { diff --git a/Sources/BackPortAsyncSequence/AsyncFlatMapSequence.swift b/Sources/BackPortAsyncSequence/AsyncFlatMapSequence.swift index 81d96c7..449bedd 100644 --- a/Sources/BackPortAsyncSequence/AsyncFlatMapSequence.swift +++ b/Sources/BackPortAsyncSequence/AsyncFlatMapSequence.swift @@ -98,7 +98,7 @@ extension BackPort.AsyncFlatMapSequence.Iterator: AsyncIteratorProtocol, TypedAs } let segment: SegmentOfResult do { - segment = try await transform(item) + segment = try await transform(Suppress(base: item).base) var iterator = segment.makeAsyncIterator() guard let element = try await iterator.next(isolation: actor) else { currentIterator = nil diff --git a/Sources/BackPortAsyncSequence/AsyncMapSequence.swift b/Sources/BackPortAsyncSequence/AsyncMapSequence.swift index 1c06cbc..4f3a9a4 100644 --- a/Sources/BackPortAsyncSequence/AsyncMapSequence.swift +++ b/Sources/BackPortAsyncSequence/AsyncMapSequence.swift @@ -96,7 +96,7 @@ extension BackPort.AsyncMapSequence.Iterator: AsyncIteratorProtocol, TypedAsyncI return nil } do { - return try await transform(element) + return try await transform(Suppress(base: element).base) } catch { finished = true throw error @@ -140,4 +140,88 @@ extension BackPort.AsyncMapSequence { } +// +//protocol AsyncProducer:~Copyable { +// +// associatedtype Failure:Error +// // this is not sendable +// associatedtype Element +// +// // since Element is not sendable we needs `sending` +// mutating func produce(isolation actor: isolated (any Actor)) async throws(Failure) -> sending Element? +// +//} +// +//struct MappingProducer: AsyncProducer { +// +// var base:Base +// // can not omit isolation parameter, can not use isolated(any) either +// let compute:(isolated (any Actor), consuming Base.Element) async throws(Base.Failure) -> sending Element +// +// mutating func produce(isolation actor: isolated (any Actor)) async throws(Base.Failure) -> sending Element? { +// if let value = try await base.produce(isolation: actor) { +// let newValue = try await compute(actor, value) +// return newValue +// } +// +// return nil +// } +// +//} +// +//struct MutatingProducer:AsyncProducer,~Copyable { +// +// var failed = false +// var state:State +// var source:EventSource +// let mutation:(isolated (any Actor), inout State ,sending EventSource.Element) async throws(EventSource.Failure) -> sending Effect +// +// mutating func produce(isolation actor: isolated (any Actor)) async throws(EventSource.Failure) -> sending Effect? { +// if failed { +// return nil +// } +// if let event = try await source.produce(isolation: actor) { +// do { +// let effect = try await mutation(actor, &state, event) +// +// return effect +// } catch { +// failed = true +// throw error +// } +// } +// +// return nil +// } +// +// +//} +//struct FilteringProducer: AsyncProducer { +// +// var failed = false +// var base:Base +// // can not omit isolation parameter, can not use isolated(any) either +// let predicate:(isolated (any Actor), borrowing Base.Element) async throws(Base.Failure) -> Bool +// +// mutating func produce(isolation actor: isolated (any Actor)) async throws(Base.Failure) -> sending Base.Element? { +// if failed { +// return nil +// } +// if let value = try await base.produce(isolation: actor) { +// do { +// // Sending 'value' risks causing data races +// // may be marking parameter as ~Escape would help? +// if try await predicate(actor, value) { +// return value +// } +// } catch { +// failed = true +// throw error +// } +// } +// +// return nil +// } +// +//} diff --git a/Sources/BackPortAsyncSequence/AsyncPrefixWhileSequence.swift b/Sources/BackPortAsyncSequence/AsyncPrefixWhileSequence.swift index d557b22..f7e30c2 100644 --- a/Sources/BackPortAsyncSequence/AsyncPrefixWhileSequence.swift +++ b/Sources/BackPortAsyncSequence/AsyncPrefixWhileSequence.swift @@ -104,7 +104,7 @@ extension BackPort.AsyncPrefixWhileSequence.Iterator: AsyncIteratorProtocol, Typ public mutating func next(isolation actor: isolated (any Actor)? = #isolation) async throws(Failure) -> Base.Element? { if !predicateHasFailed, let nextElement = try await baseIterator.next(isolation: actor) { do { - if try await predicate(nextElement) { + if try await predicate(Suppress(base: nextElement).base) { return nextElement } else { predicateHasFailed = true diff --git a/Sources/BackPortAsyncSequence/AsyncStream.swift b/Sources/BackPortAsyncSequence/AsyncStream.swift index 0979b9a..3c707ba 100644 --- a/Sources/BackPortAsyncSequence/AsyncStream.swift +++ b/Sources/BackPortAsyncSequence/AsyncStream.swift @@ -49,7 +49,11 @@ extension AsyncTypedStream.Iterator: AsyncIteratorProtocol, TypedAsyncIteratorPr if #available(macOS 15.0, iOS 18.0, watchOS 11.0, tvOS 18.0, visionOS 2.0, *) { return await baseIterator.next(isolation: actor) } else { - return await advanceNext() + nonisolated(unsafe) + var iter = self + let value = await iter.advanceNext() + self = iter + return value } } diff --git a/Sources/BackPortAsyncSequence/AsyncThrowingStream.swift b/Sources/BackPortAsyncSequence/AsyncThrowingStream.swift index db5df2d..7727868 100644 --- a/Sources/BackPortAsyncSequence/AsyncThrowingStream.swift +++ b/Sources/BackPortAsyncSequence/AsyncThrowingStream.swift @@ -49,9 +49,14 @@ extension AsyncTypedThrowingStream.Iterator: AsyncIteratorProtocol, TypedAsyncIt if #available(macOS 15.0, iOS 18.0, watchOS 11.0, tvOS 18.0, visionOS 2.0, *) { return try await baseIterator.next(isolation: actor) } else { + nonisolated(unsafe) + var iter = self do { - return try await nextValue() + let value = try await iter.nextValue() + self = iter + return value } catch { + self = iter throw (error as! Failure) } } diff --git a/Sources/BackPortAsyncSequence/LegacyTypedAsyncSequence.swift b/Sources/BackPortAsyncSequence/LegacyTypedAsyncSequence.swift index 4dd6235..6181ee0 100644 --- a/Sources/BackPortAsyncSequence/LegacyTypedAsyncSequence.swift +++ b/Sources/BackPortAsyncSequence/LegacyTypedAsyncSequence.swift @@ -34,7 +34,7 @@ extension LegacyTypedAsyncSequence: AsyncSequence, TypedAsyncSequence { public struct Iterator { @usableFromInline - var baseIterator:Base.AsyncIterator + package var baseIterator:Base.AsyncIterator @inlinable @@ -57,7 +57,16 @@ extension LegacyTypedAsyncSequence.Iterator: AsyncIteratorProtocol, TypedAsyncIt if #available(macOS 15.0, iOS 18.0, watchOS 11.0, tvOS 18.0, visionOS 2.0, *) { return try await baseIterator.next(isolation: actor) } else { - return try await advance() + nonisolated(unsafe) + var iter = self + do { + let value = try await iter.advance() + self = iter + return value + } catch { + self = iter + throw error + } } } diff --git a/Sources/BackPortAsyncSequence/ThrowingTaskGroup.swift b/Sources/BackPortAsyncSequence/ThrowingTaskGroup.swift index 1b7c394..6526776 100644 --- a/Sources/BackPortAsyncSequence/ThrowingTaskGroup.swift +++ b/Sources/BackPortAsyncSequence/ThrowingTaskGroup.swift @@ -50,7 +50,7 @@ extension TypedThrowingTaskGroup.Iterator: AsyncIteratorProtocol, TypedAsyncIter @inlinable public mutating func next(isolation actor: isolated (any Actor)? = #isolation) async throws(Failure) -> Element? { - return try await parent.nextResult()?.get() + return try await parent.nextResult(isolation: actor)?.get() } @_disfavoredOverload diff --git a/Sources/BackPortAsyncSequence/operators.swift b/Sources/BackPortAsyncSequence/operators.swift index 887a082..ba5da67 100644 --- a/Sources/BackPortAsyncSequence/operators.swift +++ b/Sources/BackPortAsyncSequence/operators.swift @@ -154,3 +154,18 @@ public extension TetraExtension { } + +@preconcurrency +@usableFromInline +struct Suppress:@unchecked Sendable { + + @usableFromInline + nonisolated(unsafe) + var base:Base + + @usableFromInline + init(base: Base) { + self.base = base + } + +} diff --git a/Sources/BackportDiscardingTaskGroup/CompatDiscardingTaskGroup.swift b/Sources/BackportDiscardingTaskGroup/CompatDiscardingTaskGroup.swift index 22d3760..0314fbf 100644 --- a/Sources/BackportDiscardingTaskGroup/CompatDiscardingTaskGroup.swift +++ b/Sources/BackportDiscardingTaskGroup/CompatDiscardingTaskGroup.swift @@ -5,11 +5,10 @@ // Created by 박병관 on 6/20/24. // @usableFromInline -package protocol CompatDiscardingTaskGroup { +package protocol CompatDiscardingTaskGroup { - - associatedtype Failure:Error = any Error - typealias Block = @Sendable @isolated(any) () async throws(Failure) -> Void + associatedtype Err:Error = any Error + typealias Block = @isolated(any) @Sendable () async throws(Err) -> Void @inlinable var isCancelled:Bool { get } @@ -23,13 +22,13 @@ package protocol CompatDiscardingTaskGroup { @inlinable mutating func addTaskUnlessCancelled( priority: TaskPriority?, - operation: @escaping Block + operation: sending @escaping Block ) -> Bool @inlinable mutating func addTask( priority: TaskPriority?, - operation: @escaping Block + operation: sending @escaping Block ) @inlinable @@ -37,7 +36,7 @@ package protocol CompatDiscardingTaskGroup { mutating func addTask( executorPreference taskExecutor: (any TaskExecutor)?, priority: TaskPriority?, - operation: @escaping Block + operation: sending @escaping Block ) @inlinable @@ -45,7 +44,7 @@ package protocol CompatDiscardingTaskGroup { mutating func addTaskUnlessCancelled( executorPreference taskExecutor: (any TaskExecutor)?, priority: TaskPriority?, - operation: @escaping Block + operation: sending @escaping Block ) -> Bool } diff --git a/Sources/BackportDiscardingTaskGroup/TaskGroup.swift b/Sources/BackportDiscardingTaskGroup/TaskGroup.swift index 5a8c67e..fcf76da 100644 --- a/Sources/BackportDiscardingTaskGroup/TaskGroup.swift +++ b/Sources/BackportDiscardingTaskGroup/TaskGroup.swift @@ -26,16 +26,21 @@ extension TaskGroup where ChildTaskResult == Void { await holder.hold() } + let suppress = Suppress(base: self) /// drain all the finished or failed Task async let subTask:Void = { - while let _ = await next(isolation: actor) { + var iter = suppress.base + while let _ = await iter.next(isolation: actor) { if await holder.isFinished { break } } }() + nonisolated(unsafe) + let block = body async let mainTask = { - let v = await runBlock(isolation: actor, body:body) + var iter = suppress.base + let v = await block(actor, &iter) await holder.markDone() return Suppress(base: v) }() @@ -43,14 +48,6 @@ extension TaskGroup where ChildTaskResult == Void { return await mainTask.base } - @usableFromInline - internal mutating func runBlock( - isolation actor: isolated T, - body: (isolated T, inout Self) async throws(ErrorRef) -> sending V - ) async throws(ErrorRef) -> sending V { - try await body(actor, &self) - } - } @inlinable @@ -84,7 +81,7 @@ package func simuateDiscardingTaskGroup( /// - Returns: which is returned from body /// - SeeAlso: withDiscardingTaskGroup(returning:body:) @inlinable -package func simuateDiscardingTaskGroup( +package func simuateDiscardingTaskGroup( body: @Sendable @isolated(any) (inout TaskGroup) async -> sending TaskResult ) async -> sending TaskResult { guard let actor = body.isolation else { @@ -94,7 +91,8 @@ package func simuateDiscardingTaskGroup( precondition(actor === region, "must be isolated on the inferred actor") nonisolated(unsafe) var unsafe = Suppress(base: group) - defer { group = unsafe.base } - return await body(&unsafe.base) + let value = await body(&unsafe.base) + group = unsafe.base + return value } } diff --git a/Sources/BackportDiscardingTaskGroup/ThrowingTaskGroup.swift b/Sources/BackportDiscardingTaskGroup/ThrowingTaskGroup.swift index d86f955..978addf 100644 --- a/Sources/BackportDiscardingTaskGroup/ThrowingTaskGroup.swift +++ b/Sources/BackportDiscardingTaskGroup/ThrowingTaskGroup.swift @@ -26,17 +26,22 @@ extension ThrowingTaskGroup where ChildTaskResult == Void, Failure == any Error /// so that subTask won't return await holder.hold() } + let suppress = Suppress(base: self) /// drain all the finished or failed Task async let subTask:Void = { - while let _ = try await next(isolation: actor) { + var iter = suppress.base + while let _ = try await iter.next(isolation: actor) { if await holder.isFinished { break } } }() + nonisolated(unsafe) + let block = body async let mainTask = { do { - let v = try await runBlock(isolation: actor, body:body) + var iter = suppress.base + let v = try await block(actor, &iter) await holder.markDone() return Suppress(base: v) } catch { @@ -59,22 +64,13 @@ extension ThrowingTaskGroup where ChildTaskResult == Void, Failure == any Error } return value } - - @usableFromInline - internal mutating func runBlock( - isolation actor: isolated T, - body: (isolated T, inout Self) async throws(ErrorRef) -> sending V - ) async throws(ErrorRef) -> sending V { - try await body(actor, &self) - } - } @inlinable package func simuateThrowingDiscardingTaskGroup( isolation actor: isolated T, body: @Sendable (isolated T, inout ThrowingTaskGroup) async throws -> sending TaskResult -) async throws -> TaskResult { +) async throws -> sending TaskResult { try await withThrowingTaskGroup(of: Void.self, returning: TaskResult.self) { try await $0.simulateDiscarding(isolation: actor, body: body) } @@ -82,7 +78,7 @@ package func simuateThrowingDiscardingTaskGroup( @inlinable -package func simuateThrowingDiscardingTaskGroup( +package func simuateThrowingDiscardingTaskGroup( body: @Sendable @isolated(any) (inout ThrowingTaskGroup) async throws -> sending TaskResult ) async throws -> sending TaskResult { guard let actor = body.isolation else { @@ -92,8 +88,14 @@ package func simuateThrowingDiscardingTaskGroup( precondition(actor === region, "must be isolated on the inferred actor") nonisolated(unsafe) var unsafe = Suppress(base: group) - defer { group = unsafe.base } - return try await body(&unsafe.base) + do { + let value = try await body(&unsafe.base) + group = unsafe.base + return value + } catch { + group = unsafe.base + throw error + } } } diff --git a/Sources/BackportDiscardingTaskGroup/conformance.swift b/Sources/BackportDiscardingTaskGroup/conformance.swift index 1fcc159..3802cf1 100644 --- a/Sources/BackportDiscardingTaskGroup/conformance.swift +++ b/Sources/BackportDiscardingTaskGroup/conformance.swift @@ -7,9 +7,12 @@ @available(macOS 14.0, iOS 17.0, watchOS 10.0, tvOS 17.0, macCatalyst 17.0, visionOS 1.0, *) extension DiscardingTaskGroup: CompatDiscardingTaskGroup { - @usableFromInline - package typealias Failure = NoThrow + @usableFromInline + package typealias Err = NoThrow +// @usableFromInline +// package typealias Failure = NoThrow +// @_disfavoredOverload @inlinable package mutating func addTaskUnlessCancelled(priority: TaskPriority?, operation: @escaping Block) -> Bool { @@ -97,7 +100,7 @@ extension TaskGroup: CompatDiscardingTaskGroup where ChildTaskResult == Void { } @usableFromInline - package typealias Failure = NoThrow + package typealias Err = NoThrow } @@ -105,11 +108,14 @@ extension TaskGroup: CompatDiscardingTaskGroup where ChildTaskResult == Void { @available(macOS 14.0, iOS 17.0, watchOS 10.0, tvOS 17.0, macCatalyst 17.0, visionOS 1.0, *) extension ThrowingDiscardingTaskGroup: CompatDiscardingTaskGroup { + @usableFromInline + package typealias Err = any Error } extension ThrowingTaskGroup: CompatDiscardingTaskGroup where ChildTaskResult == Void { - + @usableFromInline + package typealias Err = any Error } diff --git a/Sources/Tetra/Combine/AsyncSubscriberState.swift b/Sources/Tetra/Combine/AsyncSubscriberState.swift index b2ccbf9..493f8d7 100644 --- a/Sources/Tetra/Combine/AsyncSubscriberState.swift +++ b/Sources/Tetra/Combine/AsyncSubscriberState.swift @@ -70,7 +70,7 @@ struct AsyncSubscriberState { case .resumeValue(let continuation, let input): nonisolated(unsafe) let value = Result.success(consume input) - continuation.resume(returning: value) + continuation.resume(returning: Suppress(value: value).value) case .request(let subscription, let demand): subscription.request(demand) case .cancelAndDiscard(let array, discard: _): diff --git a/Sources/Tetra/Combine/ExperimentalMapTask.swift b/Sources/Tetra/Combine/ExperimentalMapTask.swift index 11a0bf2..fcb2bc8 100644 --- a/Sources/Tetra/Combine/ExperimentalMapTask.swift +++ b/Sources/Tetra/Combine/ExperimentalMapTask.swift @@ -275,7 +275,10 @@ extension MultiMapTask { } } else { await simuateDiscardingTaskGroup(isolation: SafetyRegion()) { actor, group in - await localTask(isolation: actor, group: &group) + nonisolated(unsafe) + var unsafe = Suppress(value: group) + await localTask(isolation: actor, group: &unsafe.value) + group = unsafe.value } } // we assume no one is accessing other state diff --git a/Sources/Tetra/Combine/Publishers+AsyncFlatMap.swift b/Sources/Tetra/Combine/Publishers+AsyncFlatMap.swift index bcba29b..25fb48d 100644 --- a/Sources/Tetra/Combine/Publishers+AsyncFlatMap.swift +++ b/Sources/Tetra/Combine/Publishers+AsyncFlatMap.swift @@ -145,7 +145,10 @@ extension AsyncFlatMap { } else { try? await simuateThrowingDiscardingTaskGroup(isolation: SafetyRegion()) { barrier, group in defer { terminateStream() } - await localTask(isolation: barrier, group: &group) + nonisolated(unsafe) + var unsafe = Suppress(value: group) + await localTask(isolation: barrier, group: &unsafe.value) + group = unsafe.value } } send(completion: .finished, shouldCancel: false) diff --git a/Sources/Tetra/Concurrency/Notification+AsyncSequence.swift b/Sources/Tetra/Concurrency/Notification+AsyncSequence.swift index 159162e..004d5d5 100644 --- a/Sources/Tetra/Concurrency/Notification+AsyncSequence.swift +++ b/Sources/Tetra/Concurrency/Notification+AsyncSequence.swift @@ -82,19 +82,18 @@ public final class NotificationSequence: AsyncSequence, Sendable, TypedAsyncSequ ) { self.center = center let observer = center.addObserver(forName: name, object: object, queue: nil) { [lock] notification in - nonisolated(unsafe) - let noti2 = notification + let continuation = lock.withLockUnchecked { state in let captured = state.pending.first if state.pending.isEmpty { - state.buffer.append(noti2) + state.buffer.append(notification) } else { state.pending.removeFirst() } return captured } - continuation?.resume(returning: noti2) + continuation?.resume(returning: Suppress(value: notification).value) } lock.withLockUnchecked{ $0.observer = observer From 2b20d8012d8d6af3824681abe53e59ea5c40dc24 Mon Sep 17 00:00:00 2001 From: park-byeong-gwan Date: Mon, 15 Jul 2024 18:46:55 +0900 Subject: [PATCH 43/63] update readme and implment PruneMemory which clean up memory footprints --- README.md | 35 +++ .../Tetra/Concurrency/PriorityRunLoop.swift | 4 + Sources/Tetra/Foundation/PruneMemory.swift | 231 ++++++++++++++++++ 3 files changed, 270 insertions(+) create mode 100644 Sources/Tetra/Foundation/PruneMemory.swift diff --git a/README.md b/README.md index dbcbe9a..ba3e27a 100644 --- a/README.md +++ b/README.md @@ -190,3 +190,38 @@ struct ContentView: View { - more fine grained way to introduce extension methods (maybe something like `.af` in Alamofire?) - remove all `AsyncTypedSequence` and `WrappedAsyncSequence` dummy protocol when `FullTypedThrow` is implemented. + +## WIP + +[PriorityRunLoop](./Sources/Tetra/Concurrency/PriorityRunLoop.swift) + +TaskPriority aware NSRunLoop based `Concurrency Serial Executor`. + +Can be useful for old Apple Framework that does not support `libDispatch` and only support `CFRunLoop` like `CoreLocation`, `perform(_:on:with:waitUntilDone:)` + +Reorder the jobs in Priority Order, and increase QoS level, according to the TaskPriority. + + +[AsyncFlatMap](./Sources/Tetra/Combine/Publishers+AsyncFlatMap.swift) + +Combine operator which convert the Upstream to AsyncSequence and transfer the elements to the downstream. + +uses actor based cooperative control to minimize contention. + +Investingating for a way to backport rich asyncSequence features before Swift 6 platform. + + +[BackportDiscardingTaskGroup](./Sources/BackportDiscardingTaskGroup/ThrowingTaskGroup.swift) + +Backport the behavior of `Discarding(Throwing)TaskGroup`. This now possbile thanks to Swift 6 generalized AsyncSequnce. Because we can wrap the `TaskGroup` into isolation. + +Investingating for a way to merge it with existing `Discarding(Throwing)TaskGroup` using some kind of opaque types rather than erased types. + +Investigating a way to merge NonFailure type and Failing type using typedThrow, which is currently not possible due to Swift 6 compiler bug. + + +[MemorySafe Data/String](./Sources/Tetra/Foundation/PruneMemory.swift) + +`Data` and `String` which erase its memory footprints when deallocated. implemented by `CoreFoundation` API. + +Still can not erase memory footprints caused by Swift Briding copy and os level memory paging. diff --git a/Sources/Tetra/Concurrency/PriorityRunLoop.swift b/Sources/Tetra/Concurrency/PriorityRunLoop.swift index f0a3357..d79eab4 100644 --- a/Sources/Tetra/Concurrency/PriorityRunLoop.swift +++ b/Sources/Tetra/Concurrency/PriorityRunLoop.swift @@ -70,6 +70,10 @@ struct RunLoopPriorityQueue: ~Copyable, Sendable { } } + /* + Managing QoS, boost CPU instructions about 5% and decrease CPU cycles by 5% when root task is about `low` priority. and enqueing about 500 random priority tasks at the same time. + + */ var jobPriority = currentQos.evaluateTaskPriority() while let block = queue.popMax() { let job = block.jobImp diff --git a/Sources/Tetra/Foundation/PruneMemory.swift b/Sources/Tetra/Foundation/PruneMemory.swift new file mode 100644 index 0000000..4610edc --- /dev/null +++ b/Sources/Tetra/Foundation/PruneMemory.swift @@ -0,0 +1,231 @@ +// +// PruneString.swift +// +// +// Created by 박병관 on 7/8/24. +// + +import Foundation + + + +// since core foundation and objective-c foundation has it's internal optimzation +// it's hard to gurantee if memoryerasing is actullay applied. +// the only way to gurantee is to create mutable memoryerasing instance and fill it +// this method are pretty slow and not even a perfect choice. +// instances created by these methods are guranteed to fill zero to it's owned storage before deallocated. +enum MemoryErasing { + + + // create mutable String which is guarnteed to zero underlying memory when deinitialized + // this does not erase memory foot print caused by os level memory management such as paging + // briding to Swift type create implicit copy which does have memory foot print + static func createMutableString() -> NSMutableString { + + return CFStringCreateMutable(Self.allocator, 0) + } + + static func createMutableData() -> NSMutableData { + + return CFDataCreateMutable(Self.allocator, 0) + } + + + // this Data structure keeps its internal storage shared with Swift runtime + // so you can cast it to immutable Swift.Data, but do not cast to mutable Data + static func immutableData(_ source: T) -> NSData { + let fastPath = source.withContiguousStorageIfAvailable{ + return if let base = $0.baseAddress, $0.count > 0 { + CFDataCreate(allocator, base, $0.count)! + } else { + Data() as CFData + } + } + if let fastPath { + return fastPath + } + let mutable = CFDataCreateMutable(Self.allocator, 0) as NSMutableData + for page in source.regions { + mutable.increaseLength(by: page.count) + page.withUnsafeBytes{ + if let base = $0.baseAddress, $0.count > 0 { + mutable.append(base, length: $0.count) + } + } + } + return CFDataCreateCopy(allocator, mutable) + } + + static func immutableString( + decoding source: C, + as SourceEncoding: T.Type = T.self + ) -> NSString where C.Element == T.CodeUnit { + if source.isEmpty { + return "" + } + + let fastPath = source.withContiguousStorageIfAvailable{ + + if SourceEncoding == UTF8.self { + return $0.withMemoryRebound(to: UTF8.CodeUnit.self) { buffer in + if buffer.count < 14, buffer.allSatisfy(Unicode.UTF8.isASCII) { + return CFStringCreateWithBytes(allocator, buffer.baseAddress!, buffer.count, CFStringBuiltInEncodings.nonLossyASCII.rawValue, false)! as CFString? + } + return CFStringCreateWithBytes(allocator, buffer.baseAddress!, buffer.count, CFStringBuiltInEncodings.UTF8.rawValue, false)! as CFString? + } + } + if SourceEncoding == UTF16.self { + return $0.withMemoryRebound(to: UInt8.self) { buffer in + CFStringCreateWithBytes(allocator, buffer.baseAddress!, buffer.count, CFStringBuiltInEncodings.UTF16.rawValue, false)! as CFString? + + } + } + if SourceEncoding == UTF32.self { + + return $0.withMemoryRebound(to: UInt8.self) { buffer in + var normal = CFStringCreateWithBytes(allocator, buffer.baseAddress!, buffer.count, CFStringBuiltInEncodings.UTF32LE.rawValue, false) + if normal == nil { + normal = CFStringCreateWithBytes(allocator, buffer.baseAddress!, buffer.count, CFStringBuiltInEncodings.UTF32.rawValue, false) + } + return normal + } + } + return nil + } + if let fastPath, let fastPath { + return fastPath + } + + + let mutable = CFDataCreateMutable(allocator, 0) as NSMutableData + let deallocator:CFAllocator + do { + var context = CFAllocatorContext() + context.info = Unmanaged.passUnretained(mutable).toOpaque() + context.retain = { + let ptr = Unmanaged.fromOpaque($0!).retain().toOpaque() + return .init(ptr) + } + context.release = { + Unmanaged.fromOpaque($0!).release() + } + deallocator = CFAllocatorCreate(nil, &context).takeRetainedValue() + } + var encoding:CFStringBuiltInEncodings = .nonLossyASCII + let _ = transcode(source.makeIterator(), from: SourceEncoding, to: UTF8.self, stoppingOnError: false) { code in + if !Unicode.ASCII.isASCII(code) { + encoding = .UTF8 + } + withUnsafePointer(to: code) { + CFDataAppendBytes(mutable, $0, 1) + } + } + + + let msg = CFStringCreateWithBytesNoCopy(nil, mutable.bytes.assumingMemoryBound(to: UInt8.self), mutable.length, encoding.rawValue, false, deallocator)! + return msg + } + + + static func immutableString(_ reduce: (NSMutableString) -> Void) -> NSString { + let source = CFStringCreateMutable(allocator, 0)! as NSMutableString + + reduce(source) + + if source.length == 0 { + // This will return singleton empty string + return NSString() + } + if CFStringGetLength(source) < __kCFStringInlineBufferLength { + let encoding = CFStringGetFastestEncoding(source) + if encoding != kCFStringEncodingInvalidId { + var buffer = CFStringInlineBuffer() + CFStringInitInlineBuffer(source, &buffer, .init(location: 0, length: CFStringGetLength(source))) + if encoding == CFStringBuiltInEncodings.ASCII.rawValue { + return CFStringCreateWithCString(allocator, buffer.directCStringBuffer, CFStringBuiltInEncodings.nonLossyASCII.rawValue) + } + return CFStringCreateWithCString(allocator, buffer.directCStringBuffer, encoding) + } + } + return CFStringCreateCopy(allocator, source) + } + + + static func createMutableAttributeString() -> NSMutableAttributedString { + + return CFAttributedStringCreateMutable(allocator, 0) + } + + static func immutableAttributeString( + _ reduce: (NSMutableAttributedString) -> Void + ) -> NSAttributedString { + let source = CFAttributedStringCreateMutable(allocator, 0)! + reduce(source) + return CFAttributedStringCreateCopy(allocator, source) + } + + static func createUInt16BufferDeallocator(length: Int) -> CFAllocator { + var context = CFAllocatorContext() + context.deallocate = { ptr, info in + guard let ptr else { return } + let count = Int(bitPattern: info!) + let base = ptr.assumingMemoryBound(to: UniChar.self) + let buffer = UnsafeMutableBufferPointer(start: base, count: count) + buffer.update(repeating: 0) + buffer.deinitialize().deallocate() + } + context.info = .init(bitPattern: length) + context.copyDescription = { _ in + .passRetained("NoCopyNSStringDeallocator" as CFString) + } + let deallocator = CFAllocatorCreate(nil, &context).takeRetainedValue() + return deallocator + } + + + + nonisolated(unsafe) + static let allocator:CFAllocator = { + var context = CFAllocatorContext() + context.allocate = { size, hint, _ in + return malloc(size) + } + context.info = nil + context.retain = nil + context.release = nil + context.deallocate = { ptr, _ in + if let ptr { + let size = malloc_size(ptr) + // memset_s ignore compiler optimization + memset_s(ptr, size, 0, size) + free(ptr) + print("free") + } + } + context.copyDescription = { _ in + return .passRetained("ZeroingCFAllocator" as CFString) + } + context.preferredSize = { size, flag, _ in + return malloc_good_size(size) + } + context.reallocate = { ptr, newSize, flag, _ in + guard let ptr, newSize > 0 else { + return nil + } + let old_size = malloc_size(ptr) + if old_size > newSize { + let diff = old_size - newSize + memset(ptr.advanced(by: newSize), 0, diff) + } + return realloc(ptr, newSize) + } + + let allocatorRef = CFAllocatorCreate(nil, &context) + return allocatorRef!.takeRetainedValue() + }() + + + + +} + From 8a46f77dca4938c0e75c45169bb527653c2d6f38 Mon Sep 17 00:00:00 2001 From: park-byeong-gwan Date: Wed, 17 Jul 2024 11:54:35 +0900 Subject: [PATCH 44/63] adpat new typedThrow syntax and refactor simulating DiscardTaskGroup --- .../TaskGroup.swift | 41 +++++------------ .../ThrowingTaskGroup.swift | 46 ++++++------------- .../Tetra/Combine/ExperimentalMapTask.swift | 18 ++++---- .../Combine/Publishers+AsyncFlatMap.swift | 33 ++++++------- .../CoreDataStack+Concurrency.swift | 29 ++++++------ Sources/Tetra/Foundation/Mics.swift | 25 ---------- 6 files changed, 59 insertions(+), 133 deletions(-) diff --git a/Sources/BackportDiscardingTaskGroup/TaskGroup.swift b/Sources/BackportDiscardingTaskGroup/TaskGroup.swift index fcf76da..8c29ba5 100644 --- a/Sources/BackportDiscardingTaskGroup/TaskGroup.swift +++ b/Sources/BackportDiscardingTaskGroup/TaskGroup.swift @@ -5,28 +5,23 @@ // Created by 박병관 on 6/20/24. // -extension TaskGroup where ChildTaskResult == Void { - - /// work around for simulating Discarding TaskGroup - /// - /// TaskGroup is protected by the actor isolation - /// - important: always call TaskGroup api while holding isolation - @usableFromInline - internal mutating func simulateDiscarding( - isolation actor: isolated T, - body: (isolated T, inout Self) async -> sending V - ) async -> V { +@inlinable +package func simuateDiscardingTaskGroup( + isolation actor: isolated T = #isolation, + body: @Sendable (isolated T, inout TaskGroup) async -> sending TaskResult +) async -> sending TaskResult { + return await withTaskGroup(of: Void.self, returning: TaskResult.self) { group in let holder: SafetyRegion = actor as? SafetyRegion ?? .init() if await holder.isFinished { preconditionFailure("SafetyRegion is already used!") } - addTask(priority: .background) { + group.addTask(priority: .background) { /// keep at least one child task alive /// so that subTask won't return await holder.hold() } - let suppress = Suppress(base: self) + let suppress = Suppress(base: group) /// drain all the finished or failed Task async let subTask:Void = { var iter = suppress.base @@ -36,28 +31,16 @@ extension TaskGroup where ChildTaskResult == Void { } } }() - nonisolated(unsafe) - let block = body + async let mainTask = { var iter = suppress.base - let v = await block(actor, &iter) + let v = await body(actor, &iter) await holder.markDone() return Suppress(base: v) }() await subTask return await mainTask.base } - -} - -@inlinable -package func simuateDiscardingTaskGroup( - isolation actor: isolated T = #isolation, - body: @Sendable (isolated T, inout TaskGroup) async -> sending TaskResult -) async -> sending TaskResult { - await withTaskGroup(of: Void.self, returning: TaskResult.self) { - await $0.simulateDiscarding(isolation: actor, body: body) - } } @@ -82,7 +65,7 @@ package func simuateDiscardingTaskGroup( /// - SeeAlso: withDiscardingTaskGroup(returning:body:) @inlinable package func simuateDiscardingTaskGroup( - body: @Sendable @isolated(any) (inout TaskGroup) async -> sending TaskResult + body: @isolated(any) (inout TaskGroup) async -> sending TaskResult ) async -> sending TaskResult { guard let actor = body.isolation else { preconditionFailure("body must be isolated") @@ -91,7 +74,7 @@ package func simuateDiscardingTaskGroup( precondition(actor === region, "must be isolated on the inferred actor") nonisolated(unsafe) var unsafe = Suppress(base: group) - let value = await body(&unsafe.base) + let value = await body(&unsafe.base) group = unsafe.base return value } diff --git a/Sources/BackportDiscardingTaskGroup/ThrowingTaskGroup.swift b/Sources/BackportDiscardingTaskGroup/ThrowingTaskGroup.swift index 978addf..2fd2f40 100644 --- a/Sources/BackportDiscardingTaskGroup/ThrowingTaskGroup.swift +++ b/Sources/BackportDiscardingTaskGroup/ThrowingTaskGroup.swift @@ -1,32 +1,26 @@ // // ThrowingTaskGroup.swift -// +// // // Created by 박병관 on 6/20/24. // - -extension ThrowingTaskGroup where ChildTaskResult == Void, Failure == any Error { - - /// work around for simulating Discarding TaskGroup - /// - /// TaskGroup is protected by the actor isolation - /// - important: always call TaskGroup api while holding isolation - @usableFromInline - internal mutating func simulateDiscarding( - isolation actor: isolated T, - body: (isolated T, inout Self) async throws -> sending V - ) async throws -> V { +@inlinable +package func simuateThrowingDiscardingTaskGroup( + isolation actor: isolated T, + body: @Sendable (isolated T, inout ThrowingTaskGroup) async throws -> sending TaskResult +) async throws -> sending TaskResult { + return try await withThrowingTaskGroup(of: Void.self, returning: TaskResult.self) { group in let holder: SafetyRegion = actor as? SafetyRegion ?? .init() if await holder.isFinished { preconditionFailure("SafetyRegion is already used!") } - addTask(priority: .background) { + group.addTask(priority: .background) { /// keep at least one child task alive /// so that subTask won't return await holder.hold() } - let suppress = Suppress(base: self) + let suppress = Suppress(base: group) /// drain all the finished or failed Task async let subTask:Void = { var iter = suppress.base @@ -36,12 +30,10 @@ extension ThrowingTaskGroup where ChildTaskResult == Void, Failure == any Error } } }() - nonisolated(unsafe) - let block = body async let mainTask = { + var iter = suppress.base do { - var iter = suppress.base - let v = try await block(actor, &iter) + let v = try await body(actor, &iter) await holder.markDone() return Suppress(base: v) } catch { @@ -56,6 +48,7 @@ extension ThrowingTaskGroup where ChildTaskResult == Void, Failure == any Error try await subTask errorRef = nil } catch { + group.cancelAll() errorRef = error } let value = try await mainTask.base @@ -66,20 +59,9 @@ extension ThrowingTaskGroup where ChildTaskResult == Void, Failure == any Error } } -@inlinable -package func simuateThrowingDiscardingTaskGroup( - isolation actor: isolated T, - body: @Sendable (isolated T, inout ThrowingTaskGroup) async throws -> sending TaskResult -) async throws -> sending TaskResult { - try await withThrowingTaskGroup(of: Void.self, returning: TaskResult.self) { - try await $0.simulateDiscarding(isolation: actor, body: body) - } -} - - @inlinable package func simuateThrowingDiscardingTaskGroup( - body: @Sendable @isolated(any) (inout ThrowingTaskGroup) async throws -> sending TaskResult + body: @isolated(any) (inout ThrowingTaskGroup) async throws -> sending TaskResult ) async throws -> sending TaskResult { guard let actor = body.isolation else { preconditionFailure("body must be isolated") @@ -98,5 +80,3 @@ package func simuateThrowingDiscardingTaskGroup( } } } - - diff --git a/Sources/Tetra/Combine/ExperimentalMapTask.swift b/Sources/Tetra/Combine/ExperimentalMapTask.swift index fcb2bc8..2ac2f1a 100644 --- a/Sources/Tetra/Combine/ExperimentalMapTask.swift +++ b/Sources/Tetra/Combine/ExperimentalMapTask.swift @@ -132,20 +132,18 @@ extension MultiMapTask { break case .success(let success): let flag = group.addTaskUnlessCancelled(priority: nil) { - let result = await wrapToResult(consume success, transform) - switch result { - case .failure(let error): - await barrier.markDone() - // no contention except `request` and `cancel` - await send(barrier: barrier, completion: .failure(error), cancel: true) - case .success(let success): + do throws(Failure) { + let value = try await transform(success) do { // no contention except `request` and `cancel` - try await send(isolation: barrier, success) + try await send(isolation: barrier, value) } catch { await barrier.markDone() -// token.store(true, ordering: .releasing) } + } catch { + await barrier.markDone() + // no contention except `request` and `cancel` + await send(barrier: barrier, completion: .failure(error), cancel: true) } } if !flag { @@ -274,7 +272,7 @@ extension MultiMapTask { ) } } else { - await simuateDiscardingTaskGroup(isolation: SafetyRegion()) { actor, group in + await simuateDiscardingTaskGroup(isolation: SafetyRegion()) { @Sendable actor, group in nonisolated(unsafe) var unsafe = Suppress(value: group) await localTask(isolation: actor, group: &unsafe.value) diff --git a/Sources/Tetra/Combine/Publishers+AsyncFlatMap.swift b/Sources/Tetra/Combine/Publishers+AsyncFlatMap.swift index 25fb48d..24fb6b9 100644 --- a/Sources/Tetra/Combine/Publishers+AsyncFlatMap.swift +++ b/Sources/Tetra/Combine/Publishers+AsyncFlatMap.swift @@ -367,25 +367,19 @@ extension AsyncFlatMap { barrier: some Actor ) async -> Bool { let result:Result? - do { + do throws(Failure) { + let value = try await iterator.next(isolation: nil) + if let value = try await iterator.next(isolation: nil) { - result = .success(value) + await handleDownStream(isolation: barrier, event: .success(.init(value: value))) + return true } else { - result = nil + await handleDownStream(isolation: barrier, event: .success(.none)) + return false } } catch { - result = .failure(error) - } - switch result { - case .none: - await handleDownStream(isolation: barrier, event: .success(.none)) - return false - case .failure(let error): await handleDownStream(isolation: barrier, event: .failure(error)) return false - case .success(let value): - await handleDownStream(isolation: barrier, event: .success(.init(value: value))) - return true } } @@ -407,19 +401,18 @@ extension AsyncFlatMap { return case .success(let value): let isSuccess = group.addTaskUnlessCancelled(priority: nil) { - let segmentResult = await makeSegment(value) - var iterator:Segment.AsyncIterator - switch segmentResult { - case .failure(let failure): + let segment:Segment + do throws(Failure) { + segment = try await transform(value) + } catch { await barrier.markDone() await handleDownStream( isolation: barrier, - event: .failure(failure) + event: .failure(error) ) return - case .success(let source): - iterator = source.makeAsyncIterator() } + var iterator = segment.makeAsyncIterator() while true { let isUnlimited = try await nextDemand(barrier: barrier) if isUnlimited { diff --git a/Sources/Tetra/Concurrency/CoreDataStack+Concurrency.swift b/Sources/Tetra/Concurrency/CoreDataStack+Concurrency.swift index aba1a83..926c3b3 100644 --- a/Sources/Tetra/Concurrency/CoreDataStack+Concurrency.swift +++ b/Sources/Tetra/Concurrency/CoreDataStack+Concurrency.swift @@ -53,7 +53,11 @@ extension TetraExtension where Base: NSPersistentStoreCoordinator { internal func _performAndWait(_ body: () throws(Failure) -> T) throws(Failure) -> T { var result:Result? = nil base.performAndWait { - result = wrapToResult(body) + do throws(Failure) { + result = .success(try body()) + } catch { + result = .failure(error) + } } guard let result else { preconditionFailure("performAndWait didn't run") @@ -106,6 +110,7 @@ extension TetraExtension where Base: NSManagedObjectContext { _ body: () throws(Failure) -> T ) async throws(Failure) -> T { let result: Result = await withoutActuallyEscaping(body) { escapingClosure in + let holder = ClosureHolder(closure: escapingClosure) defer { withExtendedLifetime(holder, {}) @@ -113,8 +118,7 @@ extension TetraExtension where Base: NSManagedObjectContext { return await withUnsafeContinuation { continuation in base.perform{ [unowned holder, continuation] in - nonisolated(unsafe) - let result = wrapToResult(holder.closure) + let result = holder() continuation.resume(returning: result) } } @@ -164,7 +168,11 @@ extension TetraExtension where Base: NSManagedObjectContext { internal func _performAndWait(_ body: () throws(Failure) -> T) throws(Failure) -> T { var result:Result? = nil base.performAndWait { - result = wrapToResult(body) + do throws(Failure) { + result = .success(try body()) + } catch { + result = .failure(error) + } } guard let result else { preconditionFailure("performAndWait didn't run") @@ -200,16 +208,6 @@ extension TetraExtension where Base: NSPersistentContainer { } } - @inline(__always) - @usableFromInline - internal func _convertToResult(_ context:NSManagedObjectContext, _ body: (NSManagedObjectContext) throws(Failure) -> T) -> Result { - do { - let value = try body(context) - return .success(value) - } catch { - return .failure(error) - } - } @usableFromInline @@ -222,7 +220,7 @@ extension TetraExtension where Base: NSPersistentContainer { return await withUnsafeContinuation { continuation in base.performBackgroundTask { [unowned holder, continuation] newContext in nonisolated(unsafe) - let result = _convertToResult(newContext, holder.closure) + let result = holder(newContext) continuation.resume(returning: result) } } @@ -252,4 +250,3 @@ public enum CoreDataScheduledTaskType: Sendable, Hashable { } #endif - diff --git a/Sources/Tetra/Foundation/Mics.swift b/Sources/Tetra/Foundation/Mics.swift index 4344c82..6abc58b 100644 --- a/Sources/Tetra/Foundation/Mics.swift +++ b/Sources/Tetra/Foundation/Mics.swift @@ -37,28 +37,3 @@ internal extension NSNumber { } - -@inline(__always) -@usableFromInline -internal -func wrapToResult(_ block: () throws(Failure) -> T) -> Result { - do { - return .success(try block()) - } catch { - return .failure(error) - } -} - - -@inline(__always) -@usableFromInline -internal func wrapToResult( - _ value: consuming T, _ transform: (consuming T) async throws(Failure) -> sending U -) async -> sending Result { - do { - let success = try await transform(value) - return .success(success) - } catch { - return .failure(error) - } -} From 0fb2d871c3b1c7246568961b73fddf61bd8e7aba Mon Sep 17 00:00:00 2001 From: pbk Date: Thu, 19 Dec 2024 10:25:59 +0900 Subject: [PATCH 45/63] resolve for swift6 --- Package.resolved | 11 ++- Package.swift | 12 +-- .../AsyncCompactMapSequence.swift | 14 +++- .../AsyncDropWhileSequence.swift | 2 +- .../AsyncFilterSequence.swift | 2 +- .../AsyncFlatMapSequence.swift | 14 ++-- .../AsyncMapSequence.swift | 16 +++- .../AsyncPrefixWhileSequence.swift | 2 +- .../BackPortAsyncSequence/AsyncStream.swift | 8 +- .../AsyncThrowingStream.swift | 7 +- .../LegacyTypedAsyncSequence.swift | 7 +- .../TaskGroup.swift | 11 ++- .../ThrowingTaskGroup.swift | 15 +++- Sources/CriticalSection/Cell.swift | 9 ++- Sources/CriticalSection/DarwinImpl.swift | 42 ++++++++++ .../Combine/AsyncSubscriptionState.swift | 6 +- .../Tetra/Combine/CompatAsyncPublisher.swift | 2 +- .../CompatAsyncThrowingPublisher.swift | 2 +- .../Tetra/Combine/ExperimentalMapTask.swift | 12 +-- .../Tetra/Combine/Future+Concurrency.swift | 5 +- .../Combine/Publishers+AsyncFlatMap.swift | 4 +- .../Tetra/Combine/Publishers+MapTask.swift | 2 +- .../Concurrency/AsyncSequencePublisher.swift | 6 +- .../CoreDataStack+Concurrency.swift | 79 ++++++++++--------- .../Concurrency/Dispatch+Extension.swift | 2 +- .../Notification+AsyncSequence.swift | 2 +- Sources/Tetra/Foundation/PruneMemory.swift | 1 - .../Tetra/SwiftUI/Binding+Collection.swift | 6 +- 28 files changed, 198 insertions(+), 103 deletions(-) create mode 100644 Sources/CriticalSection/DarwinImpl.swift diff --git a/Package.resolved b/Package.resolved index 390b1ac..bb910e4 100644 --- a/Package.resolved +++ b/Package.resolved @@ -1,6 +1,15 @@ { - "originHash" : "d62b4e9de455d0219ff1f8b99cc562ae9734c60cbf7d73973fd2c627c957a639", + "originHash" : "2fc989cc1b67d91eb25b0319fdf3b974c0dd91e42699a64bf0c693dd7dde9fab", "pins" : [ + { + "identity" : "swift-atomics", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-atomics.git", + "state" : { + "revision" : "cd142fd2f64be2100422d658e7411e39489da985", + "version" : "1.2.0" + } + }, { "identity" : "swift-collections", "kind" : "remoteSourceControl", diff --git a/Package.swift b/Package.swift index e225e50..202f3bb 100644 --- a/Package.swift +++ b/Package.swift @@ -36,7 +36,7 @@ let package = Package( .target( name: "Namespace", swiftSettings: [ - .swiftLanguageVersion(.v6) + .swiftLanguageMode(.v6) ] ), .target( @@ -45,7 +45,7 @@ let package = Package( "Namespace", ], swiftSettings: [ - .swiftLanguageVersion(.v6) + .swiftLanguageMode(.v6) ] ), .target( @@ -55,7 +55,7 @@ let package = Package( ], swiftSettings: [ - .swiftLanguageVersion(.v6), + .swiftLanguageMode(.v6), .enableExperimentalFeature("StaticExclusiveOnly"), .enableExperimentalFeature("RawLayout"), .enableExperimentalFeature("BuiltinModule"), @@ -69,7 +69,7 @@ let package = Package( swiftSettings: [ .enableUpcomingFeature("FullTypedThrows"), .enableExperimentalFeature("IsolatedAny"), - .swiftLanguageVersion(.v6) + .swiftLanguageMode(.v6) ] ), .target( @@ -87,14 +87,14 @@ let package = Package( swiftSettings: [ .enableUpcomingFeature("FullTypedThrows"), .enableExperimentalFeature("IsolatedAny"), - .swiftLanguageVersion(.v6) + .swiftLanguageMode(.v6) ] ), .target( name: "BackPortAsyncSequence", dependencies: [ "Namespace"], swiftSettings: [ - .swiftLanguageVersion(.v6), + .swiftLanguageMode(.v6), ] ), .testTarget( diff --git a/Sources/BackPortAsyncSequence/AsyncCompactMapSequence.swift b/Sources/BackPortAsyncSequence/AsyncCompactMapSequence.swift index b707d3a..7c8d327 100644 --- a/Sources/BackPortAsyncSequence/AsyncCompactMapSequence.swift +++ b/Sources/BackPortAsyncSequence/AsyncCompactMapSequence.swift @@ -48,7 +48,7 @@ extension BackPort.AsyncCompactMapSequence: AsyncSequence, TypedAsyncSequence { var baseIterator: Base.AsyncIterator @usableFromInline - let transform: (Base.Element) async throws(Failure) -> sending ElementOfResult? + let transform: (Base.Element) async throws(Failure) -> ElementOfResult? @usableFromInline var finished = false @@ -93,8 +93,16 @@ extension BackPort.AsyncCompactMapSequence.Iterator: AsyncIteratorProtocol, Type finished = true return nil } - do { - if let transformed = try await transform(Suppress(base: element).base) { + let wrapper: (Base.Element) async -> Result, Failure> = { [transform] in + do throws(Base.AsyncIterator.Err) { + let value = try await Suppress(base: transform($0)) + return .success(value) + } catch { + return .failure(error) + } + } + do throws(Failure) { + if let transformed = try await wrapper(Suppress(base: element).base).get().base { return transformed } } catch { diff --git a/Sources/BackPortAsyncSequence/AsyncDropWhileSequence.swift b/Sources/BackPortAsyncSequence/AsyncDropWhileSequence.swift index c3742f8..e5f28aa 100644 --- a/Sources/BackPortAsyncSequence/AsyncDropWhileSequence.swift +++ b/Sources/BackPortAsyncSequence/AsyncDropWhileSequence.swift @@ -93,7 +93,7 @@ extension BackPort.AsyncDropWhileSequence.Iterator: AsyncIteratorProtocol, Typed guard let element = try await baseIterator.next(isolation: actor) else { return nil } - do { + do throws(Failure) { if try await predicate(Suppress(base: element).base) == false { doneDropping = true diff --git a/Sources/BackPortAsyncSequence/AsyncFilterSequence.swift b/Sources/BackPortAsyncSequence/AsyncFilterSequence.swift index 744537c..522421d 100644 --- a/Sources/BackPortAsyncSequence/AsyncFilterSequence.swift +++ b/Sources/BackPortAsyncSequence/AsyncFilterSequence.swift @@ -78,7 +78,7 @@ extension BackPort.AsyncFilterSequence.Iterator: AsyncIteratorProtocol, TypedAsy guard let element = try await baseIterator.next(isolation: actor) else { return nil } - do { + do throws(Failure) { if try await isIncluded(Suppress(base: element).base) { return element } diff --git a/Sources/BackPortAsyncSequence/AsyncFlatMapSequence.swift b/Sources/BackPortAsyncSequence/AsyncFlatMapSequence.swift index 449bedd..731bd5f 100644 --- a/Sources/BackPortAsyncSequence/AsyncFlatMapSequence.swift +++ b/Sources/BackPortAsyncSequence/AsyncFlatMapSequence.swift @@ -47,7 +47,7 @@ extension BackPort.AsyncFlatMapSequence: AsyncSequence, TypedAsyncSequence { var baseIterator: Base.AsyncIterator @usableFromInline - let transform: (Base.Element) async throws(Failure) -> sending SegmentOfResult + let transform: (Base.Element) async throws(Failure) -> SegmentOfResult @usableFromInline var currentIterator: SegmentOfResult.AsyncIterator? @@ -58,7 +58,7 @@ extension BackPort.AsyncFlatMapSequence: AsyncSequence, TypedAsyncSequence { @usableFromInline init( baseIterator: Base.AsyncIterator, - transform: @escaping (Base.Element) async throws(Failure) -> sending SegmentOfResult + transform: @escaping (Base.Element) async throws(Failure) -> SegmentOfResult ) { self.baseIterator = baseIterator self.transform = transform @@ -96,9 +96,13 @@ extension BackPort.AsyncFlatMapSequence.Iterator: AsyncIteratorProtocol, TypedAs guard let item = try await baseIterator.next(isolation: actor) else { return nil } - let segment: SegmentOfResult + let block = transform + let wrapper = { + let a = try await block($0) + return Suppress(base: a) + } do { - segment = try await transform(Suppress(base: item).base) + let segment: SegmentOfResult = try await wrapper(Suppress(base: item).base).base var iterator = segment.makeAsyncIterator() guard let element = try await iterator.next(isolation: actor) else { currentIterator = nil @@ -109,7 +113,7 @@ extension BackPort.AsyncFlatMapSequence.Iterator: AsyncIteratorProtocol, TypedAs } catch { finished = true currentIterator = nil - throw error + throw error as! Failure } } } diff --git a/Sources/BackPortAsyncSequence/AsyncMapSequence.swift b/Sources/BackPortAsyncSequence/AsyncMapSequence.swift index 4f3a9a4..d73f678 100644 --- a/Sources/BackPortAsyncSequence/AsyncMapSequence.swift +++ b/Sources/BackPortAsyncSequence/AsyncMapSequence.swift @@ -49,7 +49,7 @@ extension BackPort.AsyncMapSequence: AsyncSequence, TypedAsyncSequence { var finished = false @usableFromInline - let transform: (Base.Element) async throws(Failure) -> sending Transformed + let transform: (Base.Element) async throws(Failure) -> Transformed @usableFromInline init( @@ -95,8 +95,18 @@ extension BackPort.AsyncMapSequence.Iterator: AsyncIteratorProtocol, TypedAsyncI guard !finished, let element = try await baseIterator.next(isolation: actor) else { return nil } - do { - return try await transform(Suppress(base: element).base) +// let block = transform + let wrapper: (Base.Element) async -> Result, Failure> = { [transform] in + do throws(Base.AsyncIterator.Err) { + let value = try await Suppress(base: transform($0)) + return .success(value) + } catch { + return .failure(error) + } + } + do throws(Failure) { + + return try await wrapper(Suppress(base: element).base).get().base } catch { finished = true throw error diff --git a/Sources/BackPortAsyncSequence/AsyncPrefixWhileSequence.swift b/Sources/BackPortAsyncSequence/AsyncPrefixWhileSequence.swift index f7e30c2..937dc4b 100644 --- a/Sources/BackPortAsyncSequence/AsyncPrefixWhileSequence.swift +++ b/Sources/BackPortAsyncSequence/AsyncPrefixWhileSequence.swift @@ -103,7 +103,7 @@ extension BackPort.AsyncPrefixWhileSequence.Iterator: AsyncIteratorProtocol, Typ @inlinable public mutating func next(isolation actor: isolated (any Actor)? = #isolation) async throws(Failure) -> Base.Element? { if !predicateHasFailed, let nextElement = try await baseIterator.next(isolation: actor) { - do { + do throws(Failure) { if try await predicate(Suppress(base: nextElement).base) { return nextElement } else { diff --git a/Sources/BackPortAsyncSequence/AsyncStream.swift b/Sources/BackPortAsyncSequence/AsyncStream.swift index 3c707ba..7b4adb6 100644 --- a/Sources/BackPortAsyncSequence/AsyncStream.swift +++ b/Sources/BackPortAsyncSequence/AsyncStream.swift @@ -51,7 +51,7 @@ extension AsyncTypedStream.Iterator: AsyncIteratorProtocol, TypedAsyncIteratorPr } else { nonisolated(unsafe) var iter = self - let value = await iter.advanceNext() + let value = await iter.advanceNext()?.base self = iter return value } @@ -65,8 +65,10 @@ extension AsyncTypedStream.Iterator: AsyncIteratorProtocol, TypedAsyncIteratorPr @inline(__always) @usableFromInline - internal mutating func advanceNext() async -> sending Element? { - await baseIterator.next() + @preconcurrency + internal mutating func advanceNext() async -> Suppress? { + guard let value = await baseIterator.next() else { return nil } + return .init(base: value) } } diff --git a/Sources/BackPortAsyncSequence/AsyncThrowingStream.swift b/Sources/BackPortAsyncSequence/AsyncThrowingStream.swift index 7727868..2f74c4c 100644 --- a/Sources/BackPortAsyncSequence/AsyncThrowingStream.swift +++ b/Sources/BackPortAsyncSequence/AsyncThrowingStream.swift @@ -52,7 +52,7 @@ extension AsyncTypedThrowingStream.Iterator: AsyncIteratorProtocol, TypedAsyncIt nonisolated(unsafe) var iter = self do { - let value = try await iter.nextValue() + let value = try await iter.nextValue()?.base self = iter return value } catch { @@ -70,8 +70,9 @@ extension AsyncTypedThrowingStream.Iterator: AsyncIteratorProtocol, TypedAsyncIt @inline(__always) @usableFromInline - internal mutating func nextValue() async throws -> sending Element? { - try await baseIterator.next() + internal mutating func nextValue() async throws -> Suppress? { + guard let value = try await baseIterator.next() else { return nil } + return .init(base: value) } } diff --git a/Sources/BackPortAsyncSequence/LegacyTypedAsyncSequence.swift b/Sources/BackPortAsyncSequence/LegacyTypedAsyncSequence.swift index 6181ee0..a6d9cc1 100644 --- a/Sources/BackPortAsyncSequence/LegacyTypedAsyncSequence.swift +++ b/Sources/BackPortAsyncSequence/LegacyTypedAsyncSequence.swift @@ -60,7 +60,7 @@ extension LegacyTypedAsyncSequence.Iterator: AsyncIteratorProtocol, TypedAsyncIt nonisolated(unsafe) var iter = self do { - let value = try await iter.advance() + let value = try await iter.advance()?.base self = iter return value } catch { @@ -78,8 +78,9 @@ extension LegacyTypedAsyncSequence.Iterator: AsyncIteratorProtocol, TypedAsyncIt @inline(__always) @usableFromInline - internal mutating func advance() async throws(Failure) -> sending Element? { - try await baseIterator.next() + internal mutating func advance() async throws(Failure) -> Suppress? { + guard let value = try await baseIterator.next() else { return nil } + return .init(base: value) } diff --git a/Sources/BackportDiscardingTaskGroup/TaskGroup.swift b/Sources/BackportDiscardingTaskGroup/TaskGroup.swift index 8c29ba5..f75d4dd 100644 --- a/Sources/BackportDiscardingTaskGroup/TaskGroup.swift +++ b/Sources/BackportDiscardingTaskGroup/TaskGroup.swift @@ -10,7 +10,7 @@ package func simuateDiscardingTaskGroup( isolation actor: isolated T = #isolation, body: @Sendable (isolated T, inout TaskGroup) async -> sending TaskResult ) async -> sending TaskResult { - return await withTaskGroup(of: Void.self, returning: TaskResult.self) { group in + let wrapped = await withTaskGroup(of: Void.self, returning: Suppress.self) { group in let holder: SafetyRegion = actor as? SafetyRegion ?? .init() if await holder.isFinished { preconditionFailure("SafetyRegion is already used!") @@ -24,6 +24,7 @@ package func simuateDiscardingTaskGroup( let suppress = Suppress(base: group) /// drain all the finished or failed Task async let subTask:Void = { + nonisolated(unsafe) var iter = suppress.base while let _ = await iter.next(isolation: actor) { if await holder.isFinished { @@ -39,8 +40,10 @@ package func simuateDiscardingTaskGroup( return Suppress(base: v) }() await subTask - return await mainTask.base + let value = await mainTask.base + return Suppress(base: value) } + return wrapped.base } @@ -70,11 +73,13 @@ package func simuateDiscardingTaskGroup( guard let actor = body.isolation else { preconditionFailure("body must be isolated") } + nonisolated(unsafe) + let block = body return await simuateDiscardingTaskGroup(isolation: actor) { region, group in precondition(actor === region, "must be isolated on the inferred actor") nonisolated(unsafe) var unsafe = Suppress(base: group) - let value = await body(&unsafe.base) + let value = await block(&unsafe.base) group = unsafe.base return value } diff --git a/Sources/BackportDiscardingTaskGroup/ThrowingTaskGroup.swift b/Sources/BackportDiscardingTaskGroup/ThrowingTaskGroup.swift index 2fd2f40..200ef73 100644 --- a/Sources/BackportDiscardingTaskGroup/ThrowingTaskGroup.swift +++ b/Sources/BackportDiscardingTaskGroup/ThrowingTaskGroup.swift @@ -10,7 +10,7 @@ package func simuateThrowingDiscardingTaskGroup( isolation actor: isolated T, body: @Sendable (isolated T, inout ThrowingTaskGroup) async throws -> sending TaskResult ) async throws -> sending TaskResult { - return try await withThrowingTaskGroup(of: Void.self, returning: TaskResult.self) { group in + let wrapped:Suppress = try await withThrowingTaskGroup(of: Void.self, returning: Suppress.self, isolation: actor) { group in let holder: SafetyRegion = actor as? SafetyRegion ?? .init() if await holder.isFinished { preconditionFailure("SafetyRegion is already used!") @@ -23,6 +23,7 @@ package func simuateThrowingDiscardingTaskGroup( let suppress = Suppress(base: group) /// drain all the finished or failed Task async let subTask:Void = { + nonisolated(unsafe) var iter = suppress.base while let _ = try await iter.next(isolation: actor) { if await holder.isFinished { @@ -31,6 +32,7 @@ package func simuateThrowingDiscardingTaskGroup( } }() async let mainTask = { + nonisolated(unsafe) var iter = suppress.base do { let v = try await body(actor, &iter) @@ -51,27 +53,32 @@ package func simuateThrowingDiscardingTaskGroup( group.cancelAll() errorRef = error } + nonisolated(unsafe) let value = try await mainTask.base if let errorRef { throw errorRef } - return value + return Suppress(base: value) } + return wrapped.base } @inlinable package func simuateThrowingDiscardingTaskGroup( - body: @isolated(any) (inout ThrowingTaskGroup) async throws -> sending TaskResult + body: @isolated(any) (inout ThrowingTaskGroup) async throws -> sending TaskResult ) async throws -> sending TaskResult { guard let actor = body.isolation else { preconditionFailure("body must be isolated") } + nonisolated(unsafe) + let block = body return try await simuateThrowingDiscardingTaskGroup(isolation: actor) { region, group in precondition(actor === region, "must be isolated on the inferred actor") nonisolated(unsafe) var unsafe = Suppress(base: group) + do { - let value = try await body(&unsafe.base) + let value = try await block(&unsafe.base) group = unsafe.base return value } catch { diff --git a/Sources/CriticalSection/Cell.swift b/Sources/CriticalSection/Cell.swift index a06b520..92627cc 100644 --- a/Sources/CriticalSection/Cell.swift +++ b/Sources/CriticalSection/Cell.swift @@ -5,26 +5,27 @@ // Created by 박병관 on 6/26/24. // import Builtin - +import Synchronization #if $BuiltinAddressOfRawLayout @frozen @usableFromInline -//@_rawLayout(like: Value, movesAsLike) +@_rawLayout(like: Value, movesAsLike) internal struct _Cell: ~Copyable { - + @_transparent @usableFromInline internal var _address: UnsafeMutablePointer { UnsafeMutablePointer(_rawAddress) } + @_transparent @usableFromInline internal var _rawAddress: Builtin.RawPointer { Builtin.addressOfRawLayout(self) } - + @_transparent @usableFromInline internal init(_ initialValue: consuming Value) { _address.initialize(to: initialValue) diff --git a/Sources/CriticalSection/DarwinImpl.swift b/Sources/CriticalSection/DarwinImpl.swift new file mode 100644 index 0000000..d4b915c --- /dev/null +++ b/Sources/CriticalSection/DarwinImpl.swift @@ -0,0 +1,42 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift Atomics open source project +// +// Copyright (c) 2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import Darwin + +@frozen +@_staticExclusiveOnly +public struct _MutexHandle: ~Copyable { + @usableFromInline + let value: _Cell + + @_transparent + public init() { + value = _Cell(os_unfair_lock()) + } + + @_transparent + internal borrowing func _lock() { + os_unfair_lock_lock(value._address) + } + + @_transparent + internal borrowing func _tryLock() -> Bool { + os_unfair_lock_trylock(value._address) + } + + + @_transparent + internal borrowing func _unlock() { + os_unfair_lock_unlock(value._address) + } +} + diff --git a/Sources/Tetra/Combine/AsyncSubscriptionState.swift b/Sources/Tetra/Combine/AsyncSubscriptionState.swift index 543e637..ffd13da 100644 --- a/Sources/Tetra/Combine/AsyncSubscriptionState.swift +++ b/Sources/Tetra/Combine/AsyncSubscriptionState.swift @@ -6,7 +6,7 @@ // import Foundation -import Combine +@preconcurrency import Combine enum AsyncSubscriptionState { @@ -147,6 +147,7 @@ enum AsyncSubscriptionState { } } + @preconcurrency private mutating func finish() -> sending Effect? { switch self { case .suspending(let unsafeContinuation): @@ -154,7 +155,8 @@ enum AsyncSubscriptionState { return .resume(unsafeContinuation) case .cached(let subscription): self = .finished - return .discard(subscription) + let what = consume subscription + return .discard(what) case .waiting: self = .finished fallthrough diff --git a/Sources/Tetra/Combine/CompatAsyncPublisher.swift b/Sources/Tetra/Combine/CompatAsyncPublisher.swift index 14cc637..d9c3472 100644 --- a/Sources/Tetra/Combine/CompatAsyncPublisher.swift +++ b/Sources/Tetra/Combine/CompatAsyncPublisher.swift @@ -8,7 +8,7 @@ import Foundation @preconcurrency import Combine -internal import BackPortAsyncSequence +public import BackPortAsyncSequence public struct CompatAsyncPublisher: AsyncSequence where P.Failure == Never { diff --git a/Sources/Tetra/Combine/CompatAsyncThrowingPublisher.swift b/Sources/Tetra/Combine/CompatAsyncThrowingPublisher.swift index 47c7918..cbe037c 100644 --- a/Sources/Tetra/Combine/CompatAsyncThrowingPublisher.swift +++ b/Sources/Tetra/Combine/CompatAsyncThrowingPublisher.swift @@ -8,7 +8,7 @@ import Foundation @preconcurrency import Combine -internal import BackPortAsyncSequence +public import BackPortAsyncSequence public struct CompatAsyncThrowingPublisher: AsyncSequence, TypedAsyncSequence { diff --git a/Sources/Tetra/Combine/ExperimentalMapTask.swift b/Sources/Tetra/Combine/ExperimentalMapTask.swift index 2ac2f1a..1d0438c 100644 --- a/Sources/Tetra/Combine/ExperimentalMapTask.swift +++ b/Sources/Tetra/Combine/ExperimentalMapTask.swift @@ -20,10 +20,12 @@ public struct MultiMapTask: Publisher where Upstream public typealias Output = Output public typealias Failure = Upstream.Failure + public typealias Transformer = @Sendable @isolated(any) (Upstream.Output) async throws(Failure) -> sending Output + public var priority:TaskPriority? = nil public var maxTasks:Subscribers.Demand public let upstream:Upstream - public let transform: @isolated(any) @Sendable (Upstream.Output) async throws(Failure) -> sending Output + public let transform: Transformer public let taskExecutor: (any Executor)? public func receive(subscriber: S) where S : Subscriber, Upstream.Failure == S.Failure, Output == S.Input { @@ -42,7 +44,7 @@ public struct MultiMapTask: Publisher where Upstream priority: TaskPriority? = nil, maxTasks: Subscribers.Demand = .max(1), upstream: Upstream, - transform: @Sendable @escaping @isolated(any) (Upstream.Output) async throws(Failure) -> Output + transform: @escaping Transformer ) { precondition(maxTasks != .none, "maxTasks can not be zero") self.maxTasks = maxTasks @@ -58,7 +60,7 @@ public struct MultiMapTask: Publisher where Upstream maxTasks: Subscribers.Demand = .max(1), executor:(any TaskExecutor)? = nil, upstream: Upstream, - transform: @Sendable @escaping @isolated(any) (Upstream.Output) async throws(Failure) -> Output + transform: @escaping Transformer ) { precondition(maxTasks != .none, "maxTasks can not be zero") self.maxTasks = maxTasks @@ -89,13 +91,13 @@ extension MultiMapTask { // accessed from Combine intferface or isolated Actor // which ever guarantee serialized access private let state: some UnfairStateLock> = createUncheckedStateLock(uncheckedState: TaskState()) - private let transform:@Sendable (Upstream.Output) async throws(Failure) -> Output + private let transform:Transformer let combineIdentifier = CombineIdentifier() init( maxTasks:Subscribers.Demand, subscriber:S, - transform: @escaping @Sendable (Upstream.Output) async throws(Failure) -> Output + transform: @escaping Transformer ) { self.maxTasks = maxTasks self.transform = transform diff --git a/Sources/Tetra/Combine/Future+Concurrency.swift b/Sources/Tetra/Combine/Future+Concurrency.swift index c162572..a4801dc 100644 --- a/Sources/Tetra/Combine/Future+Concurrency.swift +++ b/Sources/Tetra/Combine/Future+Concurrency.swift @@ -26,8 +26,9 @@ extension TetraExtension where Base: _CombineFuterProtocol { $0.request(.max(1)) subscription = $0 }, - receiveValue: { (value: sending Base.Output) in - continuation.resume(returning: .success(value)) + receiveValue: { (value: Base.Output) in + let variable = Suppress(value: value) + continuation.resume(returning: .success(variable.value)) return .none }, receiveCompletion: { diff --git a/Sources/Tetra/Combine/Publishers+AsyncFlatMap.swift b/Sources/Tetra/Combine/Publishers+AsyncFlatMap.swift index 24fb6b9..ed8af51 100644 --- a/Sources/Tetra/Combine/Publishers+AsyncFlatMap.swift +++ b/Sources/Tetra/Combine/Publishers+AsyncFlatMap.swift @@ -70,7 +70,7 @@ struct AsyncFlatMap: Publisher where taskExecutor: (any TaskExecutor)? = nil, maxTasks: Subscribers.Demand, upstream: Upstream, - typedTransform: @escaping @isolated(any) @Sendable (Upstream.Output) async throws(Failure) -> sending Source + typedTransform: @escaping @Sendable (Upstream.Output) async throws(Failure) -> sending Source ) where Source: AsyncSequence, Segment == WrappedAsyncSequence { self.priority = priority self.maxTasks = maxTasks @@ -160,7 +160,7 @@ extension AsyncFlatMap { var description: String { "AsyncFlatMap" } - func receive(_ input: sending Input) -> Subscribers.Demand { + func receive(_ input: Input) -> Subscribers.Demand { valueSource.continuation.yield(.success(input)) return .none } diff --git a/Sources/Tetra/Combine/Publishers+MapTask.swift b/Sources/Tetra/Combine/Publishers+MapTask.swift index 6acb9ec..426fccc 100644 --- a/Sources/Tetra/Combine/Publishers+MapTask.swift +++ b/Sources/Tetra/Combine/Publishers+MapTask.swift @@ -76,7 +76,7 @@ public struct MapTask: Publisher where Upstream.Outp maxTasks: .max(1), upstream: upstream, transform: { [transform] value throws(Failure) in - try await transform(consume value).get() + Suppress(value: try await transform(value).get()).value }) .subscribe(MapTaskInner(description: "MapTask", downstream: subscriber)) } diff --git a/Sources/Tetra/Concurrency/AsyncSequencePublisher.swift b/Sources/Tetra/Concurrency/AsyncSequencePublisher.swift index da1e99b..58f29f6 100644 --- a/Sources/Tetra/Concurrency/AsyncSequencePublisher.swift +++ b/Sources/Tetra/Concurrency/AsyncSequencePublisher.swift @@ -73,10 +73,6 @@ public struct AsyncSequencePublisher: Publisher where Base. public func receive(subscriber: S) where S : Subscriber, Failure == S.Failure, Base.AsyncIterator.Element == S.Input { let processor = Inner(subscriber: subscriber) - // transfer the AsyncIterator to the Task - // can not tell the compiler that this is safe, - // but this transfer is safe from data race - nonisolated(unsafe) let unsafe = Suppress(value: base.makeAsyncIterator()) let task = Task(priority: priority) { [capture = consume unsafe, barrier] in var iter = capture.value @@ -202,7 +198,7 @@ extension AsyncSequencePublisher { func run( _ actor: isolated (any Actor)? = #isolation, - _ iterator: inout sending Base.AsyncIterator + _ iterator: inout Base.AsyncIterator ) async { let token:Void? = try? await waitForCondition() if token == nil { diff --git a/Sources/Tetra/Concurrency/CoreDataStack+Concurrency.swift b/Sources/Tetra/Concurrency/CoreDataStack+Concurrency.swift index 926c3b3..151ca8d 100644 --- a/Sources/Tetra/Concurrency/CoreDataStack+Concurrency.swift +++ b/Sources/Tetra/Concurrency/CoreDataStack+Concurrency.swift @@ -9,7 +9,7 @@ import Foundation import _Concurrency #if canImport(CoreData) -import CoreData +@preconcurrency import CoreData import Namespace @@ -126,43 +126,44 @@ extension TetraExtension where Base: NSManagedObjectContext { return try result.get() } - /// Asynchronously performs the specified closure on the context’s queue. - @inlinable - @_unsafeInheritExecutor - public func perform( - schedule:CoreDataScheduledTaskType = .immediate, - _ body: () throws -> T - ) async rethrows -> T { - /* - - Since this method and NSManagedObjectContext peform has no actor preference and isolation restriction. - These two are always called on global nonisolated context (Actor switching happen). - Which means that `immediate` execution option is totally no-op. - - - - `_performImmediate:` is never called when using `NSManagedObjectContext.perform(schedule: .immediate)` in iOS 15 ~ iOS 17 - - @_unsafeInheritExecutor do fix the above problem - */ - return if #available(iOS 15.0, tvOS 15.0, macCatalyst 15.0, watchOS 8.0, macOS 12.0, *) { - try await withoutActuallyEscaping(body) { - let block = ClosureHolder(closure: $0) - defer { - withExtendedLifetime(block, {}) - } - return try await base.perform(schedule: schedule.platformValue) { [unowned block] in - return try block().get() - } - } - } else if schedule == .enqueued { - try await _performEnqueue(body) - } else { - if let result = try _performImmediate(body) { - result.get() - } else { - try await _performEnqueue(body) - } - } - } +// /// Asynchronously performs the specified closure on the context’s queue. +// @inlinable +// @preconcurrency +// +// public func perform( +// schedule:CoreDataScheduledTaskType = .immediate +// , isolation: isolated (any Actor)? = #isolation, _ body: @Sendable () throws -> T +// ) async rethrows -> T where T:Sendable { +// /* +// +// Since this method and NSManagedObjectContext peform has no actor preference and isolation restriction. +// These two are always called on global nonisolated context (Actor switching happen). +// Which means that `immediate` execution option is totally no-op. +// +// +// - `_performImmediate:` is never called when using `NSManagedObjectContext.perform(schedule: .immediate)` in iOS 15 ~ iOS 17 +// - @_unsafeInheritExecutor do fix the above problem +// */ +// return if #available(iOS 15.0, tvOS 15.0, macCatalyst 15.0, watchOS 8.0, macOS 12.0, *) { +// try await withoutActuallyEscaping(body) { +// let block = ClosureHolder(closure: $0) +// defer { +// withExtendedLifetime(block, {}) +// } +// return try await base.perform(schedule: schedule.platformValue) { [unowned block] in +// return try block().get() +// } +// } +// } else if schedule == .enqueued { +// try await _performEnqueue(body) +// } else { +// if let result = try _performImmediate(body) { +// result.get() +// } else { +// try await _performEnqueue(body) +// } +// } +// } @usableFromInline internal func _performAndWait(_ body: () throws(Failure) -> T) throws(Failure) -> T { @@ -250,3 +251,5 @@ public enum CoreDataScheduledTaskType: Sendable, Hashable { } #endif + +extension TetraExtension: Sendable where Base: Sendable {} diff --git a/Sources/Tetra/Concurrency/Dispatch+Extension.swift b/Sources/Tetra/Concurrency/Dispatch+Extension.swift index 66b4e04..82c5401 100644 --- a/Sources/Tetra/Concurrency/Dispatch+Extension.swift +++ b/Sources/Tetra/Concurrency/Dispatch+Extension.swift @@ -6,7 +6,7 @@ // @preconcurrency import Foundation -import Dispatch +@preconcurrency import Dispatch internal import CriticalSection import Namespace diff --git a/Sources/Tetra/Concurrency/Notification+AsyncSequence.swift b/Sources/Tetra/Concurrency/Notification+AsyncSequence.swift index 004d5d5..cf39d5f 100644 --- a/Sources/Tetra/Concurrency/Notification+AsyncSequence.swift +++ b/Sources/Tetra/Concurrency/Notification+AsyncSequence.swift @@ -8,7 +8,7 @@ import Foundation import _Concurrency -internal import BackPortAsyncSequence +public import BackPortAsyncSequence import Namespace public import CriticalSection diff --git a/Sources/Tetra/Foundation/PruneMemory.swift b/Sources/Tetra/Foundation/PruneMemory.swift index 4610edc..46e7372 100644 --- a/Sources/Tetra/Foundation/PruneMemory.swift +++ b/Sources/Tetra/Foundation/PruneMemory.swift @@ -199,7 +199,6 @@ enum MemoryErasing { // memset_s ignore compiler optimization memset_s(ptr, size, 0, size) free(ptr) - print("free") } } context.copyDescription = { _ in diff --git a/Sources/Tetra/SwiftUI/Binding+Collection.swift b/Sources/Tetra/SwiftUI/Binding+Collection.swift index e745c78..a664561 100644 --- a/Sources/Tetra/SwiftUI/Binding+Collection.swift +++ b/Sources/Tetra/SwiftUI/Binding+Collection.swift @@ -44,12 +44,14 @@ public struct BindingCollection: Collection { return binding[position] } else { nonisolated(unsafe) - let index = consume position + let index = position return .init { [binding] in binding.wrappedValue[index] } set: { [binding] newValue, transaction in + nonisolated(unsafe) + let ref = binding withTransaction(transaction) { - binding.wrappedValue[index] = newValue + ref.wrappedValue[index] = newValue } } From 1fc4ef1c4b488ad7c2a4dad3242332eebbeb669a Mon Sep 17 00:00:00 2001 From: pbk Date: Thu, 19 Dec 2024 11:38:52 +0900 Subject: [PATCH 46/63] fix test failure --- Package.swift | 5 ++++- .../BackPortAsyncSequence/AsyncStream.swift | 8 +++++--- .../AsyncThrowingStream.swift | 5 +++-- .../Combine/Publishers+AsyncFlatMap.swift | 4 ++-- Tests/TetraTests/AsyncFlatMapTests.swift | 19 ++++++++++--------- Tests/TetraTests/MultiMapTaskTests.swift | 2 +- .../TetraTests/URLSessionDownloadTests.swift | 3 ++- .../Utlitity/SimpleHTTPServer.swift | 6 +++--- .../Utlitity/UnsafeCancellableHolder.swift | 2 +- 9 files changed, 31 insertions(+), 23 deletions(-) diff --git a/Package.swift b/Package.swift index 202f3bb..f624fff 100644 --- a/Package.swift +++ b/Package.swift @@ -102,7 +102,10 @@ let package = Package( dependencies: [ "Tetra" ], - resources: [.process("Resources")] + resources: [.process("Resources")], + swiftSettings: [ + .swiftLanguageMode(.v5) + ] ) ] ) diff --git a/Sources/BackPortAsyncSequence/AsyncStream.swift b/Sources/BackPortAsyncSequence/AsyncStream.swift index 7b4adb6..a77a912 100644 --- a/Sources/BackPortAsyncSequence/AsyncStream.swift +++ b/Sources/BackPortAsyncSequence/AsyncStream.swift @@ -51,8 +51,10 @@ extension AsyncTypedStream.Iterator: AsyncIteratorProtocol, TypedAsyncIteratorPr } else { nonisolated(unsafe) var iter = self + defer { + self = iter + } let value = await iter.advanceNext()?.base - self = iter return value } } @@ -65,8 +67,8 @@ extension AsyncTypedStream.Iterator: AsyncIteratorProtocol, TypedAsyncIteratorPr @inline(__always) @usableFromInline - @preconcurrency - internal mutating func advanceNext() async -> Suppress? { +// @preconcurrency + internal mutating func advanceNext() async -> sending Suppress? { guard let value = await baseIterator.next() else { return nil } return .init(base: value) } diff --git a/Sources/BackPortAsyncSequence/AsyncThrowingStream.swift b/Sources/BackPortAsyncSequence/AsyncThrowingStream.swift index 2f74c4c..f4bd486 100644 --- a/Sources/BackPortAsyncSequence/AsyncThrowingStream.swift +++ b/Sources/BackPortAsyncSequence/AsyncThrowingStream.swift @@ -51,12 +51,13 @@ extension AsyncTypedThrowingStream.Iterator: AsyncIteratorProtocol, TypedAsyncIt } else { nonisolated(unsafe) var iter = self + defer { + self = iter + } do { let value = try await iter.nextValue()?.base - self = iter return value } catch { - self = iter throw (error as! Failure) } } diff --git a/Sources/Tetra/Combine/Publishers+AsyncFlatMap.swift b/Sources/Tetra/Combine/Publishers+AsyncFlatMap.swift index ed8af51..781f174 100644 --- a/Sources/Tetra/Combine/Publishers+AsyncFlatMap.swift +++ b/Sources/Tetra/Combine/Publishers+AsyncFlatMap.swift @@ -366,9 +366,9 @@ extension AsyncFlatMap { iterator: inout Segment.AsyncIterator, barrier: some Actor ) async -> Bool { - let result:Result? +// let result:Result? do throws(Failure) { - let value = try await iterator.next(isolation: nil) +// let value = try await iterator.next(isolation: nil) if let value = try await iterator.next(isolation: nil) { await handleDownStream(isolation: barrier, event: .success(.init(value: value))) diff --git a/Tests/TetraTests/AsyncFlatMapTests.swift b/Tests/TetraTests/AsyncFlatMapTests.swift index 42b76c6..fa82066 100644 --- a/Tests/TetraTests/AsyncFlatMapTests.swift +++ b/Tests/TetraTests/AsyncFlatMapTests.swift @@ -26,30 +26,30 @@ final class AsyncFlatMapTests: XCTestCase { .asyncFlatMap(maxTasks: .max(1)) { value in return AsyncStream{ continuation in sample.forEach{ - continuation.yield($0) + let result = continuation.yield($0) + if case .enqueued(_) = result { + + } else { + XCTFail("should not reach") + } } continuation.finish() }.tetra.bridge() - .tetra.map{ - await Task.yield() - return $0 - } } .handleEvents( receiveSubscription: { XCTAssertEqual("\($0)", "AsyncFlatMap") } ) - // ensure downstream do not request unlimited .buffer(size: 1, prefetch: .keepFull, whenFull: .customError{ fatalError() }) - .prefix(10) + //.prefix(10) .sink { _ in completion.fulfill() } receiveValue: { array.append($0) } - wait(for: [completion], timeout: 0.1) + wait(for: [completion], timeout: 0.5) bag.cancel() XCTAssertEqual(array, sample + sample) } @@ -253,6 +253,7 @@ final class AsyncFlatMapTests: XCTestCase { } } let transformTask = withUnsafeCurrentTask{ $0 }?.hashValue + // every segment runs in separate child task let stream = AsyncStream{ await Task.yield() withUnsafeCurrentTask { @@ -270,7 +271,7 @@ final class AsyncFlatMapTests: XCTestCase { } receiveValue: { buffer.append($0) }.store(in: &holder.bag) - wait(for: [completion], timeout: 0.2) + wait(for: [completion], timeout: 200) XCTAssertEqual(buffer, [1,2,3,4]) } diff --git a/Tests/TetraTests/MultiMapTaskTests.swift b/Tests/TetraTests/MultiMapTaskTests.swift index a510531..bc4f648 100644 --- a/Tests/TetraTests/MultiMapTaskTests.swift +++ b/Tests/TetraTests/MultiMapTaskTests.swift @@ -74,7 +74,7 @@ final class MultiMapTaskTests: XCTestCase { let target = try XCTUnwrap(sequence.randomElement()) let upstream = sequence.publisher.setFailureType(to: CancellationError.self) let expect = expectation(description: "task failure") - let block:@Sendable (Int) async throws(CancellationError) -> Int = { + let block:@Sendable (Int) async throws(CancellationError) -> sending Int = { if $0 == target { try Result.failure(CancellationError()).get() } diff --git a/Tests/TetraTests/URLSessionDownloadTests.swift b/Tests/TetraTests/URLSessionDownloadTests.swift index d648531..b0dc082 100644 --- a/Tests/TetraTests/URLSessionDownloadTests.swift +++ b/Tests/TetraTests/URLSessionDownloadTests.swift @@ -9,11 +9,12 @@ import XCTest @testable import Tetra import Namespace +@preconcurrency final class URLSessionDownloadTests: XCTestCase { private static let text = UUID().uuidString - private static var webserver:Result? = nil + nonisolated(unsafe) private static var webserver:Result? = nil private var url: URL { get throws { diff --git a/Tests/TetraTests/Utlitity/SimpleHTTPServer.swift b/Tests/TetraTests/Utlitity/SimpleHTTPServer.swift index 19fe8e8..c086020 100644 --- a/Tests/TetraTests/Utlitity/SimpleHTTPServer.swift +++ b/Tests/TetraTests/Utlitity/SimpleHTTPServer.swift @@ -8,7 +8,7 @@ import Foundation import Network -class SimpleHTTPServer { +class SimpleHTTPServer: @unchecked Sendable { var port: NWEndpoint.Port? { listener.port @@ -16,12 +16,12 @@ class SimpleHTTPServer { let queue:DispatchQueue let listener: NWListener - let errorHandler: (any Error) -> Void + let errorHandler: @Sendable (any Error) -> Void var response:String = "Hello, World!" init( queue:DispatchQueue, port: NWEndpoint.Port, - errorHandle: @escaping (any Error) -> Void + errorHandle: @escaping @Sendable (any Error) -> Void ) throws { self.listener = try NWListener(using: .tcp, on: port) self.queue = queue diff --git a/Tests/TetraTests/Utlitity/UnsafeCancellableHolder.swift b/Tests/TetraTests/Utlitity/UnsafeCancellableHolder.swift index 79812c9..8c19023 100644 --- a/Tests/TetraTests/Utlitity/UnsafeCancellableHolder.swift +++ b/Tests/TetraTests/Utlitity/UnsafeCancellableHolder.swift @@ -6,7 +6,7 @@ // import Combine import Foundation - +@preconcurrency class UnsafeCancellableHolder { var bag = Set() From f8b5c2327105925ff43c568996e7b8348f6f8eee Mon Sep 17 00:00:00 2001 From: pbk Date: Sat, 21 Dec 2024 12:06:05 +0900 Subject: [PATCH 47/63] backport CoreDataContext perform method --- .../Tetra/Combine/ExperimentalMapTask.swift | 2 +- .../CoreDataStack+Concurrency.swift | 114 +++++++++++------- Sources/Tetra/Foundation/Suppress.swift | 2 + 3 files changed, 76 insertions(+), 42 deletions(-) diff --git a/Sources/Tetra/Combine/ExperimentalMapTask.swift b/Sources/Tetra/Combine/ExperimentalMapTask.swift index 1d0438c..4655d68 100644 --- a/Sources/Tetra/Combine/ExperimentalMapTask.swift +++ b/Sources/Tetra/Combine/ExperimentalMapTask.swift @@ -183,7 +183,7 @@ extension MultiMapTask { private func send( isolation actor: isolated some Actor, _ value: S.Input - ) async throws { + ) async throws(CancellationError) { let (subscriber, subscription) = state.withLockUnchecked{ return ($0.subscriber, $0.upstreamSubscription.subscription) diff --git a/Sources/Tetra/Concurrency/CoreDataStack+Concurrency.swift b/Sources/Tetra/Concurrency/CoreDataStack+Concurrency.swift index 151ca8d..cdca0b6 100644 --- a/Sources/Tetra/Concurrency/CoreDataStack+Concurrency.swift +++ b/Sources/Tetra/Concurrency/CoreDataStack+Concurrency.swift @@ -110,14 +110,12 @@ extension TetraExtension where Base: NSManagedObjectContext { _ body: () throws(Failure) -> T ) async throws(Failure) -> T { let result: Result = await withoutActuallyEscaping(body) { escapingClosure in - let holder = ClosureHolder(closure: escapingClosure) defer { withExtendedLifetime(holder, {}) } - return await withUnsafeContinuation { continuation in - base.perform{ [unowned holder, continuation] in + base.perform{ [unowned(unsafe) holder, continuation] in let result = holder() continuation.resume(returning: result) } @@ -126,44 +124,52 @@ extension TetraExtension where Base: NSManagedObjectContext { return try result.get() } -// /// Asynchronously performs the specified closure on the context’s queue. -// @inlinable -// @preconcurrency -// -// public func perform( -// schedule:CoreDataScheduledTaskType = .immediate -// , isolation: isolated (any Actor)? = #isolation, _ body: @Sendable () throws -> T -// ) async rethrows -> T where T:Sendable { -// /* -// -// Since this method and NSManagedObjectContext peform has no actor preference and isolation restriction. -// These two are always called on global nonisolated context (Actor switching happen). -// Which means that `immediate` execution option is totally no-op. -// -// -// - `_performImmediate:` is never called when using `NSManagedObjectContext.perform(schedule: .immediate)` in iOS 15 ~ iOS 17 -// - @_unsafeInheritExecutor do fix the above problem -// */ -// return if #available(iOS 15.0, tvOS 15.0, macCatalyst 15.0, watchOS 8.0, macOS 12.0, *) { -// try await withoutActuallyEscaping(body) { -// let block = ClosureHolder(closure: $0) -// defer { -// withExtendedLifetime(block, {}) -// } -// return try await base.perform(schedule: schedule.platformValue) { [unowned block] in -// return try block().get() -// } -// } -// } else if schedule == .enqueued { -// try await _performEnqueue(body) -// } else { -// if let result = try _performImmediate(body) { -// result.get() -// } else { -// try await _performEnqueue(body) -// } -// } -// } + @usableFromInline + internal func _perform( + schedule:CoreDataScheduledTaskType = .immediate, + isolation: isolated (any Actor)? = #isolation, + _ body: @escaping () throws(Failure) -> T + ) async throws(Failure) -> Suppress { + nonisolated(unsafe) + let callRef = self + let ref = UnsafeClosureHolder(closure: body) + defer { + withExtendedLifetime(ref, {}) + } + if schedule == .immediate, let result = try _performImmediate(body) { + let value = result.get() + return .init(value: value) + } else { + return try await callRef._performEnqueue { [unowned(unsafe) ref] in + ref().map(Suppress.init) + }.get() + } + } + + + /// Asynchronously performs the specified closure on the context’s queue. + @inlinable + public func perform( + schedule:CoreDataScheduledTaskType = .immediate, + isolation: isolated (any Actor)? = #isolation, + _ body: () throws(Failure) -> T + ) async throws(Failure) -> T { + let box: Suppress = try await withoutActuallyEscaping(body) { escapingClosure async throws(Failure) in + if #available(iOS 15.0, tvOS 15.0, macCatalyst 15.0, watchOS 8.0, macOS 12.0, *) { + let ref:UnsafeClosureHolder = UnsafeClosureHolder(closure: escapingClosure) + defer { + withExtendedLifetime(ref, {}) + } + let wrapped = await base.perform(schedule: schedule.platformValue) { [unowned(unsafe) ref] in + ref().map(Suppress.init) + } + return try wrapped.get() + } else { + return try await self._perform(schedule: schedule, isolation: isolation, escapingClosure) + } + } + return box.value + } @usableFromInline internal func _performAndWait(_ body: () throws(Failure) -> T) throws(Failure) -> T { @@ -253,3 +259,29 @@ public enum CoreDataScheduledTaskType: Sendable, Hashable { #endif extension TetraExtension: Sendable where Base: Sendable {} + +@usableFromInline +internal final class UnsafeClosureHolder: @unchecked Sendable { + + @inline(__always) + @usableFromInline let closure: () throws(Failure) -> R + + @inline(__always) + @inlinable + init(closure: @escaping () throws(Failure) -> R) { + self.closure = closure + } + + @inline(__always) + @inlinable + func callAsFunction() -> Result { + do { + let value = try closure() + return .success(value) + } catch { + return .failure(error) + } + } + +} + diff --git a/Sources/Tetra/Foundation/Suppress.swift b/Sources/Tetra/Foundation/Suppress.swift index ffcbf7c..7a3393d 100644 --- a/Sources/Tetra/Foundation/Suppress.swift +++ b/Sources/Tetra/Foundation/Suppress.swift @@ -11,9 +11,11 @@ import Foundation @usableFromInline struct Suppress: @unchecked Sendable { + @inline(__always) @usableFromInline var value:T + @inline(__always) @usableFromInline init(value: T) { self.value = value From 8faf2ff0a8a3120a758a3fac7727b101a771d7ad Mon Sep 17 00:00:00 2001 From: pbk Date: Sat, 21 Dec 2024 12:38:32 +0900 Subject: [PATCH 48/63] expose generic Error for coreData --- .../Combine/Publishers+AsyncFlatMap.swift | 8 +++--- .../CoreDataStack+Concurrency.swift | 26 ++++++++++--------- 2 files changed, 18 insertions(+), 16 deletions(-) diff --git a/Sources/Tetra/Combine/Publishers+AsyncFlatMap.swift b/Sources/Tetra/Combine/Publishers+AsyncFlatMap.swift index 781f174..8e5193b 100644 --- a/Sources/Tetra/Combine/Publishers+AsyncFlatMap.swift +++ b/Sources/Tetra/Combine/Publishers+AsyncFlatMap.swift @@ -15,7 +15,7 @@ struct AsyncFlatMap: Publisher where typealias Output = Segment.Element typealias Failure = Upstream.Failure - typealias Transform = @Sendable (Upstream.Output) async throws(Failure) -> sending Segment + typealias Transform = @Sendable (Upstream.Output) async throws(Failure) -> Segment var priority: TaskPriority? = nil let taskExecutor: (any Executor)? var maxTasks:Subscribers.Demand @@ -52,7 +52,7 @@ struct AsyncFlatMap: Publisher where priority: TaskPriority? = nil, maxTasks: Subscribers.Demand, upstream: Upstream, - transform: @escaping @Sendable (Upstream.Output) async throws(Failure) -> sending Source + transform: @escaping @Sendable (Upstream.Output) async throws(Failure) -> Source ) where Source: AsyncSequence, Segment == LegacyTypedAsyncSequence, Failure == any Error { self.priority = priority self.maxTasks = maxTasks @@ -70,7 +70,7 @@ struct AsyncFlatMap: Publisher where taskExecutor: (any TaskExecutor)? = nil, maxTasks: Subscribers.Demand, upstream: Upstream, - typedTransform: @escaping @Sendable (Upstream.Output) async throws(Failure) -> sending Source + typedTransform: @escaping @Sendable (Upstream.Output) async throws(Failure) -> Source ) where Source: AsyncSequence, Segment == WrappedAsyncSequence { self.priority = priority self.maxTasks = maxTasks @@ -89,7 +89,7 @@ extension AsyncFlatMap { struct Inner: Subscriber, Sendable, Subscription, CustomStringConvertible, CustomPlaygroundDisplayConvertible where Segment.Element == Down.Input, Down.Failure == AsyncFlatMap.Failure { - typealias Transformer = @Sendable (Upstream.Output) async throws(Failure) -> sending Segment + typealias Transformer = @Sendable (Upstream.Output) async throws(Failure) -> Segment typealias Input = Upstream.Output typealias Failure = Upstream.Failure diff --git a/Sources/Tetra/Concurrency/CoreDataStack+Concurrency.swift b/Sources/Tetra/Concurrency/CoreDataStack+Concurrency.swift index cdca0b6..ee37570 100644 --- a/Sources/Tetra/Concurrency/CoreDataStack+Concurrency.swift +++ b/Sources/Tetra/Concurrency/CoreDataStack+Concurrency.swift @@ -33,17 +33,17 @@ extension TetraExtension where Base: NSPersistentStoreCoordinator { } @inlinable - public func perform(_ body: () throws -> T) async rethrows -> T { + public func perform(_ body: () throws(Failure) -> T) async throws(Failure) -> T { return if #available(iOS 15.0, tvOS 15.0, macCatalyst 15.0, watchOS 8.0, macOS 12.0, *) { try await withoutActuallyEscaping(body) { let block = ClosureHolder(closure: $0) defer { withExtendedLifetime(block, {}) } - return try await base.perform{ [unowned block] in - try block().get() + return await base.perform{ [unowned block] in + block() } - } + }.get() } else { try await _perform(body) } @@ -66,9 +66,11 @@ extension TetraExtension where Base: NSPersistentStoreCoordinator { } @inlinable - public func performAndWait(_ body: () throws -> T) rethrows -> T { + public func performAndWait(_ body: () throws(Failure) -> T) throws(Failure) -> T { return if #available(iOS 15.0, tvOS 15.0, macCatalyst 15.0, watchOS 8.0, macOS 12.0, *) { - try base.performAndWait(body) + try base.performAndWait{ + Result(catching: body) + }.get() } else { try _performAndWait(body) } @@ -188,9 +190,9 @@ extension TetraExtension where Base: NSManagedObjectContext { } @inlinable - public func performAndWait(_ body: () throws -> T) rethrows -> T { + public func performAndWait(_ body: () throws(Failure) -> T) throws(Failure) -> T { return if #available(iOS 15.0, tvOS 15.0, macCatalyst 15.0, watchOS 8.0, macOS 12.0, *) { - try base.performAndWait(body) + try base.performAndWait{ Result(catching: body) }.get() } else { try _performAndWait(body) } @@ -201,15 +203,15 @@ extension TetraExtension where Base: NSManagedObjectContext { extension TetraExtension where Base: NSPersistentContainer { @inlinable - public func performBackground(_ body: (NSManagedObjectContext) throws -> T) async rethrows -> T { + public func performBackground(_ body: (NSManagedObjectContext) throws(Failure) -> T) async throws(Failure) -> T { return if #available(iOS 15.0, tvOS 15.0, macCatalyst 15.0, watchOS 8.0, macOS 12.0, *) { try await withoutActuallyEscaping(body) { let block = CoreDataContextClosureHolder(closure: $0) defer { withExtendedLifetime(block, {}) } - return try await base.performBackgroundTask{ [unowned block] in - try block($0).get() + return await base.performBackgroundTask{ [unowned block] in + block($0) } - } + }.get() } else { try await _performBackground(body) } From a12533421f705a870932093cb43585a8253186a0 Mon Sep 17 00:00:00 2001 From: pbk Date: Sat, 21 Dec 2024 12:39:49 +0900 Subject: [PATCH 49/63] hide wip executor class --- Sources/Tetra/Concurrency/DispatchSerialExecutor.swift | 2 +- Sources/Tetra/Concurrency/PriorityRunLoop.swift | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Sources/Tetra/Concurrency/DispatchSerialExecutor.swift b/Sources/Tetra/Concurrency/DispatchSerialExecutor.swift index 88d413b..3599b98 100644 --- a/Sources/Tetra/Concurrency/DispatchSerialExecutor.swift +++ b/Sources/Tetra/Concurrency/DispatchSerialExecutor.swift @@ -8,7 +8,7 @@ import Foundation import Dispatch -public final class DispatchQueueExecutor: SerialExecutor { +package final class DispatchQueueExecutor: SerialExecutor { let queue:DispatchQueue diff --git a/Sources/Tetra/Concurrency/PriorityRunLoop.swift b/Sources/Tetra/Concurrency/PriorityRunLoop.swift index d79eab4..b6f5ac3 100644 --- a/Sources/Tetra/Concurrency/PriorityRunLoop.swift +++ b/Sources/Tetra/Concurrency/PriorityRunLoop.swift @@ -160,7 +160,7 @@ struct RunLoopPriorityQueue: ~Copyable, Sendable { heaps.withLockUnchecked{ [job] in // lastest has the lower id which results lower priority let id = -$0.count - let item = JobBlock(id: id, jobRef: consume job) + let item = JobBlock(id: id, jobRef: job) $0.insert(item) if #available(macOS 14.0, iOS 17.0, watchOS 10.0, tvOS 17.0, *), var max = $0.popMax() { if max.token == nil, let qos, qos.qosClass.rawValue != qos_class, item.priority == max.priority { @@ -198,7 +198,7 @@ struct RunLoopPriorityQueue: ~Copyable, Sendable { } -public final class RunLoopPriorityExecutor { +package final class RunLoopPriorityExecutor { // cache for faster comparsion, RunLoop comparsion trigger creating extra RunLoop From cb4800e04187d3539083b6c01571d8f0e7a223bb Mon Sep 17 00:00:00 2001 From: pbk Date: Sat, 21 Dec 2024 22:58:01 +0900 Subject: [PATCH 50/63] refactor backport discardingTask --- .../TaskGroup.swift | 76 +++++++++++++--- .../ThrowingTaskGroup.swift | 91 +++++++++++++++---- Sources/CriticalSection/Cell.swift | 2 +- Sources/CriticalSection/DarwinImpl.swift | 2 + .../NamespaceExtension/TetraExtended.swift | 6 +- .../Combine/Publishers+AsyncFlatMap.swift | 16 ++-- 6 files changed, 150 insertions(+), 43 deletions(-) diff --git a/Sources/BackportDiscardingTaskGroup/TaskGroup.swift b/Sources/BackportDiscardingTaskGroup/TaskGroup.swift index f75d4dd..a828a84 100644 --- a/Sources/BackportDiscardingTaskGroup/TaskGroup.swift +++ b/Sources/BackportDiscardingTaskGroup/TaskGroup.swift @@ -67,20 +67,70 @@ package func simuateDiscardingTaskGroup( /// - Returns: which is returned from body /// - SeeAlso: withDiscardingTaskGroup(returning:body:) @inlinable -package func simuateDiscardingTaskGroup( - body: @isolated(any) (inout TaskGroup) async -> sending TaskResult -) async -> sending TaskResult { - guard let actor = body.isolation else { - preconditionFailure("body must be isolated") +package func simuateDiscardingTaskGroup2( + isolation actor: isolated (any Actor)? = #isolation, + body: (inout TaskGroup) async -> TaskResult +) async -> TaskResult { + guard actor != nil else { + preconditionFailure("actor should not be nil") } - nonisolated(unsafe) - let block = body - return await simuateDiscardingTaskGroup(isolation: actor) { region, group in - precondition(actor === region, "must be isolated on the inferred actor") + return await withoutActuallyEscaping(body) { escapingClosure in + await __simuateDiscardingTaskGroup2(isolation: actor) { group in + let value = await escapingClosure(&group) + return Suppress(base: value) + } + }.base +} + +@usableFromInline +internal func __simuateDiscardingTaskGroup2( + isolation actor: isolated (any Actor)?, + body: @escaping (inout TaskGroup) async -> sending TaskResult +) async -> TaskResult { + let wrapped:Suppress = await withTaskGroup(of: Void.self, returning: Suppress.self, isolation: actor) { group in + let holder: SafetyRegion = actor as? SafetyRegion ?? .init() + if await holder.isFinished { + preconditionFailure("SafetyRegion is already used!") + } + group.addTask(priority: .background) { + /// keep at least one child task alive + /// so that subTask won't return + await holder.hold() + } + let suppress = Suppress(base: group) + /// drain all the finished or failed Task + async let subTask:Void = { (barrier: isolated (any Actor)?) in + + var iter = suppress.base + while let _ = await iter.next(isolation: barrier) { + if await holder.isFinished { + break + } + } + }(actor) + nonisolated(unsafe) + let body2 = body + nonisolated(unsafe) + let block2 = { (barrier: isolated (any Actor)?) in + var iter = suppress.base + return Suppress(base: await body2(&iter)) + } + async let mainTask = { + do { + let v = await block2(actor) + await holder.markDone() + return v + } + }() + do { + // wait for subTask first to trigger priority elavation + // (release finished tasks as soon as possible) + await subTask + } nonisolated(unsafe) - var unsafe = Suppress(base: group) - let value = await block(&unsafe.base) - group = unsafe.base - return value + let value = await mainTask.base + + return .init(base: value) } + return wrapped.base } diff --git a/Sources/BackportDiscardingTaskGroup/ThrowingTaskGroup.swift b/Sources/BackportDiscardingTaskGroup/ThrowingTaskGroup.swift index 200ef73..c31fa87 100644 --- a/Sources/BackportDiscardingTaskGroup/ThrowingTaskGroup.swift +++ b/Sources/BackportDiscardingTaskGroup/ThrowingTaskGroup.swift @@ -5,6 +5,22 @@ // Created by 박병관 on 6/20/24. // +@inlinable +package func simuateThrowingDiscardingTaskGroup2( + isolation actor: isolated (any Actor)? = #isolation, + body: (inout ThrowingTaskGroup) async throws -> TaskResult +) async throws -> TaskResult { + guard actor != nil else { + preconditionFailure("actor should not be nil") + } + return try await withoutActuallyEscaping(body) { escapingClosure in + try await __simuateThrowingDiscardingTaskGroup2(isolation: actor) { group in + let value = try await escapingClosure(&group) + return Suppress(base: value) + } + }.base +} + @inlinable package func simuateThrowingDiscardingTaskGroup( isolation actor: isolated T, @@ -63,27 +79,66 @@ package func simuateThrowingDiscardingTaskGroup( return wrapped.base } -@inlinable -package func simuateThrowingDiscardingTaskGroup( - body: @isolated(any) (inout ThrowingTaskGroup) async throws -> sending TaskResult -) async throws -> sending TaskResult { - guard let actor = body.isolation else { - preconditionFailure("body must be isolated") - } - nonisolated(unsafe) - let block = body - return try await simuateThrowingDiscardingTaskGroup(isolation: actor) { region, group in - precondition(actor === region, "must be isolated on the inferred actor") +@usableFromInline +internal func __simuateThrowingDiscardingTaskGroup2( + isolation actor: isolated (any Actor)?, + body: @escaping (inout ThrowingTaskGroup) async throws -> sending TaskResult +) async throws -> TaskResult { + let wrapped:Suppress = try await withThrowingTaskGroup(of: Void.self, returning: Suppress.self, isolation: actor) { group in + let holder: SafetyRegion = actor as? SafetyRegion ?? .init() + if await holder.isFinished { + preconditionFailure("SafetyRegion is already used!") + } + group.addTask(priority: .background) { + /// keep at least one child task alive + /// so that subTask won't return + await holder.hold() + } + let suppress = Suppress(base: group) + /// drain all the finished or failed Task + async let subTask:Void = { + nonisolated(unsafe) + var iter = suppress.base + while let _ = try await iter.next(isolation: actor) { + if await holder.isFinished { + break + } + } + }() nonisolated(unsafe) - var unsafe = Suppress(base: group) - + let body2 = body + nonisolated(unsafe) + let block2 = { (barrier: isolated (any Actor)?) in + nonisolated(unsafe) + var iter = suppress.base + return Suppress(base: try await body2(&iter)) + } + async let mainTask = { + do { + let v = try await block2(actor) + await holder.markDone() + return v + } catch { + await holder.markDone() + throw error + } + }() + let errorRef:(any Error)? do { - let value = try await block(&unsafe.base) - group = unsafe.base - return value + // wait for subTask first to trigger priority elavation + // (release finished tasks as soon as possible) + try await subTask + errorRef = nil } catch { - group = unsafe.base - throw error + group.cancelAll() + errorRef = error } + nonisolated(unsafe) + let value = try await mainTask.base + if let errorRef { + throw errorRef + } + return .init(base: value) } + return wrapped.base } diff --git a/Sources/CriticalSection/Cell.swift b/Sources/CriticalSection/Cell.swift index 92627cc..3b298d0 100644 --- a/Sources/CriticalSection/Cell.swift +++ b/Sources/CriticalSection/Cell.swift @@ -4,9 +4,9 @@ // // Created by 박병관 on 6/26/24. // +#if $BuiltinAddressOfRawLayout import Builtin import Synchronization -#if $BuiltinAddressOfRawLayout @frozen @usableFromInline diff --git a/Sources/CriticalSection/DarwinImpl.swift b/Sources/CriticalSection/DarwinImpl.swift index d4b915c..dc2cd16 100644 --- a/Sources/CriticalSection/DarwinImpl.swift +++ b/Sources/CriticalSection/DarwinImpl.swift @@ -10,6 +10,7 @@ // //===----------------------------------------------------------------------===// +#if $BuiltinAddressOfRawLayout && canImport(Darwin) import Darwin @frozen @@ -40,3 +41,4 @@ public struct _MutexHandle: ~Copyable { } } +#endif diff --git a/Sources/NamespaceExtension/TetraExtended.swift b/Sources/NamespaceExtension/TetraExtended.swift index 4e2ab52..7d77ebf 100644 --- a/Sources/NamespaceExtension/TetraExtended.swift +++ b/Sources/NamespaceExtension/TetraExtended.swift @@ -12,10 +12,10 @@ public protocol TetraExtended { /// Static Tetra extension point. @inlinable - static var tetra: TetraExtension.Type { get set } + static var tetra: TetraExtension.Type { get } /// Instance Tetra extension point. @inlinable - var tetra: TetraExtension { get set } + var tetra: TetraExtension { get } } extension TetraExtended where Base == Self { @@ -24,14 +24,12 @@ extension TetraExtended where Base == Self { @inlinable public static var tetra: TetraExtension.Type { get { TetraExtension.self } - set {} } /// Instance Tetra extension point. @inlinable public var tetra: TetraExtension { get { TetraExtension(self) } - set {} } } diff --git a/Sources/Tetra/Combine/Publishers+AsyncFlatMap.swift b/Sources/Tetra/Combine/Publishers+AsyncFlatMap.swift index 8e5193b..3899110 100644 --- a/Sources/Tetra/Combine/Publishers+AsyncFlatMap.swift +++ b/Sources/Tetra/Combine/Publishers+AsyncFlatMap.swift @@ -143,13 +143,15 @@ extension AsyncFlatMap { ) } } else { - try? await simuateThrowingDiscardingTaskGroup(isolation: SafetyRegion()) { barrier, group in - defer { terminateStream() } - nonisolated(unsafe) - var unsafe = Suppress(value: group) - await localTask(isolation: barrier, group: &unsafe.value) - group = unsafe.value + let barrier:SafetyRegion = .init() + let block = { (isolation: isolated SafetyRegion) in + try? await simuateThrowingDiscardingTaskGroup2(isolation: isolation) { group in + defer { terminateStream() } + await localTask(isolation: isolation, group: &group) + } + return () } + await block(barrier) } send(completion: .finished, shouldCancel: false) @@ -229,8 +231,8 @@ extension AsyncFlatMap { } if let subscription { subscription.request(.max(1)) - effect?.run() } + effect?.run() } case .success(let success?): send( From 32073f61ca578fdef1717be62351e808c94dd055 Mon Sep 17 00:00:00 2001 From: pbk Date: Sat, 21 Dec 2024 23:02:24 +0900 Subject: [PATCH 51/63] disable PruneMemory feature in non-Objective-C platform --- Sources/Tetra/Foundation/PruneMemory.swift | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/Sources/Tetra/Foundation/PruneMemory.swift b/Sources/Tetra/Foundation/PruneMemory.swift index 46e7372..e529e4f 100644 --- a/Sources/Tetra/Foundation/PruneMemory.swift +++ b/Sources/Tetra/Foundation/PruneMemory.swift @@ -7,6 +7,7 @@ import Foundation +#if canImport(ObjectiveC) // since core foundation and objective-c foundation has it's internal optimzation @@ -228,3 +229,12 @@ enum MemoryErasing { } +#else + +@available(*, unavailable) +enum MemoryErasing { + + +} + +#endif From 3ea51abfabdd2c4b17951b6efdc9f18fb7170ed0 Mon Sep 17 00:00:00 2001 From: pbk Date: Sun, 22 Dec 2024 01:03:40 +0900 Subject: [PATCH 52/63] ensure isolation for backport discardingTask usecase --- Sources/BackportDiscardingTaskGroup/TaskGroup.swift | 7 ++++--- .../ThrowingTaskGroup.swift | 11 ++++++----- Sources/Tetra/Combine/ExperimentalMapTask.swift | 13 +++++++------ Sources/Tetra/Combine/Publishers+AsyncFlatMap.swift | 9 ++++----- 4 files changed, 21 insertions(+), 19 deletions(-) diff --git a/Sources/BackportDiscardingTaskGroup/TaskGroup.swift b/Sources/BackportDiscardingTaskGroup/TaskGroup.swift index a828a84..c7ef40d 100644 --- a/Sources/BackportDiscardingTaskGroup/TaskGroup.swift +++ b/Sources/BackportDiscardingTaskGroup/TaskGroup.swift @@ -8,7 +8,7 @@ @inlinable package func simuateDiscardingTaskGroup( isolation actor: isolated T = #isolation, - body: @Sendable (isolated T, inout TaskGroup) async -> sending TaskResult + body: (isolated T, inout TaskGroup) async -> sending TaskResult ) async -> sending TaskResult { let wrapped = await withTaskGroup(of: Void.self, returning: Suppress.self) { group in let holder: SafetyRegion = actor as? SafetyRegion ?? .init() @@ -32,10 +32,11 @@ package func simuateDiscardingTaskGroup( } } }() - + nonisolated(unsafe) + let body2 = body async let mainTask = { var iter = suppress.base - let v = await body(actor, &iter) + let v = await body2(actor, &iter) await holder.markDone() return Suppress(base: v) }() diff --git a/Sources/BackportDiscardingTaskGroup/ThrowingTaskGroup.swift b/Sources/BackportDiscardingTaskGroup/ThrowingTaskGroup.swift index c31fa87..6809d07 100644 --- a/Sources/BackportDiscardingTaskGroup/ThrowingTaskGroup.swift +++ b/Sources/BackportDiscardingTaskGroup/ThrowingTaskGroup.swift @@ -24,7 +24,7 @@ package func simuateThrowingDiscardingTaskGroup2( @inlinable package func simuateThrowingDiscardingTaskGroup( isolation actor: isolated T, - body: @Sendable (isolated T, inout ThrowingTaskGroup) async throws -> sending TaskResult + body: (isolated T, inout ThrowingTaskGroup) async throws -> sending TaskResult ) async throws -> sending TaskResult { let wrapped:Suppress = try await withThrowingTaskGroup(of: Void.self, returning: Suppress.self, isolation: actor) { group in let holder: SafetyRegion = actor as? SafetyRegion ?? .init() @@ -47,18 +47,19 @@ package func simuateThrowingDiscardingTaskGroup( } } }() - async let mainTask = { - nonisolated(unsafe) + nonisolated(unsafe) + let body2 = body + async let mainTask = { (isolation: isolated T) in var iter = suppress.base do { - let v = try await body(actor, &iter) + let v = try await body2(isolation, &iter) await holder.markDone() return Suppress(base: v) } catch { await holder.markDone() throw error } - }() + }(actor) let errorRef:(any Error)? do { // wait for subTask first to trigger priority elavation diff --git a/Sources/Tetra/Combine/ExperimentalMapTask.swift b/Sources/Tetra/Combine/ExperimentalMapTask.swift index 4655d68..a4c2f43 100644 --- a/Sources/Tetra/Combine/ExperimentalMapTask.swift +++ b/Sources/Tetra/Combine/ExperimentalMapTask.swift @@ -274,12 +274,13 @@ extension MultiMapTask { ) } } else { - await simuateDiscardingTaskGroup(isolation: SafetyRegion()) { @Sendable actor, group in - nonisolated(unsafe) - var unsafe = Suppress(value: group) - await localTask(isolation: actor, group: &unsafe.value) - group = unsafe.value - } + let barrier = SafetyRegion() + await { (a:isolated SafetyRegion) in + await simuateDiscardingTaskGroup(isolation: a) { actor, group in + await localTask(isolation: actor, group: &group) + } + return () + }(barrier) } // we assume no one is accessing other state // except `Subscription.cancel()` diff --git a/Sources/Tetra/Combine/Publishers+AsyncFlatMap.swift b/Sources/Tetra/Combine/Publishers+AsyncFlatMap.swift index 3899110..0189714 100644 --- a/Sources/Tetra/Combine/Publishers+AsyncFlatMap.swift +++ b/Sources/Tetra/Combine/Publishers+AsyncFlatMap.swift @@ -144,14 +144,13 @@ extension AsyncFlatMap { } } else { let barrier:SafetyRegion = .init() - let block = { (isolation: isolated SafetyRegion) in - try? await simuateThrowingDiscardingTaskGroup2(isolation: isolation) { group in + await { (isolation: isolated SafetyRegion) in + try? await simuateThrowingDiscardingTaskGroup(isolation: isolation) { defer { terminateStream() } - await localTask(isolation: isolation, group: &group) + await localTask(isolation: $0, group: &$1) } return () - } - await block(barrier) + }(barrier) } send(completion: .finished, shouldCancel: false) From c60b47f6dfe9203aa33ad658fe0b9960e61a64db Mon Sep 17 00:00:00 2001 From: pbk Date: Thu, 2 Jan 2025 15:08:59 +0900 Subject: [PATCH 53/63] refactor for concurrency checking --- Package.swift | 1 + .../AsyncFlatMapSequence.swift | 10 ++--- .../AsyncMapErrorSequence.swift | 2 + .../TaskGroup.swift | 40 +++++++++---------- .../ThrowingTaskGroup.swift | 33 +++++++-------- 5 files changed, 43 insertions(+), 43 deletions(-) diff --git a/Package.swift b/Package.swift index f624fff..bc0e9d6 100644 --- a/Package.swift +++ b/Package.swift @@ -65,6 +65,7 @@ let package = Package( name: "BackportDiscardingTaskGroup", dependencies: [ "Namespace", + "CriticalSection", ], swiftSettings: [ .enableUpcomingFeature("FullTypedThrows"), diff --git a/Sources/BackPortAsyncSequence/AsyncFlatMapSequence.swift b/Sources/BackPortAsyncSequence/AsyncFlatMapSequence.swift index 731bd5f..6e50da1 100644 --- a/Sources/BackPortAsyncSequence/AsyncFlatMapSequence.swift +++ b/Sources/BackPortAsyncSequence/AsyncFlatMapSequence.swift @@ -97,12 +97,12 @@ extension BackPort.AsyncFlatMapSequence.Iterator: AsyncIteratorProtocol, TypedAs return nil } let block = transform - let wrapper = { - let a = try await block($0) + let wrapper = { (input:Suppress) async throws(Failure) in + let a = try await block(input.base) return Suppress(base: a) } - do { - let segment: SegmentOfResult = try await wrapper(Suppress(base: item).base).base + do throws(Failure) { + let segment: SegmentOfResult = try await wrapper(.init(base: item)).base var iterator = segment.makeAsyncIterator() guard let element = try await iterator.next(isolation: actor) else { currentIterator = nil @@ -113,7 +113,7 @@ extension BackPort.AsyncFlatMapSequence.Iterator: AsyncIteratorProtocol, TypedAs } catch { finished = true currentIterator = nil - throw error as! Failure + throw error } } } diff --git a/Sources/BackPortAsyncSequence/AsyncMapErrorSequence.swift b/Sources/BackPortAsyncSequence/AsyncMapErrorSequence.swift index 22db97b..2240e1b 100644 --- a/Sources/BackPortAsyncSequence/AsyncMapErrorSequence.swift +++ b/Sources/BackPortAsyncSequence/AsyncMapErrorSequence.swift @@ -5,6 +5,8 @@ // Created by 박병관 on 6/13/24. // +import Namespace + public struct AsyncMapErrorSequence where Base.AsyncIterator: TypedAsyncIteratorProtocol { @usableFromInline diff --git a/Sources/BackportDiscardingTaskGroup/TaskGroup.swift b/Sources/BackportDiscardingTaskGroup/TaskGroup.swift index c7ef40d..09d8777 100644 --- a/Sources/BackportDiscardingTaskGroup/TaskGroup.swift +++ b/Sources/BackportDiscardingTaskGroup/TaskGroup.swift @@ -5,10 +5,12 @@ // Created by 박병관 on 6/20/24. // -@inlinable +internal import CriticalSection + +@usableFromInline package func simuateDiscardingTaskGroup( isolation actor: isolated T = #isolation, - body: (isolated T, inout TaskGroup) async -> sending TaskResult + body: (isolated T, inout TaskGroup) async -> TaskResult ) async -> sending TaskResult { let wrapped = await withTaskGroup(of: Void.self, returning: Suppress.self) { group in let holder: SafetyRegion = actor as? SafetyRegion ?? .init() @@ -23,23 +25,23 @@ package func simuateDiscardingTaskGroup( } let suppress = Suppress(base: group) /// drain all the finished or failed Task - async let subTask:Void = { - nonisolated(unsafe) + async let subTask:Void = { (act: isolated T) in +// nonisolated(unsafe) var iter = suppress.base - while let _ = await iter.next(isolation: actor) { + while let _ = await iter.next(isolation: act) { if await holder.isFinished { break } } - }() + }(actor) nonisolated(unsafe) let body2 = body - async let mainTask = { + async let mainTask = { (act: isolated T) in var iter = suppress.base - let v = await body2(actor, &iter) + let v = await body2(act, &iter) await holder.markDone() return Suppress(base: v) - }() + }(actor) await subTask let value = await mainTask.base return Suppress(base: value) @@ -111,18 +113,12 @@ internal func __simuateDiscardingTaskGroup2( }(actor) nonisolated(unsafe) let body2 = body - nonisolated(unsafe) - let block2 = { (barrier: isolated (any Actor)?) in + async let mainTask = { (barrier: isolated (any Actor)?) in var iter = suppress.base - return Suppress(base: await body2(&iter)) - } - async let mainTask = { - do { - let v = await block2(actor) - await holder.markDone() - return v - } - }() + let v = await body2(&iter) + await holder.markDone() + return Suppress(base: v) + }(actor) do { // wait for subTask first to trigger priority elavation // (release finished tasks as soon as possible) @@ -135,3 +131,7 @@ internal func __simuateDiscardingTaskGroup2( } return wrapped.base } + +extension Suppress: BitwiseCopyable where Base: BitwiseCopyable { + +} diff --git a/Sources/BackportDiscardingTaskGroup/ThrowingTaskGroup.swift b/Sources/BackportDiscardingTaskGroup/ThrowingTaskGroup.swift index 6809d07..703a87c 100644 --- a/Sources/BackportDiscardingTaskGroup/ThrowingTaskGroup.swift +++ b/Sources/BackportDiscardingTaskGroup/ThrowingTaskGroup.swift @@ -24,7 +24,7 @@ package func simuateThrowingDiscardingTaskGroup2( @inlinable package func simuateThrowingDiscardingTaskGroup( isolation actor: isolated T, - body: (isolated T, inout ThrowingTaskGroup) async throws -> sending TaskResult + body: (isolated T, inout ThrowingTaskGroup) async throws -> TaskResult ) async throws -> sending TaskResult { let wrapped:Suppress = try await withThrowingTaskGroup(of: Void.self, returning: Suppress.self, isolation: actor) { group in let holder: SafetyRegion = actor as? SafetyRegion ?? .init() @@ -38,15 +38,15 @@ package func simuateThrowingDiscardingTaskGroup( } let suppress = Suppress(base: group) /// drain all the finished or failed Task - async let subTask:Void = { - nonisolated(unsafe) + async let subTask:Void = { (barrier: isolated T) in +// nonisolated(unsafe) var iter = suppress.base - while let _ = try await iter.next(isolation: actor) { + while let _ = try await iter.next(isolation: barrier) { if await holder.isFinished { break } } - }() + }(actor) nonisolated(unsafe) let body2 = body async let mainTask = { (isolation: isolated T) in @@ -97,33 +97,30 @@ internal func __simuateThrowingDiscardingTaskGroup2( } let suppress = Suppress(base: group) /// drain all the finished or failed Task - async let subTask:Void = { + async let subTask:Void = {(barrier: isolated (any Actor)?) in nonisolated(unsafe) var iter = suppress.base - while let _ = try await iter.next(isolation: actor) { + while let _ = try await iter.next(isolation: barrier) { if await holder.isFinished { break } } - }() + }(#isolation) nonisolated(unsafe) let body2 = body - nonisolated(unsafe) - let block2 = { (barrier: isolated (any Actor)?) in - nonisolated(unsafe) - var iter = suppress.base - return Suppress(base: try await body2(&iter)) - } - async let mainTask = { + async let mainTask = { (barrier: isolated (any Actor)?) in do { - let v = try await block2(actor) + nonisolated(unsafe) + var iter = suppress.base + + let v = try await body2(&iter) await holder.markDone() - return v + return Suppress(base: v) } catch { await holder.markDone() throw error } - }() + }(#isolation) let errorRef:(any Error)? do { // wait for subTask first to trigger priority elavation From bc0ddb93b47479f107747050ff701a5939147744 Mon Sep 17 00:00:00 2001 From: pbk Date: Sun, 5 Jan 2025 20:47:02 +0900 Subject: [PATCH 54/63] add isolation parameter --- Sources/CriticalSection/CompatMutex.swift | 8 ++++ .../Tetra/Combine/ExperimentalMapTask.swift | 4 +- .../Combine/Publishers+AsyncFlatMap.swift | 4 +- .../Tetra/Combine/Publishers+MapTask.swift | 8 ++-- .../Tetra/Combine/Publishers+TryMapTask.swift | 12 +++--- .../Concurrency/AsyncSequencePublisher.swift | 4 +- Sources/Tetra/Concurrency/BroadCast2.swift | 8 ++++ .../Notification+AsyncSequence.swift | 38 +++++++++++-------- 8 files changed, 55 insertions(+), 31 deletions(-) create mode 100644 Sources/CriticalSection/CompatMutex.swift create mode 100644 Sources/Tetra/Concurrency/BroadCast2.swift diff --git a/Sources/CriticalSection/CompatMutex.swift b/Sources/CriticalSection/CompatMutex.swift new file mode 100644 index 0000000..953c54a --- /dev/null +++ b/Sources/CriticalSection/CompatMutex.swift @@ -0,0 +1,8 @@ +// +// File.swift +// Tetra +// +// Created by 박병관 on 1/4/25. +// + +import Foundation diff --git a/Sources/Tetra/Combine/ExperimentalMapTask.swift b/Sources/Tetra/Combine/ExperimentalMapTask.swift index a4c2f43..486e4e0 100644 --- a/Sources/Tetra/Combine/ExperimentalMapTask.swift +++ b/Sources/Tetra/Combine/ExperimentalMapTask.swift @@ -212,7 +212,7 @@ extension MultiMapTask { } } - private func waitForUpStream() async throws { + private func waitForUpStream( isolation actor:isolated (any Actor)? = #isolation) async throws { try await withTaskCancellationHandler { try await withUnsafeThrowingContinuation { coninuation in state.withLockUnchecked{ @@ -232,7 +232,7 @@ extension MultiMapTask { }?.run() } - private func waitForCondition() async throws { + private func waitForCondition( isolation actor:isolated (any Actor)? = #isolation) async throws { try await withUnsafeThrowingContinuation{ continuation in state.withLock{ $0.condition.transition(.suspend(continuation)) diff --git a/Sources/Tetra/Combine/Publishers+AsyncFlatMap.swift b/Sources/Tetra/Combine/Publishers+AsyncFlatMap.swift index 0189714..3ebe32c 100644 --- a/Sources/Tetra/Combine/Publishers+AsyncFlatMap.swift +++ b/Sources/Tetra/Combine/Publishers+AsyncFlatMap.swift @@ -295,7 +295,7 @@ extension AsyncFlatMap { } // contention case - private func waitForUpStream() async throws { + private func waitForUpStream( isolation actor:isolated (any Actor)? = #isolation) async throws { try await withTaskCancellationHandler { try await withUnsafeThrowingContinuation { coninuation in lock.withLockUnchecked{ @@ -317,7 +317,7 @@ extension AsyncFlatMap { } // contention case - private func waitForCondition() async throws { + private func waitForCondition( isolation actor:isolated (any Actor)? = #isolation) async throws { try await withUnsafeThrowingContinuation{ continuation in lock.withLock{ $0.taskCondition.transition(.suspend(continuation)) diff --git a/Sources/Tetra/Combine/Publishers+MapTask.swift b/Sources/Tetra/Combine/Publishers+MapTask.swift index 426fccc..f3e2188 100644 --- a/Sources/Tetra/Combine/Publishers+MapTask.swift +++ b/Sources/Tetra/Combine/Publishers+MapTask.swift @@ -154,7 +154,7 @@ extension MapTask { valueSource.continuation.finish() } - private func waitForUpStream() async throws { + private func waitForUpStream( isolation actor:isolated (any Actor)? = #isolation) async throws { try await withTaskCancellationHandler { try await withUnsafeThrowingContinuation { coninuation in state.withLockUnchecked{ @@ -174,7 +174,7 @@ extension MapTask { }?.run() } - private func waitForCondition() async throws { + private func waitForCondition( isolation actor:isolated (any Actor)? = #isolation) async throws { try await withUnsafeThrowingContinuation{ continuation in state.withLock{ $0.condition.transition(.suspend(continuation)) @@ -190,7 +190,7 @@ extension MapTask { @Sendable nonisolated func run() async { - let token:Void? = try? await waitForCondition() + let token:Void? = try? await waitForCondition(isolation: transform.isolation) if token == nil { withUnsafeCurrentTask{ $0?.cancel() @@ -199,7 +199,7 @@ extension MapTask { defer { clearCondition() } - let success: Void? = try? await waitForUpStream() + let success: Void? = try? await waitForUpStream(isolation: transform.isolation) defer { terminateStream() } state.withLockUnchecked{ $0.subscriber diff --git a/Sources/Tetra/Combine/Publishers+TryMapTask.swift b/Sources/Tetra/Combine/Publishers+TryMapTask.swift index 00b2e19..d10efc5 100644 --- a/Sources/Tetra/Combine/Publishers+TryMapTask.swift +++ b/Sources/Tetra/Combine/Publishers+TryMapTask.swift @@ -140,8 +140,8 @@ extension TryMapTask { valueSource.continuation.finish() } - nonisolated - private func waitForUpStream() async throws { + + private func waitForUpStream( isolation actor:isolated (any Actor)? = #isolation) async throws { try await withTaskCancellationHandler { try await withUnsafeThrowingContinuation { coninuation in state.withLockUnchecked { @@ -161,8 +161,8 @@ extension TryMapTask { }?.run() } - nonisolated - private func waitForCondition() async throws { + + private func waitForCondition( isolation actor:isolated (any Actor)? = #isolation) async throws { try await withUnsafeThrowingContinuation{ continuation in state.withLock{ $0.condition.transition(.suspend(continuation)) @@ -178,7 +178,7 @@ extension TryMapTask { @Sendable nonisolated func run() async { - let token:Void? = try? await waitForCondition() + let token:Void? = try? await waitForCondition(isolation: transform.isolation) if token == nil { withUnsafeCurrentTask{ $0?.cancel() @@ -187,7 +187,7 @@ extension TryMapTask { defer { clearCondition() } - let subscription: Void? = try? await waitForUpStream() + let subscription: Void? = try? await waitForUpStream(isolation: transform.isolation) state.withLockUnchecked{ $0.subscriber }?.receive(subscription: self) diff --git a/Sources/Tetra/Concurrency/AsyncSequencePublisher.swift b/Sources/Tetra/Concurrency/AsyncSequencePublisher.swift index 58f29f6..decfd95 100644 --- a/Sources/Tetra/Concurrency/AsyncSequencePublisher.swift +++ b/Sources/Tetra/Concurrency/AsyncSequencePublisher.swift @@ -161,7 +161,7 @@ extension AsyncSequencePublisher { }?.run() } - private func waitForCondition() async throws { + private func waitForCondition( _ actor: isolated (any Actor)? = #isolation) async throws { try await withUnsafeThrowingContinuation{ continuation in state.withLock{ $0.condition.transition(.suspend(continuation)) @@ -169,7 +169,7 @@ extension AsyncSequencePublisher { } } - private func nextDemand() async -> Subscribers.Demand? { + private func nextDemand( _ actor: isolated (any Actor)? = #isolation) async -> Subscribers.Demand? { await withUnsafeContinuation{ continuation in let demand:Subscribers.Demand? = state.withLock{ if $0.demand > .none { diff --git a/Sources/Tetra/Concurrency/BroadCast2.swift b/Sources/Tetra/Concurrency/BroadCast2.swift new file mode 100644 index 0000000..953c54a --- /dev/null +++ b/Sources/Tetra/Concurrency/BroadCast2.swift @@ -0,0 +1,8 @@ +// +// File.swift +// Tetra +// +// Created by 박병관 on 1/4/25. +// + +import Foundation diff --git a/Sources/Tetra/Concurrency/Notification+AsyncSequence.swift b/Sources/Tetra/Concurrency/Notification+AsyncSequence.swift index cf39d5f..735ecf9 100644 --- a/Sources/Tetra/Concurrency/Notification+AsyncSequence.swift +++ b/Sources/Tetra/Concurrency/Notification+AsyncSequence.swift @@ -11,6 +11,7 @@ import _Concurrency public import BackPortAsyncSequence import Namespace +internal import struct DequeModule.Deque public import CriticalSection @@ -52,7 +53,8 @@ public final class NotificationSequence: AsyncSequence, Sendable, TypedAsyncSequ operation: { [parent] in await parent.next(isolation: actor) }, - onCancel: parent.cancel + onCancel: parent.cancel, + isolation: actor ) } @@ -66,12 +68,25 @@ public final class NotificationSequence: AsyncSequence, Sendable, TypedAsyncSequ @usableFromInline internal struct NotficationState { - @usableFromInline - var buffer:[Notification] = [] - @usableFromInline - var pending:[UnsafeContinuation] = [] +// @usableFromInline + var buffer:Deque = [] +// @usableFromInline + var pending:Deque> = [] @usableFromInline var observer:NSObjectProtocol? + + @usableFromInline + mutating func resume(_ value:Notification) -> UnsafeContinuation? { + let captured = pending.first + + if pending.isEmpty { + buffer.append(value) + } else { + pending.removeFirst() + } + return captured + } + } @inlinable @@ -84,14 +99,7 @@ public final class NotificationSequence: AsyncSequence, Sendable, TypedAsyncSequ let observer = center.addObserver(forName: name, object: object, queue: nil) { [lock] notification in let continuation = lock.withLockUnchecked { state in - let captured = state.pending.first - - if state.pending.isEmpty { - state.buffer.append(notification) - } else { - state.pending.removeFirst() - } - return captured + return state.resume(notification) } continuation?.resume(returning: Suppress(value: notification).value) } @@ -122,8 +130,8 @@ public final class NotificationSequence: AsyncSequence, Sendable, TypedAsyncSequ } @usableFromInline - func next(isolation: isolated (any Actor)?) async -> Notification? { - await withUnsafeContinuation { continuation in + func next(isolation: isolated (any Actor)? = #isolation) async -> Notification? { + await withUnsafeContinuation(isolation: isolation) { continuation in let (notification, isCancelled) = lock.withLockUnchecked { state in if !state.buffer.isEmpty { return (state.buffer.removeFirst() as Notification?, false) From 271ed0ab3338034767f4e0cd953cd1ca94f48599 Mon Sep 17 00:00:00 2001 From: park-byeong-gwan Date: Mon, 6 Jan 2025 19:49:58 +0900 Subject: [PATCH 55/63] spawn child task for receive(subscription:) callback --- .../CriticalSection/ManagedUnfairLock.swift | 15 +- .../Combine/AsyncFlatMapDemandState.swift | 2 +- .../Tetra/Combine/ExperimentalMapTask.swift | 3 +- .../Combine/Publishers+AsyncFlatMap.swift | 3 +- .../Tetra/Combine/Publishers+MapTask.swift | 235 ------------------ .../Tetra/Combine/Publishers+TryMapTask.swift | 223 ----------------- .../Concurrency/AsyncSequencePublisher.swift | 10 +- 7 files changed, 25 insertions(+), 466 deletions(-) diff --git a/Sources/CriticalSection/ManagedUnfairLock.swift b/Sources/CriticalSection/ManagedUnfairLock.swift index 397f61f..d7dee90 100644 --- a/Sources/CriticalSection/ManagedUnfairLock.swift +++ b/Sources/CriticalSection/ManagedUnfairLock.swift @@ -8,6 +8,17 @@ import Foundation import os +@usableFromInline +internal final class LockBuffer: ManagedBuffer { + + @usableFromInline + deinit { + self.withUnsafeMutablePointerToElements{ + let _ = $0.deinitialize(count: 1) + } + } +} + @available(iOS, deprecated: 16.0, renamed: "OSAllocatedUnfairLock") @available(tvOS, deprecated: 16.0, renamed: "OSAllocatedUnfairLock") @available(macCatalyst, deprecated: 16.0, renamed: "OSAllocatedUnfairLock") @@ -31,7 +42,7 @@ public struct ManagedUnfairLock: @unchecked Sendable { /// @inlinable public init(uncheckedState initialState: State) { - __lock = .create(minimumCapacity: 1) { buffer in + __lock = LockBuffer.create(minimumCapacity: 1) { buffer in buffer.withUnsafeMutablePointerToElements{ $0.initialize(to: .init()) } return initialState } @@ -139,7 +150,7 @@ public extension ManagedUnfairLock where State == Void { /// Initialize an SwiftUnfairLock with no protected state. @inlinable init() { - __lock = .create(minimumCapacity: 1) { buffer in + __lock = LockBuffer.create(minimumCapacity: 1) { buffer in buffer.withUnsafeMutablePointerToElements{ $0.initialize(to: .init()) } } } diff --git a/Sources/Tetra/Combine/AsyncFlatMapDemandState.swift b/Sources/Tetra/Combine/AsyncFlatMapDemandState.swift index 00f625c..8c71b35 100644 --- a/Sources/Tetra/Combine/AsyncFlatMapDemandState.swift +++ b/Sources/Tetra/Combine/AsyncFlatMapDemandState.swift @@ -7,7 +7,7 @@ import Foundation import Combine -internal import DequeModule +internal import struct DequeModule.Deque struct AsyncFlatMapDemandState: Sendable { diff --git a/Sources/Tetra/Combine/ExperimentalMapTask.swift b/Sources/Tetra/Combine/ExperimentalMapTask.swift index 486e4e0..8858757 100644 --- a/Sources/Tetra/Combine/ExperimentalMapTask.swift +++ b/Sources/Tetra/Combine/ExperimentalMapTask.swift @@ -260,9 +260,10 @@ extension MultiMapTask { } // contention can happen with `receive(subscription:) let success:Void? = try? await waitForUpStream() - state.withLockUnchecked{ + async let job:Void? = state.withLockUnchecked{ $0.subscriber }?.receive(subscription: self) + await job guard success != nil else { terminateStream() return diff --git a/Sources/Tetra/Combine/Publishers+AsyncFlatMap.swift b/Sources/Tetra/Combine/Publishers+AsyncFlatMap.swift index 3ebe32c..3ce24f9 100644 --- a/Sources/Tetra/Combine/Publishers+AsyncFlatMap.swift +++ b/Sources/Tetra/Combine/Publishers+AsyncFlatMap.swift @@ -128,9 +128,10 @@ extension AsyncFlatMap { clearCondition() } let success:Void? = try? await waitForUpStream() - lock.withLockUnchecked{ + async let job:Void? = lock.withLockUnchecked{ $0.subscriber }?.receive(subscription: self) + await job guard success != nil else { terminateStream() return diff --git a/Sources/Tetra/Combine/Publishers+MapTask.swift b/Sources/Tetra/Combine/Publishers+MapTask.swift index f3e2188..ddaf701 100644 --- a/Sources/Tetra/Combine/Publishers+MapTask.swift +++ b/Sources/Tetra/Combine/Publishers+MapTask.swift @@ -85,238 +85,3 @@ public struct MapTask: Publisher where Upstream.Outp extension MapTask: Sendable where Upstream: Sendable {} - -extension MapTask { - - - struct TaskState where S.Failure == Failure, S.Input == Output { - - var subscriber:S? = nil - var upstreamSubscription = AsyncSubscriptionState.waiting - var condition = TaskValueContinuation.waiting - var isSleeping = true - var pending = Subscribers.Demand.none - } - - struct Inner: CustomCombineIdentifierConvertible, Sendable where S.Failure == Failure, S.Input == Output { - - private let valueSource = AsyncStream>.makeStream(bufferingPolicy: .bufferingNewest(2)) - private let state: some UnfairStateLock> = createUncheckedStateLock(uncheckedState: .init()) - private let transform: Transform - let combineIdentifier = CombineIdentifier() - - init( - subscriber:S, - transform: @escaping Transform - ) { - self.transform = transform - state.withLockUnchecked{ $0.subscriber = subscriber } - } - - - private func send(completion: Subscribers.Completion?, cancel:Bool = false) { - let (subscriber, effect, taskEffect) = state.withLockUnchecked{ - let old = $0.subscriber - $0.subscriber = nil - let effect = if cancel { - $0.upstreamSubscription.transition(.cancel) - } else { - $0.upstreamSubscription.transition(.finish) - } - let taskEffect = if cancel { - $0.condition.transition(.cancel) - } else { - $0.condition.transition(.finish) - } - return (old, effect, taskEffect) - } - // tell compiler we hope to remove these objects as soon as possible - (consume effect)?.run() - if let completion { - (consume subscriber)?.receive(completion: completion) - } - (consume taskEffect)?.run() - } - - private func send(_ value:Output) throws { - let subscriber = state.withLockUnchecked{ - $0.isSleeping = true - return $0.subscriber - } - guard let subscriber else { - throw CancellationError() - } - let demand = subscriber.receive(value) - request(demand) - } - - private func terminateStream() { - valueSource.continuation.finish() - } - - private func waitForUpStream( isolation actor:isolated (any Actor)? = #isolation) async throws { - try await withTaskCancellationHandler { - try await withUnsafeThrowingContinuation { coninuation in - state.withLockUnchecked{ - $0.upstreamSubscription.transition(.suspend(coninuation)) - }?.run() - } - } onCancel: { [state] in - state.withLockUnchecked{ - $0.upstreamSubscription.transition(.cancel) - }?.run() - } - } - - func resumeCondition(_ task:Task) { - state.withLock{ - $0.condition.transition(.resume(task)) - }?.run() - } - - private func waitForCondition( isolation actor:isolated (any Actor)? = #isolation) async throws { - try await withUnsafeThrowingContinuation{ continuation in - state.withLock{ - $0.condition.transition(.suspend(continuation)) - }?.run() - } - } - - private func clearCondition() { - state.withLock{ - $0.condition.transition(.finish) - }?.run() - } - - @Sendable - nonisolated func run() async { - let token:Void? = try? await waitForCondition(isolation: transform.isolation) - if token == nil { - withUnsafeCurrentTask{ - $0?.cancel() - } - } - defer { - clearCondition() - } - let success: Void? = try? await waitForUpStream(isolation: transform.isolation) - defer { terminateStream() } - state.withLockUnchecked{ - $0.subscriber - }?.receive(subscription: self) - guard success != nil else { - return - } - await runIn(isolation: transform.isolation) - - } - - internal func runIn( - isolation actor:isolated (any Actor)? = #isolation - ) async { - let block = { @Sendable in - let value = await transform($0) - return value.map(Suppress.init) - } - for await upstreamResult in valueSource.stream { - let upValue: Upstream.Output - switch upstreamResult { - case .failure(let error): - send(completion: .failure(error), cancel: false) - return - case .success(let value): - upValue = value - } - - // enqueue to separate task to prevent transformer cancelling root task using `UnsafeCurrentTask` - async let job = block(upValue) - switch (await job).map(\.value) { - case .failure(let error): - send(completion: .failure(error), cancel: true) - return - case .success(let value): - do { - try send(value) - } catch { - return - } - } - } - send(completion: .finished) - } - - } - -} - -extension MapTask.Inner: Subscriber { - - func receive(subscription: any Subscription) { - state.withLockUnchecked { - $0.upstreamSubscription.transition(.resume(subscription)) - }?.run() - } - - func receive(_ input: Upstream.Output) -> Subscribers.Demand { - let result = valueSource.continuation.yield(.success(input)) - switch result { - case .terminated: - break - case .enqueued: - break - case .dropped: - preconditionFailure("buffer overflow") - @unknown default: - fatalError("unknown case") - } - return .none - } - - func receive(completion: Subscribers.Completion) { - state.withLockUnchecked { - $0.upstreamSubscription.transition(.finish) - }?.run() - switch completion { - case .finished: - break - case .failure(let failure): - valueSource.continuation.yield(.failure(failure)) - } - valueSource.continuation.finish() - } - - -} - -extension MapTask.Inner: Subscription { - - func cancel() { - send(completion: nil, cancel: true) - } - - func request(_ demand: Subscribers.Demand) { - let subscription = state.withLockUnchecked { - $0.pending += demand - if $0.isSleeping && $0.pending > .none { - $0.isSleeping = false - $0.pending -= 1 - return $0.upstreamSubscription.subscription - } else { - - return nil - } - } - subscription?.request(.max(1)) - } - - -} - -extension MapTask.Inner: CustomStringConvertible, CustomPlaygroundDisplayConvertible { - - var playgroundDescription: Any { description } - - var description: String { "MapTask" } - -} - diff --git a/Sources/Tetra/Combine/Publishers+TryMapTask.swift b/Sources/Tetra/Combine/Publishers+TryMapTask.swift index d10efc5..639d595 100644 --- a/Sources/Tetra/Combine/Publishers+TryMapTask.swift +++ b/Sources/Tetra/Combine/Publishers+TryMapTask.swift @@ -72,226 +72,3 @@ public struct TryMapTask: Publisher where Upstream.O extension TryMapTask: Sendable where Upstream: Sendable {} - -extension TryMapTask { - - internal struct TaskState where S.Failure == Failure, S.Input == Output { - - var subscriber:S? = nil - var upstreamSubscription = AsyncSubscriptionState.waiting - var condition = TaskValueContinuation.waiting - var isSleeping = true - var pending = Subscribers.Demand.none - } - - internal struct Inner: Sendable, CustomCombineIdentifierConvertible where S.Failure == Failure, S.Input == Output { - - private let valueSource = AsyncThrowingStream.makeStream(bufferingPolicy: .bufferingNewest(2)) - private let state: some UnfairStateLock> = createUncheckedStateLock(uncheckedState: .init()) - private let transform: Transform - let combineIdentifier = CombineIdentifier() - - init( - subscriber:S, - transform: @escaping Transform - ) { - - self.transform = transform - state.withLockUnchecked{ $0.subscriber = subscriber } - } - - private func send(completion: Subscribers.Completion?, cancel:Bool = false) { - terminateStream() - let (subscriber, effect, taskEffect) = state.withLockUnchecked{ - let old = $0.subscriber - $0.subscriber = nil - let effect = if cancel { - $0.upstreamSubscription.transition(.cancel) - } else { - $0.upstreamSubscription.transition(.finish) - } - let taskEffect = if cancel { - $0.condition.transition(.cancel) - } else { - $0.condition.transition(.finish) - } - return (old, effect, taskEffect) - } - (consume effect)?.run() - if let completion { - (consume subscriber)?.receive(completion: completion) - } - (consume taskEffect)?.run() - } - - private func send(_ value:Output) throws { - let subscriber = state.withLockUnchecked{ - $0.isSleeping = true - return $0.subscriber - } - guard let subscriber else { - throw CancellationError() - } - let demand = subscriber.receive(value) - request(demand) - } - - private func terminateStream() { - valueSource.continuation.finish() - } - - - private func waitForUpStream( isolation actor:isolated (any Actor)? = #isolation) async throws { - try await withTaskCancellationHandler { - try await withUnsafeThrowingContinuation { coninuation in - state.withLockUnchecked { - $0.upstreamSubscription.transition(.suspend(coninuation)) - }?.run() - } - } onCancel: { - state.withLockUnchecked{ - $0.upstreamSubscription.transition(.cancel) - }?.run() - } - } - - func resumeCondition(_ task:Task) { - state.withLock{ - $0.condition.transition(.resume(task)) - }?.run() - } - - - private func waitForCondition( isolation actor:isolated (any Actor)? = #isolation) async throws { - try await withUnsafeThrowingContinuation{ continuation in - state.withLock{ - $0.condition.transition(.suspend(continuation)) - }?.run() - } - } - - private func clearCondition() { - state.withLock{ - $0.condition.transition(.finish) - }?.run() - } - - @Sendable - nonisolated func run() async { - let token:Void? = try? await waitForCondition(isolation: transform.isolation) - if token == nil { - withUnsafeCurrentTask{ - $0?.cancel() - } - } - defer { - clearCondition() - } - let subscription: Void? = try? await waitForUpStream(isolation: transform.isolation) - state.withLockUnchecked{ - $0.subscriber - }?.receive(subscription: self) - defer { terminateStream() } - guard subscription != nil else { - return - } - await runInIsolation(isolation: transform.isolation) - } - - func runInIsolation( - isolation actor: isolated (any Actor)? = #isolation - ) async { - let block = { @Sendable in - let value = try await transform($0) - return Suppress(value: value) - } - do { - for try await upValue in valueSource.stream { - do { - // enqueue to separate task to prevent transformer cancelling root task using `UnsafeCurrentTask` - async let job = block(upValue) - let value = try await job.value - guard let _ = try? send(value) else { - return - } - } catch { - send(completion: .failure(error), cancel: true) - return - } - } - send(completion: .finished, cancel: false) - } catch { - send(completion: .failure(error), cancel: false) - } - } - - } - - -} - -extension TryMapTask.Inner: Subscription { - - func cancel() { - send(completion: nil, cancel: true) - } - - func request(_ demand: Subscribers.Demand) { - let subscription = state.withLockUnchecked { - $0.pending += demand - if $0.isSleeping && $0.pending > .none { - $0.isSleeping = false - $0.pending -= 1 - return $0.upstreamSubscription.subscription - } else { - - return nil - } - } - subscription?.request(.max(1)) - } - -} - -extension TryMapTask.Inner: Subscriber { - - func receive(subscription: any Subscription) { - state.withLockUnchecked { - $0.upstreamSubscription.transition(.resume(subscription)) - }?.run() - } - - func receive(_ input: Upstream.Output) -> Subscribers.Demand { - let result = valueSource.continuation.yield(input) - switch result { - case .enqueued, .terminated: - break - case .dropped: - preconditionFailure("Buffer overflow") - @unknown default: - fatalError("unknown case") - } - return .none - } - - func receive(completion: Subscribers.Completion) { - state.withLockUnchecked { - $0.upstreamSubscription.transition(.finish) - }?.run() - switch completion { - case .finished: - valueSource.continuation.finish(throwing: nil) - case .failure(let failure): - valueSource.continuation.finish(throwing: failure) - } - } - -} - -extension TryMapTask.Inner: CustomStringConvertible, CustomPlaygroundDisplayConvertible { - - var playgroundDescription: Any { description } - - var description: String { "TryMapTask" } - -} diff --git a/Sources/Tetra/Concurrency/AsyncSequencePublisher.swift b/Sources/Tetra/Concurrency/AsyncSequencePublisher.swift index decfd95..e82a84f 100644 --- a/Sources/Tetra/Concurrency/AsyncSequencePublisher.swift +++ b/Sources/Tetra/Concurrency/AsyncSequencePublisher.swift @@ -204,9 +204,13 @@ extension AsyncSequencePublisher { if token == nil { send(completion: nil) } - state.withLockUnchecked{ - $0.subscriber - }?.receive(subscription: self) + + async let job:Void = { (actor:isolated (any Actor)?) in + state.withLockUnchecked{ + $0.subscriber + }?.receive(subscription: self) + }(actor) + await job do { while var pending = await nextDemand() { while pending > .none { From e83bb22a660d65ce1496f920e9035b0c22cf5d9c Mon Sep 17 00:00:00 2001 From: pbk Date: Sun, 26 Jan 2025 18:24:10 +0900 Subject: [PATCH 56/63] adapt Swift Testing for some test case --- Tests/TetraTests/AnyEncodableTests.swift | 59 ++++++++++-------------- Tests/TetraTests/PlistWrapperTests.swift | 55 +++++++++++++--------- 2 files changed, 59 insertions(+), 55 deletions(-) diff --git a/Tests/TetraTests/AnyEncodableTests.swift b/Tests/TetraTests/AnyEncodableTests.swift index ecf1eb8..98412c5 100644 --- a/Tests/TetraTests/AnyEncodableTests.swift +++ b/Tests/TetraTests/AnyEncodableTests.swift @@ -4,60 +4,51 @@ // // Created by pbk on 2023/01/27. // - -import XCTest +import Foundation @testable import Tetra +import Testing -final class AnyEncodableTests: XCTestCase { +@Suite +struct AnyEncodableTests { + @Test func testURLEncoding() throws { let targetURL = FileManager.default.temporaryDirectory - - XCTAssertEqual( - try JSONEncoder().encode(AnyEncodable(targetURL)), - try JSONEncoder().encode(targetURL) - ) - - XCTAssertNotEqual( - try JSONEncoder().encode(AnyErasedEncodable(value: targetURL)), - try JSONEncoder().encode(targetURL) - ) - - + let defaultValue = try JSONEncoder().encode(targetURL) + let anyValue = try JSONEncoder().encode(AnyEncodable(targetURL)) + let erasedValue = try JSONEncoder().encode(AnyErasedEncodable(value: targetURL)) + #expect(defaultValue == anyValue) + #expect(defaultValue != erasedValue) } - + + @Test func testURLObjectEncoding() throws { let targetURL = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) let objectForm = ["A": targetURL, "B": targetURL] let wrappedFrom = AnyEncodable(objectForm) - XCTAssertEqual( - try JSONSerialization.jsonObject(with: JSONEncoder().encode(objectForm)) as! NSDictionary, - try JSONSerialization.jsonObject(with: JSONEncoder().encode(wrappedFrom)) as! NSDictionary - ) - - XCTAssertEqual( - try PropertyListSerialization.propertyList(from: PropertyListEncoder().encode(objectForm), format: nil) as! NSDictionary, - try PropertyListSerialization.propertyList(from: PropertyListEncoder().encode(wrappedFrom), format: nil) as! NSDictionary - ) + var wrappedObject = try JSONSerialization.jsonObject(with: JSONEncoder().encode(wrappedFrom)) as! NSDictionary + var rawObject = try JSONSerialization.jsonObject(with: JSONEncoder().encode(objectForm)) as! NSDictionary + #expect(wrappedObject == rawObject) + rawObject = try PropertyListSerialization.propertyList(from: PropertyListEncoder().encode(objectForm), format: nil) as! NSDictionary + wrappedObject = try PropertyListSerialization.propertyList(from: PropertyListEncoder().encode(wrappedFrom), format: nil) as! NSDictionary + #expect(rawObject == wrappedObject) } + @Test func testURLArrayEncoding() throws { let targetURL = FileManager.default.temporaryDirectory let arrayForm = (0..<10).map{ _ in targetURL.appendingPathComponent(UUID().uuidString) } let wrappedFrom = AnyEncodable(arrayForm) - XCTAssertEqual( - try JSONSerialization.jsonObject(with: JSONEncoder().encode(arrayForm)) as! NSArray, - try JSONSerialization.jsonObject(with: JSONEncoder().encode(wrappedFrom)) as! NSArray - ) - - XCTAssertEqual( - try PropertyListSerialization.propertyList(from: PropertyListEncoder().encode(arrayForm), format: nil) as! NSArray, - try PropertyListSerialization.propertyList(from: PropertyListEncoder().encode(wrappedFrom), format: nil) as! NSArray - ) + var rawArray = try JSONSerialization.jsonObject(with: JSONEncoder().encode(arrayForm)) as! NSArray + var wrappedArray = try JSONSerialization.jsonObject(with: JSONEncoder().encode(wrappedFrom)) as! NSArray + #expect(rawArray == wrappedArray) + rawArray = try PropertyListSerialization.propertyList(from: PropertyListEncoder().encode(arrayForm), format: nil) as! NSArray + wrappedArray = try PropertyListSerialization.propertyList(from: PropertyListEncoder().encode(wrappedFrom), format: nil) as! NSArray + #expect(rawArray == wrappedArray) } } diff --git a/Tests/TetraTests/PlistWrapperTests.swift b/Tests/TetraTests/PlistWrapperTests.swift index 25ad5f2..1028b59 100644 --- a/Tests/TetraTests/PlistWrapperTests.swift +++ b/Tests/TetraTests/PlistWrapperTests.swift @@ -5,13 +5,15 @@ // Created by pbk on 2023/05/30. // -import XCTest +import Testing +import Foundation @testable import Tetra -final class PlistWrapperTests: XCTestCase { - +@Suite +struct PlistWrapperTests { - func testDataArraySerialization() throws { + @Test + func dataArraySerialization() throws { let data = Data(UUID().uuidString.utf8) let wrapper:PlistWrapper = [.data(data)] let actual = NSArray(object: (data as NSData)) @@ -19,13 +21,16 @@ final class PlistWrapperTests: XCTestCase { pEncoder.outputFormat = .xml let serializedXMLData = try PropertyListSerialization.data(fromPropertyList: actual, format: .xml, options: 0) let encodedXMLData = try pEncoder.encode(wrapper) - XCTAssertEqual(serializedXMLData, encodedXMLData) + #expect(serializedXMLData == encodedXMLData) +// XCTAssertEqual(serializedXMLData, encodedXMLData) pEncoder.outputFormat = .binary let serializedBinaryData = try PropertyListSerialization.data(fromPropertyList: actual, format: .binary, options: 0) let encodedBinaryData = try pEncoder.encode(wrapper) - XCTAssertEqual(serializedBinaryData, encodedBinaryData) + #expect(serializedBinaryData == encodedBinaryData) +// XCTAssertEqual(serializedBinaryData, encodedBinaryData) } + @Test func testDataDictionarySerialization() throws { let object = ["A": Data(UUID().uuidString.utf8)] let wrapper:PlistWrapper = .object(object.mapValues{ .data($0) }) @@ -34,13 +39,14 @@ final class PlistWrapperTests: XCTestCase { pEncoder.outputFormat = .xml let serializedXMLData = try PropertyListSerialization.data(fromPropertyList: actual, format: .xml, options: 0) let encodedXMLData = try pEncoder.encode(wrapper) - XCTAssertEqual(serializedXMLData, encodedXMLData) + #expect(serializedXMLData == encodedXMLData) pEncoder.outputFormat = .binary let serializedBinaryData = try PropertyListSerialization.data(fromPropertyList: actual, format: .binary, options: 0) let encodedBinaryData = try pEncoder.encode(wrapper) - XCTAssertEqual(serializedBinaryData, encodedBinaryData) + #expect(serializedBinaryData == encodedBinaryData) } + @Test func testDateArraySerialization() throws { let date = Date() let wrapper:PlistWrapper = [.date(date)] @@ -49,13 +55,14 @@ final class PlistWrapperTests: XCTestCase { pEncoder.outputFormat = .xml let serializedXMLData = try PropertyListSerialization.data(fromPropertyList: actual, format: .xml, options: 0) let encodedXMLData = try pEncoder.encode(wrapper) - XCTAssertEqual(serializedXMLData, encodedXMLData) + #expect(serializedXMLData == encodedXMLData) pEncoder.outputFormat = .binary let serializedBinaryData = try PropertyListSerialization.data(fromPropertyList: actual, format: .binary, options: 0) let encodedBinaryData = try pEncoder.encode(wrapper) - XCTAssertEqual(serializedBinaryData, encodedBinaryData) + #expect(serializedBinaryData == encodedBinaryData) } + @Test func testDateDictionarySerialization() throws { let object = [ UUID().uuidString: Date() @@ -66,35 +73,38 @@ final class PlistWrapperTests: XCTestCase { pEncoder.outputFormat = .xml let serializedXMLData = try PropertyListSerialization.data(fromPropertyList: actual, format: .xml, options: 0) let encodedXMLData = try pEncoder.encode(wrapper) - XCTAssertEqual(serializedXMLData, encodedXMLData) + #expect(serializedXMLData == encodedXMLData) pEncoder.outputFormat = .binary let serializedBinaryData = try PropertyListSerialization.data(fromPropertyList: actual, format: .binary, options: 0) let encodedBinaryData = try pEncoder.encode(wrapper) - XCTAssertEqual(serializedBinaryData, encodedBinaryData) + #expect(serializedBinaryData == encodedBinaryData) } + @Test func testCustomDecoder1() throws { try runCustomDecoder( JsonSample1Model.self, - url: XCTUnwrap( + url: #require( Bundle.module.url(forResource: "PlistSample1", withExtension: "plist") ) ) } + @Test func testCustomDecoder2() throws { try runCustomDecoder( JsonSample2Model.self, - url: XCTUnwrap( + url: #require( Bundle.module.url(forResource: "PlistSample2", withExtension: "plist") ) ) } + @Test func testCustomDecoder3() throws { try runCustomDecoder( JsonSample3Model.self, - url: XCTUnwrap( + url: #require( Bundle.module.url(forResource: "PlistSample3", withExtension: "plist") ) ) @@ -105,33 +115,36 @@ final class PlistWrapperTests: XCTestCase { let model = try PropertyListDecoder().decode(type, from: data) let jsonWrapper = try PlistWrapper(from: data) let model2 = try PlistWrapperDecoder().decode(type, from: jsonWrapper) - XCTAssertEqual(model, model2) + #expect(model == model2) } private func runCustomEncoder(_ value:T) throws { let data = try PropertyListEncoder().encode(value) let jsonWrapper = try PropertyListSerialization.propertyList(from: data, format: nil) as! NSObject let model = try PlistWrapperEncoder().encode(value).propertyObject as! NSObject - XCTAssertEqual(model, jsonWrapper) + #expect(model == jsonWrapper) } + @Test func testCustomEncoder1() throws { - let url = try XCTUnwrap(Bundle.module.url(forResource: "PlistSample1", withExtension: "plist")) + let url = try #require(Bundle.module.url(forResource: "PlistSample1", withExtension: "plist")) let model = try PropertyListDecoder().decode(JsonSample1Model.self, from: Data(contentsOf: url)) try runCustomEncoder(model) } + @Test func testCustomEncoder2() throws { - let url = try XCTUnwrap(Bundle.module.url(forResource: "PlistSample2", withExtension: "plist")) + let url = try #require(Bundle.module.url(forResource: "PlistSample2", withExtension: "plist")) let model = try PropertyListDecoder().decode(JsonSample2Model.self, from: Data(contentsOf: url)) try runCustomEncoder(model) } + @Test func testCustomEncoder3() throws { - let url = try XCTUnwrap(Bundle.module.url(forResource: "PlistSample3", withExtension: "plist")) + let url = try #require(Bundle.module.url(forResource: "PlistSample3", withExtension: "plist")) let model = try PropertyListDecoder().decode(JsonSample3Model.self, from: Data(contentsOf: url)) try runCustomEncoder(model) } - + } From f8d202d3bbb758612ee464cfe00bc2332c41fcfa Mon Sep 17 00:00:00 2001 From: pbk Date: Mon, 27 Jan 2025 21:50:31 +0900 Subject: [PATCH 57/63] implement moody::camel::ConcurrentQueue based RunLoop SerialExecutor --- Package.swift | 18 +- .../concurrentqueue.h | 3747 +++++++++++++++++ .../include/TetraConcurrentQueueShim.h | 13 + .../TetraConcurrentQueueShim/include/sim.h | 62 + Sources/TetraConcurrentQueueShim/sim.cpp | 318 ++ .../TetraRunLoopExecutor.swift | 198 + 6 files changed, 4355 insertions(+), 1 deletion(-) create mode 100644 Sources/TetraConcurrentQueueShim/concurrentqueue.h create mode 100644 Sources/TetraConcurrentQueueShim/include/TetraConcurrentQueueShim.h create mode 100644 Sources/TetraConcurrentQueueShim/include/sim.h create mode 100644 Sources/TetraConcurrentQueueShim/sim.cpp create mode 100644 Sources/TetraRunLoopConcurrency/TetraRunLoopExecutor.swift diff --git a/Package.swift b/Package.swift index bc0e9d6..e1f7e34 100644 --- a/Package.swift +++ b/Package.swift @@ -91,6 +91,21 @@ let package = Package( .swiftLanguageMode(.v6) ] ), + .target( + name: "TetraConcurrentQueueShim", + linkerSettings: [ + .linkedFramework("CoreFoundation") + ] + ), + .target( + name: "TetraRunLoopConcurrency", + dependencies: [ + "TetraConcurrentQueueShim", + ], + swiftSettings: [ + .swiftLanguageMode(.v6) + ] + ), .target( name: "BackPortAsyncSequence", dependencies: [ "Namespace"], @@ -108,5 +123,6 @@ let package = Package( .swiftLanguageMode(.v5) ] ) - ] + ], + cxxLanguageStandard: .cxx17 ) diff --git a/Sources/TetraConcurrentQueueShim/concurrentqueue.h b/Sources/TetraConcurrentQueueShim/concurrentqueue.h new file mode 100644 index 0000000..99caefc --- /dev/null +++ b/Sources/TetraConcurrentQueueShim/concurrentqueue.h @@ -0,0 +1,3747 @@ +// Provides a C++11 implementation of a multi-producer, multi-consumer lock-free queue. +// An overview, including benchmark results, is provided here: +// http://moodycamel.com/blog/2014/a-fast-general-purpose-lock-free-queue-for-c++ +// The full design is also described in excruciating detail at: +// http://moodycamel.com/blog/2014/detailed-design-of-a-lock-free-queue + +// Simplified BSD license: +// Copyright (c) 2013-2020, Cameron Desrochers. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, +// are permitted provided that the following conditions are met: +// +// - Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// - Redistributions in binary form must reproduce the above copyright notice, this list of +// conditions and the following disclaimer in the documentation and/or other materials +// provided with the distribution. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT +// OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +// HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR +// TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, +// EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// Also dual-licensed under the Boost Software License (see LICENSE.md) + +#pragma once + +#if defined(__GNUC__) && !defined(__INTEL_COMPILER) +// Disable -Wconversion warnings (spuriously triggered when Traits::size_t and +// Traits::index_t are set to < 32 bits, causing integer promotion, causing warnings +// upon assigning any computed values) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wconversion" + +#ifdef MCDBGQ_USE_RELACY +#pragma GCC diagnostic ignored "-Wint-to-pointer-cast" +#endif +#endif + +#if defined(_MSC_VER) && (!defined(_HAS_CXX17) || !_HAS_CXX17) +// VS2019 with /W4 warns about constant conditional expressions but unless /std=c++17 or higher +// does not support `if constexpr`, so we have no choice but to simply disable the warning +#pragma warning(push) +#pragma warning(disable: 4127) // conditional expression is constant +#endif + +#if defined(__APPLE__) +#include "TargetConditionals.h" +#endif + +#ifdef MCDBGQ_USE_RELACY +#include "relacy/relacy_std.hpp" +#include "relacy_shims.h" +// We only use malloc/free anyway, and the delete macro messes up `= delete` method declarations. +// We'll override the default trait malloc ourselves without a macro. +#undef new +#undef delete +#undef malloc +#undef free +#else +#include // Requires C++11. Sorry VS2010. +#include +#endif +#include // for max_align_t +#include +#include +#include +#include +#include +#include +#include // for CHAR_BIT +#include +#include // partly for __WINPTHREADS_VERSION if on MinGW-w64 w/ POSIX threading +#include // used for thread exit synchronization + +// Platform-specific definitions of a numeric thread ID type and an invalid value +namespace moodycamel { namespace details { + template struct thread_id_converter { + typedef thread_id_t thread_id_numeric_size_t; + typedef thread_id_t thread_id_hash_t; + static thread_id_hash_t prehash(thread_id_t const& x) { return x; } + }; +} } +#if defined(MCDBGQ_USE_RELACY) +namespace moodycamel { namespace details { + typedef std::uint32_t thread_id_t; + static const thread_id_t invalid_thread_id = 0xFFFFFFFFU; + static const thread_id_t invalid_thread_id2 = 0xFFFFFFFEU; + static inline thread_id_t thread_id() { return rl::thread_index(); } +} } +#elif defined(_WIN32) || defined(__WINDOWS__) || defined(__WIN32__) +// No sense pulling in windows.h in a header, we'll manually declare the function +// we use and rely on backwards-compatibility for this not to break +extern "C" __declspec(dllimport) unsigned long __stdcall GetCurrentThreadId(void); +namespace moodycamel { namespace details { + static_assert(sizeof(unsigned long) == sizeof(std::uint32_t), "Expected size of unsigned long to be 32 bits on Windows"); + typedef std::uint32_t thread_id_t; + static const thread_id_t invalid_thread_id = 0; // See http://blogs.msdn.com/b/oldnewthing/archive/2004/02/23/78395.aspx + static const thread_id_t invalid_thread_id2 = 0xFFFFFFFFU; // Not technically guaranteed to be invalid, but is never used in practice. Note that all Win32 thread IDs are presently multiples of 4. + static inline thread_id_t thread_id() { return static_cast(::GetCurrentThreadId()); } +} } +#elif defined(__arm__) || defined(_M_ARM) || defined(__aarch64__) || (defined(__APPLE__) && TARGET_OS_IPHONE) || defined(__MVS__) || defined(MOODYCAMEL_NO_THREAD_LOCAL) +namespace moodycamel { namespace details { + static_assert(sizeof(std::thread::id) == 4 || sizeof(std::thread::id) == 8, "std::thread::id is expected to be either 4 or 8 bytes"); + + typedef std::thread::id thread_id_t; + static const thread_id_t invalid_thread_id; // Default ctor creates invalid ID + + // Note we don't define a invalid_thread_id2 since std::thread::id doesn't have one; it's + // only used if MOODYCAMEL_CPP11_THREAD_LOCAL_SUPPORTED is defined anyway, which it won't + // be. + static inline thread_id_t thread_id() { return std::this_thread::get_id(); } + + template struct thread_id_size { }; + template<> struct thread_id_size<4> { typedef std::uint32_t numeric_t; }; + template<> struct thread_id_size<8> { typedef std::uint64_t numeric_t; }; + + template<> struct thread_id_converter { + typedef thread_id_size::numeric_t thread_id_numeric_size_t; +#ifndef __APPLE__ + typedef std::size_t thread_id_hash_t; +#else + typedef thread_id_numeric_size_t thread_id_hash_t; +#endif + + static thread_id_hash_t prehash(thread_id_t const& x) + { +#ifndef __APPLE__ + return std::hash()(x); +#else + return *reinterpret_cast(&x); +#endif + } + }; +} } +#else +// Use a nice trick from this answer: http://stackoverflow.com/a/8438730/21475 +// In order to get a numeric thread ID in a platform-independent way, we use a thread-local +// static variable's address as a thread identifier :-) +#if defined(__GNUC__) || defined(__INTEL_COMPILER) +#define MOODYCAMEL_THREADLOCAL __thread +#elif defined(_MSC_VER) +#define MOODYCAMEL_THREADLOCAL __declspec(thread) +#else +// Assume C++11 compliant compiler +#define MOODYCAMEL_THREADLOCAL thread_local +#endif +namespace moodycamel { namespace details { + typedef std::uintptr_t thread_id_t; + static const thread_id_t invalid_thread_id = 0; // Address can't be nullptr + static const thread_id_t invalid_thread_id2 = 1; // Member accesses off a null pointer are also generally invalid. Plus it's not aligned. + inline thread_id_t thread_id() { static MOODYCAMEL_THREADLOCAL int x; return reinterpret_cast(&x); } +} } +#endif + +// Constexpr if +#ifndef MOODYCAMEL_CONSTEXPR_IF +#if (defined(_MSC_VER) && defined(_HAS_CXX17) && _HAS_CXX17) || __cplusplus > 201402L +#define MOODYCAMEL_CONSTEXPR_IF if constexpr +#define MOODYCAMEL_MAYBE_UNUSED [[maybe_unused]] +#else +#define MOODYCAMEL_CONSTEXPR_IF if +#define MOODYCAMEL_MAYBE_UNUSED +#endif +#endif + +// Exceptions +#ifndef MOODYCAMEL_EXCEPTIONS_ENABLED +#if (defined(_MSC_VER) && defined(_CPPUNWIND)) || (defined(__GNUC__) && defined(__EXCEPTIONS)) || (!defined(_MSC_VER) && !defined(__GNUC__)) +#define MOODYCAMEL_EXCEPTIONS_ENABLED +#endif +#endif +#ifdef MOODYCAMEL_EXCEPTIONS_ENABLED +#define MOODYCAMEL_TRY try +#define MOODYCAMEL_CATCH(...) catch(__VA_ARGS__) +#define MOODYCAMEL_RETHROW throw +#define MOODYCAMEL_THROW(expr) throw (expr) +#else +#define MOODYCAMEL_TRY MOODYCAMEL_CONSTEXPR_IF (true) +#define MOODYCAMEL_CATCH(...) else MOODYCAMEL_CONSTEXPR_IF (false) +#define MOODYCAMEL_RETHROW +#define MOODYCAMEL_THROW(expr) +#endif + +#ifndef MOODYCAMEL_NOEXCEPT +#if !defined(MOODYCAMEL_EXCEPTIONS_ENABLED) +#define MOODYCAMEL_NOEXCEPT +#define MOODYCAMEL_NOEXCEPT_CTOR(type, valueType, expr) true +#define MOODYCAMEL_NOEXCEPT_ASSIGN(type, valueType, expr) true +#elif defined(_MSC_VER) && defined(_NOEXCEPT) && _MSC_VER < 1800 +// VS2012's std::is_nothrow_[move_]constructible is broken and returns true when it shouldn't :-( +// We have to assume *all* non-trivial constructors may throw on VS2012! +#define MOODYCAMEL_NOEXCEPT _NOEXCEPT +#define MOODYCAMEL_NOEXCEPT_CTOR(type, valueType, expr) (std::is_rvalue_reference::value && std::is_move_constructible::value ? std::is_trivially_move_constructible::value : std::is_trivially_copy_constructible::value) +#define MOODYCAMEL_NOEXCEPT_ASSIGN(type, valueType, expr) ((std::is_rvalue_reference::value && std::is_move_assignable::value ? std::is_trivially_move_assignable::value || std::is_nothrow_move_assignable::value : std::is_trivially_copy_assignable::value || std::is_nothrow_copy_assignable::value) && MOODYCAMEL_NOEXCEPT_CTOR(type, valueType, expr)) +#elif defined(_MSC_VER) && defined(_NOEXCEPT) && _MSC_VER < 1900 +#define MOODYCAMEL_NOEXCEPT _NOEXCEPT +#define MOODYCAMEL_NOEXCEPT_CTOR(type, valueType, expr) (std::is_rvalue_reference::value && std::is_move_constructible::value ? std::is_trivially_move_constructible::value || std::is_nothrow_move_constructible::value : std::is_trivially_copy_constructible::value || std::is_nothrow_copy_constructible::value) +#define MOODYCAMEL_NOEXCEPT_ASSIGN(type, valueType, expr) ((std::is_rvalue_reference::value && std::is_move_assignable::value ? std::is_trivially_move_assignable::value || std::is_nothrow_move_assignable::value : std::is_trivially_copy_assignable::value || std::is_nothrow_copy_assignable::value) && MOODYCAMEL_NOEXCEPT_CTOR(type, valueType, expr)) +#else +#define MOODYCAMEL_NOEXCEPT noexcept +#define MOODYCAMEL_NOEXCEPT_CTOR(type, valueType, expr) noexcept(expr) +#define MOODYCAMEL_NOEXCEPT_ASSIGN(type, valueType, expr) noexcept(expr) +#endif +#endif + +#ifndef MOODYCAMEL_CPP11_THREAD_LOCAL_SUPPORTED +#ifdef MCDBGQ_USE_RELACY +#define MOODYCAMEL_CPP11_THREAD_LOCAL_SUPPORTED +#else +// VS2013 doesn't support `thread_local`, and MinGW-w64 w/ POSIX threading has a crippling bug: http://sourceforge.net/p/mingw-w64/bugs/445 +// g++ <=4.7 doesn't support thread_local either. +// Finally, iOS/ARM doesn't have support for it either, and g++/ARM allows it to compile but it's unconfirmed to actually work +#if (!defined(_MSC_VER) || _MSC_VER >= 1900) && (!defined(__MINGW32__) && !defined(__MINGW64__) || !defined(__WINPTHREADS_VERSION)) && (!defined(__GNUC__) || __GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 8)) && (!defined(__APPLE__) || !TARGET_OS_IPHONE) && !defined(__arm__) && !defined(_M_ARM) && !defined(__aarch64__) && !defined(__MVS__) +// Assume `thread_local` is fully supported in all other C++11 compilers/platforms +#define MOODYCAMEL_CPP11_THREAD_LOCAL_SUPPORTED // tentatively enabled for now; years ago several users report having problems with it on +#endif +#endif +#endif + +// VS2012 doesn't support deleted functions. +// In this case, we declare the function normally but don't define it. A link error will be generated if the function is called. +#ifndef MOODYCAMEL_DELETE_FUNCTION +#if defined(_MSC_VER) && _MSC_VER < 1800 +#define MOODYCAMEL_DELETE_FUNCTION +#else +#define MOODYCAMEL_DELETE_FUNCTION = delete +#endif +#endif + +namespace moodycamel { namespace details { +#ifndef MOODYCAMEL_ALIGNAS +// VS2013 doesn't support alignas or alignof, and align() requires a constant literal +#if defined(_MSC_VER) && _MSC_VER <= 1800 +#define MOODYCAMEL_ALIGNAS(alignment) __declspec(align(alignment)) +#define MOODYCAMEL_ALIGNOF(obj) __alignof(obj) +#define MOODYCAMEL_ALIGNED_TYPE_LIKE(T, obj) typename details::Vs2013Aligned::value, T>::type + template struct Vs2013Aligned { }; // default, unsupported alignment + template struct Vs2013Aligned<1, T> { typedef __declspec(align(1)) T type; }; + template struct Vs2013Aligned<2, T> { typedef __declspec(align(2)) T type; }; + template struct Vs2013Aligned<4, T> { typedef __declspec(align(4)) T type; }; + template struct Vs2013Aligned<8, T> { typedef __declspec(align(8)) T type; }; + template struct Vs2013Aligned<16, T> { typedef __declspec(align(16)) T type; }; + template struct Vs2013Aligned<32, T> { typedef __declspec(align(32)) T type; }; + template struct Vs2013Aligned<64, T> { typedef __declspec(align(64)) T type; }; + template struct Vs2013Aligned<128, T> { typedef __declspec(align(128)) T type; }; + template struct Vs2013Aligned<256, T> { typedef __declspec(align(256)) T type; }; +#else + template struct identity { typedef T type; }; +#define MOODYCAMEL_ALIGNAS(alignment) alignas(alignment) +#define MOODYCAMEL_ALIGNOF(obj) alignof(obj) +#define MOODYCAMEL_ALIGNED_TYPE_LIKE(T, obj) alignas(alignof(obj)) typename details::identity::type +#endif +#endif +} } + + +// TSAN can false report races in lock-free code. To enable TSAN to be used from projects that use this one, +// we can apply per-function compile-time suppression. +// See https://clang.llvm.org/docs/ThreadSanitizer.html#has-feature-thread-sanitizer +#define MOODYCAMEL_NO_TSAN +#if defined(__has_feature) + #if __has_feature(thread_sanitizer) + #undef MOODYCAMEL_NO_TSAN + #define MOODYCAMEL_NO_TSAN __attribute__((no_sanitize("thread"))) + #endif // TSAN +#endif // TSAN + +// Compiler-specific likely/unlikely hints +namespace moodycamel { namespace details { +#if defined(__GNUC__) + static inline bool (likely)(bool x) { return __builtin_expect((x), true); } + static inline bool (unlikely)(bool x) { return __builtin_expect((x), false); } +#else + static inline bool (likely)(bool x) { return x; } + static inline bool (unlikely)(bool x) { return x; } +#endif +} } + +#ifdef MOODYCAMEL_QUEUE_INTERNAL_DEBUG +#include "internal/concurrentqueue_internal_debug.h" +#endif + +namespace moodycamel { +namespace details { + template + struct const_numeric_max { + static_assert(std::is_integral::value, "const_numeric_max can only be used with integers"); + static const T value = std::numeric_limits::is_signed + ? (static_cast(1) << (sizeof(T) * CHAR_BIT - 1)) - static_cast(1) + : static_cast(-1); + }; + +#if defined(__GLIBCXX__) + typedef ::max_align_t std_max_align_t; // libstdc++ forgot to add it to std:: for a while +#else + typedef std::max_align_t std_max_align_t; // Others (e.g. MSVC) insist it can *only* be accessed via std:: +#endif + + // Some platforms have incorrectly set max_align_t to a type with <8 bytes alignment even while supporting + // 8-byte aligned scalar values (*cough* 32-bit iOS). Work around this with our own union. See issue #64. + typedef union { + std_max_align_t x; + long long y; + void* z; + } max_align_t; +} + +// Default traits for the ConcurrentQueue. To change some of the +// traits without re-implementing all of them, inherit from this +// struct and shadow the declarations you wish to be different; +// since the traits are used as a template type parameter, the +// shadowed declarations will be used where defined, and the defaults +// otherwise. +struct ConcurrentQueueDefaultTraits +{ + // General-purpose size type. std::size_t is strongly recommended. + typedef std::size_t size_t; + + // The type used for the enqueue and dequeue indices. Must be at least as + // large as size_t. Should be significantly larger than the number of elements + // you expect to hold at once, especially if you have a high turnover rate; + // for example, on 32-bit x86, if you expect to have over a hundred million + // elements or pump several million elements through your queue in a very + // short space of time, using a 32-bit type *may* trigger a race condition. + // A 64-bit int type is recommended in that case, and in practice will + // prevent a race condition no matter the usage of the queue. Note that + // whether the queue is lock-free with a 64-int type depends on the whether + // std::atomic is lock-free, which is platform-specific. + typedef std::size_t index_t; + + // Internally, all elements are enqueued and dequeued from multi-element + // blocks; this is the smallest controllable unit. If you expect few elements + // but many producers, a smaller block size should be favoured. For few producers + // and/or many elements, a larger block size is preferred. A sane default + // is provided. Must be a power of 2. + static const size_t BLOCK_SIZE = 32; + + // For explicit producers (i.e. when using a producer token), the block is + // checked for being empty by iterating through a list of flags, one per element. + // For large block sizes, this is too inefficient, and switching to an atomic + // counter-based approach is faster. The switch is made for block sizes strictly + // larger than this threshold. + static const size_t EXPLICIT_BLOCK_EMPTY_COUNTER_THRESHOLD = 32; + + // How many full blocks can be expected for a single explicit producer? This should + // reflect that number's maximum for optimal performance. Must be a power of 2. + static const size_t EXPLICIT_INITIAL_INDEX_SIZE = 32; + + // How many full blocks can be expected for a single implicit producer? This should + // reflect that number's maximum for optimal performance. Must be a power of 2. + static const size_t IMPLICIT_INITIAL_INDEX_SIZE = 32; + + // The initial size of the hash table mapping thread IDs to implicit producers. + // Note that the hash is resized every time it becomes half full. + // Must be a power of two, and either 0 or at least 1. If 0, implicit production + // (using the enqueue methods without an explicit producer token) is disabled. + static const size_t INITIAL_IMPLICIT_PRODUCER_HASH_SIZE = 32; + + // Controls the number of items that an explicit consumer (i.e. one with a token) + // must consume before it causes all consumers to rotate and move on to the next + // internal queue. + static const std::uint32_t EXPLICIT_CONSUMER_CONSUMPTION_QUOTA_BEFORE_ROTATE = 256; + + // The maximum number of elements (inclusive) that can be enqueued to a sub-queue. + // Enqueue operations that would cause this limit to be surpassed will fail. Note + // that this limit is enforced at the block level (for performance reasons), i.e. + // it's rounded up to the nearest block size. + static const size_t MAX_SUBQUEUE_SIZE = details::const_numeric_max::value; + + // The number of times to spin before sleeping when waiting on a semaphore. + // Recommended values are on the order of 1000-10000 unless the number of + // consumer threads exceeds the number of idle cores (in which case try 0-100). + // Only affects instances of the BlockingConcurrentQueue. + static const int MAX_SEMA_SPINS = 10000; + + // Whether to recycle dynamically-allocated blocks into an internal free list or + // not. If false, only pre-allocated blocks (controlled by the constructor + // arguments) will be recycled, and all others will be `free`d back to the heap. + // Note that blocks consumed by explicit producers are only freed on destruction + // of the queue (not following destruction of the token) regardless of this trait. + static const bool RECYCLE_ALLOCATED_BLOCKS = false; + + +#ifndef MCDBGQ_USE_RELACY + // Memory allocation can be customized if needed. + // malloc should return nullptr on failure, and handle alignment like std::malloc. +#if defined(malloc) || defined(free) + // Gah, this is 2015, stop defining macros that break standard code already! + // Work around malloc/free being special macros: + static inline void* WORKAROUND_malloc(size_t size) { return malloc(size); } + static inline void WORKAROUND_free(void* ptr) { return free(ptr); } + static inline void* (malloc)(size_t size) { return WORKAROUND_malloc(size); } + static inline void (free)(void* ptr) { return WORKAROUND_free(ptr); } +#else + static inline void* malloc(size_t size) { return std::malloc(size); } + static inline void free(void* ptr) { return std::free(ptr); } +#endif +#else + // Debug versions when running under the Relacy race detector (ignore + // these in user code) + static inline void* malloc(size_t size) { return rl::rl_malloc(size, $); } + static inline void free(void* ptr) { return rl::rl_free(ptr, $); } +#endif +}; + + +// When producing or consuming many elements, the most efficient way is to: +// 1) Use one of the bulk-operation methods of the queue with a token +// 2) Failing that, use the bulk-operation methods without a token +// 3) Failing that, create a token and use that with the single-item methods +// 4) Failing that, use the single-parameter methods of the queue +// Having said that, don't create tokens willy-nilly -- ideally there should be +// a maximum of one token per thread (of each kind). +struct ProducerToken; +struct ConsumerToken; + +template class ConcurrentQueue; +template class BlockingConcurrentQueue; +class ConcurrentQueueTests; + + +namespace details +{ + struct ConcurrentQueueProducerTypelessBase + { + ConcurrentQueueProducerTypelessBase* next; + std::atomic inactive; + ProducerToken* token; + + ConcurrentQueueProducerTypelessBase() + : next(nullptr), inactive(false), token(nullptr) + { + } + }; + + template struct _hash_32_or_64 { + static inline std::uint32_t hash(std::uint32_t h) + { + // MurmurHash3 finalizer -- see https://code.google.com/p/smhasher/source/browse/trunk/MurmurHash3.cpp + // Since the thread ID is already unique, all we really want to do is propagate that + // uniqueness evenly across all the bits, so that we can use a subset of the bits while + // reducing collisions significantly + h ^= h >> 16; + h *= 0x85ebca6b; + h ^= h >> 13; + h *= 0xc2b2ae35; + return h ^ (h >> 16); + } + }; + template<> struct _hash_32_or_64<1> { + static inline std::uint64_t hash(std::uint64_t h) + { + h ^= h >> 33; + h *= 0xff51afd7ed558ccd; + h ^= h >> 33; + h *= 0xc4ceb9fe1a85ec53; + return h ^ (h >> 33); + } + }; + template struct hash_32_or_64 : public _hash_32_or_64<(size > 4)> { }; + + static inline size_t hash_thread_id(thread_id_t id) + { + static_assert(sizeof(thread_id_t) <= 8, "Expected a platform where thread IDs are at most 64-bit values"); + return static_cast(hash_32_or_64::thread_id_hash_t)>::hash( + thread_id_converter::prehash(id))); + } + + template + static inline bool circular_less_than(T a, T b) + { + static_assert(std::is_integral::value && !std::numeric_limits::is_signed, "circular_less_than is intended to be used only with unsigned integer types"); + return static_cast(a - b) > static_cast(static_cast(1) << (static_cast(sizeof(T) * CHAR_BIT - 1))); + // Note: extra parens around rhs of operator<< is MSVC bug: https://developercommunity2.visualstudio.com/t/C4554-triggers-when-both-lhs-and-rhs-is/10034931 + // silencing the bug requires #pragma warning(disable: 4554) around the calling code and has no effect when done here. + } + + template + static inline char* align_for(char* ptr) + { + const std::size_t alignment = std::alignment_of::value; + return ptr + (alignment - (reinterpret_cast(ptr) % alignment)) % alignment; + } + + template + static inline T ceil_to_pow_2(T x) + { + static_assert(std::is_integral::value && !std::numeric_limits::is_signed, "ceil_to_pow_2 is intended to be used only with unsigned integer types"); + + // Adapted from http://graphics.stanford.edu/~seander/bithacks.html#RoundUpPowerOf2 + --x; + x |= x >> 1; + x |= x >> 2; + x |= x >> 4; + for (std::size_t i = 1; i < sizeof(T); i <<= 1) { + x |= x >> (i << 3); + } + ++x; + return x; + } + + template + static inline void swap_relaxed(std::atomic& left, std::atomic& right) + { + T temp = std::move(left.load(std::memory_order_relaxed)); + left.store(std::move(right.load(std::memory_order_relaxed)), std::memory_order_relaxed); + right.store(std::move(temp), std::memory_order_relaxed); + } + + template + static inline T const& nomove(T const& x) + { + return x; + } + + template + struct nomove_if + { + template + static inline T const& eval(T const& x) + { + return x; + } + }; + + template<> + struct nomove_if + { + template + static inline auto eval(U&& x) + -> decltype(std::forward(x)) + { + return std::forward(x); + } + }; + + template + static inline auto deref_noexcept(It& it) MOODYCAMEL_NOEXCEPT -> decltype(*it) + { + return *it; + } + +#if defined(__clang__) || !defined(__GNUC__) || __GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 8) + template struct is_trivially_destructible : std::is_trivially_destructible { }; +#else + template struct is_trivially_destructible : std::has_trivial_destructor { }; +#endif + +#ifdef MOODYCAMEL_CPP11_THREAD_LOCAL_SUPPORTED +#ifdef MCDBGQ_USE_RELACY + typedef RelacyThreadExitListener ThreadExitListener; + typedef RelacyThreadExitNotifier ThreadExitNotifier; +#else + class ThreadExitNotifier; + + struct ThreadExitListener + { + typedef void (*callback_t)(void*); + callback_t callback; + void* userData; + + ThreadExitListener* next; // reserved for use by the ThreadExitNotifier + ThreadExitNotifier* chain; // reserved for use by the ThreadExitNotifier + }; + + class ThreadExitNotifier + { + public: + static void subscribe(ThreadExitListener* listener) + { + auto& tlsInst = instance(); + std::lock_guard guard(mutex()); + listener->next = tlsInst.tail; + listener->chain = &tlsInst; + tlsInst.tail = listener; + } + + static void unsubscribe(ThreadExitListener* listener) + { + std::lock_guard guard(mutex()); + if (!listener->chain) { + return; // race with ~ThreadExitNotifier + } + auto& tlsInst = *listener->chain; + listener->chain = nullptr; + ThreadExitListener** prev = &tlsInst.tail; + for (auto ptr = tlsInst.tail; ptr != nullptr; ptr = ptr->next) { + if (ptr == listener) { + *prev = ptr->next; + break; + } + prev = &ptr->next; + } + } + + private: + ThreadExitNotifier() : tail(nullptr) { } + ThreadExitNotifier(ThreadExitNotifier const&) MOODYCAMEL_DELETE_FUNCTION; + ThreadExitNotifier& operator=(ThreadExitNotifier const&) MOODYCAMEL_DELETE_FUNCTION; + + ~ThreadExitNotifier() + { + // This thread is about to exit, let everyone know! + assert(this == &instance() && "If this assert fails, you likely have a buggy compiler! Change the preprocessor conditions such that MOODYCAMEL_CPP11_THREAD_LOCAL_SUPPORTED is no longer defined."); + std::lock_guard guard(mutex()); + for (auto ptr = tail; ptr != nullptr; ptr = ptr->next) { + ptr->chain = nullptr; + ptr->callback(ptr->userData); + } + } + + // Thread-local + static inline ThreadExitNotifier& instance() + { + static thread_local ThreadExitNotifier notifier; + return notifier; + } + + static inline std::mutex& mutex() + { + // Must be static because the ThreadExitNotifier could be destroyed while unsubscribe is called + static std::mutex mutex; + return mutex; + } + + private: + ThreadExitListener* tail; + }; +#endif +#endif + + template struct static_is_lock_free_num { enum { value = 0 }; }; + template<> struct static_is_lock_free_num { enum { value = ATOMIC_CHAR_LOCK_FREE }; }; + template<> struct static_is_lock_free_num { enum { value = ATOMIC_SHORT_LOCK_FREE }; }; + template<> struct static_is_lock_free_num { enum { value = ATOMIC_INT_LOCK_FREE }; }; + template<> struct static_is_lock_free_num { enum { value = ATOMIC_LONG_LOCK_FREE }; }; + template<> struct static_is_lock_free_num { enum { value = ATOMIC_LLONG_LOCK_FREE }; }; + template struct static_is_lock_free : static_is_lock_free_num::type> { }; + template<> struct static_is_lock_free { enum { value = ATOMIC_BOOL_LOCK_FREE }; }; + template struct static_is_lock_free { enum { value = ATOMIC_POINTER_LOCK_FREE }; }; +} + + +struct ProducerToken +{ + template + explicit ProducerToken(ConcurrentQueue& queue); + + template + explicit ProducerToken(BlockingConcurrentQueue& queue); + + ProducerToken(ProducerToken&& other) MOODYCAMEL_NOEXCEPT + : producer(other.producer) + { + other.producer = nullptr; + if (producer != nullptr) { + producer->token = this; + } + } + + inline ProducerToken& operator=(ProducerToken&& other) MOODYCAMEL_NOEXCEPT + { + swap(other); + return *this; + } + + void swap(ProducerToken& other) MOODYCAMEL_NOEXCEPT + { + std::swap(producer, other.producer); + if (producer != nullptr) { + producer->token = this; + } + if (other.producer != nullptr) { + other.producer->token = &other; + } + } + + // A token is always valid unless: + // 1) Memory allocation failed during construction + // 2) It was moved via the move constructor + // (Note: assignment does a swap, leaving both potentially valid) + // 3) The associated queue was destroyed + // Note that if valid() returns true, that only indicates + // that the token is valid for use with a specific queue, + // but not which one; that's up to the user to track. + inline bool valid() const { return producer != nullptr; } + + ~ProducerToken() + { + if (producer != nullptr) { + producer->token = nullptr; + producer->inactive.store(true, std::memory_order_release); + } + } + + // Disable copying and assignment + ProducerToken(ProducerToken const&) MOODYCAMEL_DELETE_FUNCTION; + ProducerToken& operator=(ProducerToken const&) MOODYCAMEL_DELETE_FUNCTION; + +private: + template friend class ConcurrentQueue; + friend class ConcurrentQueueTests; + +protected: + details::ConcurrentQueueProducerTypelessBase* producer; +}; + + +struct ConsumerToken +{ + template + explicit ConsumerToken(ConcurrentQueue& q); + + template + explicit ConsumerToken(BlockingConcurrentQueue& q); + + ConsumerToken(ConsumerToken&& other) MOODYCAMEL_NOEXCEPT + : initialOffset(other.initialOffset), lastKnownGlobalOffset(other.lastKnownGlobalOffset), itemsConsumedFromCurrent(other.itemsConsumedFromCurrent), currentProducer(other.currentProducer), desiredProducer(other.desiredProducer) + { + } + + inline ConsumerToken& operator=(ConsumerToken&& other) MOODYCAMEL_NOEXCEPT + { + swap(other); + return *this; + } + + void swap(ConsumerToken& other) MOODYCAMEL_NOEXCEPT + { + std::swap(initialOffset, other.initialOffset); + std::swap(lastKnownGlobalOffset, other.lastKnownGlobalOffset); + std::swap(itemsConsumedFromCurrent, other.itemsConsumedFromCurrent); + std::swap(currentProducer, other.currentProducer); + std::swap(desiredProducer, other.desiredProducer); + } + + // Disable copying and assignment + ConsumerToken(ConsumerToken const&) MOODYCAMEL_DELETE_FUNCTION; + ConsumerToken& operator=(ConsumerToken const&) MOODYCAMEL_DELETE_FUNCTION; + +private: + template friend class ConcurrentQueue; + friend class ConcurrentQueueTests; + +private: // but shared with ConcurrentQueue + std::uint32_t initialOffset; + std::uint32_t lastKnownGlobalOffset; + std::uint32_t itemsConsumedFromCurrent; + details::ConcurrentQueueProducerTypelessBase* currentProducer; + details::ConcurrentQueueProducerTypelessBase* desiredProducer; +}; + +// Need to forward-declare this swap because it's in a namespace. +// See http://stackoverflow.com/questions/4492062/why-does-a-c-friend-class-need-a-forward-declaration-only-in-other-namespaces +template +inline void swap(typename ConcurrentQueue::ImplicitProducerKVP& a, typename ConcurrentQueue::ImplicitProducerKVP& b) MOODYCAMEL_NOEXCEPT; + + +template +class ConcurrentQueue +{ +public: + typedef ::moodycamel::ProducerToken producer_token_t; + typedef ::moodycamel::ConsumerToken consumer_token_t; + + typedef typename Traits::index_t index_t; + typedef typename Traits::size_t size_t; + + static const size_t BLOCK_SIZE = static_cast(Traits::BLOCK_SIZE); + static const size_t EXPLICIT_BLOCK_EMPTY_COUNTER_THRESHOLD = static_cast(Traits::EXPLICIT_BLOCK_EMPTY_COUNTER_THRESHOLD); + static const size_t EXPLICIT_INITIAL_INDEX_SIZE = static_cast(Traits::EXPLICIT_INITIAL_INDEX_SIZE); + static const size_t IMPLICIT_INITIAL_INDEX_SIZE = static_cast(Traits::IMPLICIT_INITIAL_INDEX_SIZE); + static const size_t INITIAL_IMPLICIT_PRODUCER_HASH_SIZE = static_cast(Traits::INITIAL_IMPLICIT_PRODUCER_HASH_SIZE); + static const std::uint32_t EXPLICIT_CONSUMER_CONSUMPTION_QUOTA_BEFORE_ROTATE = static_cast(Traits::EXPLICIT_CONSUMER_CONSUMPTION_QUOTA_BEFORE_ROTATE); +#ifdef _MSC_VER +#pragma warning(push) +#pragma warning(disable: 4307) // + integral constant overflow (that's what the ternary expression is for!) +#pragma warning(disable: 4309) // static_cast: Truncation of constant value +#endif + static const size_t MAX_SUBQUEUE_SIZE = (details::const_numeric_max::value - static_cast(Traits::MAX_SUBQUEUE_SIZE) < BLOCK_SIZE) ? details::const_numeric_max::value : ((static_cast(Traits::MAX_SUBQUEUE_SIZE) + (BLOCK_SIZE - 1)) / BLOCK_SIZE * BLOCK_SIZE); +#ifdef _MSC_VER +#pragma warning(pop) +#endif + + static_assert(!std::numeric_limits::is_signed && std::is_integral::value, "Traits::size_t must be an unsigned integral type"); + static_assert(!std::numeric_limits::is_signed && std::is_integral::value, "Traits::index_t must be an unsigned integral type"); + static_assert(sizeof(index_t) >= sizeof(size_t), "Traits::index_t must be at least as wide as Traits::size_t"); + static_assert((BLOCK_SIZE > 1) && !(BLOCK_SIZE & (BLOCK_SIZE - 1)), "Traits::BLOCK_SIZE must be a power of 2 (and at least 2)"); + static_assert((EXPLICIT_BLOCK_EMPTY_COUNTER_THRESHOLD > 1) && !(EXPLICIT_BLOCK_EMPTY_COUNTER_THRESHOLD & (EXPLICIT_BLOCK_EMPTY_COUNTER_THRESHOLD - 1)), "Traits::EXPLICIT_BLOCK_EMPTY_COUNTER_THRESHOLD must be a power of 2 (and greater than 1)"); + static_assert((EXPLICIT_INITIAL_INDEX_SIZE > 1) && !(EXPLICIT_INITIAL_INDEX_SIZE & (EXPLICIT_INITIAL_INDEX_SIZE - 1)), "Traits::EXPLICIT_INITIAL_INDEX_SIZE must be a power of 2 (and greater than 1)"); + static_assert((IMPLICIT_INITIAL_INDEX_SIZE > 1) && !(IMPLICIT_INITIAL_INDEX_SIZE & (IMPLICIT_INITIAL_INDEX_SIZE - 1)), "Traits::IMPLICIT_INITIAL_INDEX_SIZE must be a power of 2 (and greater than 1)"); + static_assert((INITIAL_IMPLICIT_PRODUCER_HASH_SIZE == 0) || !(INITIAL_IMPLICIT_PRODUCER_HASH_SIZE & (INITIAL_IMPLICIT_PRODUCER_HASH_SIZE - 1)), "Traits::INITIAL_IMPLICIT_PRODUCER_HASH_SIZE must be a power of 2"); + static_assert(INITIAL_IMPLICIT_PRODUCER_HASH_SIZE == 0 || INITIAL_IMPLICIT_PRODUCER_HASH_SIZE >= 1, "Traits::INITIAL_IMPLICIT_PRODUCER_HASH_SIZE must be at least 1 (or 0 to disable implicit enqueueing)"); + +public: + // Creates a queue with at least `capacity` element slots; note that the + // actual number of elements that can be inserted without additional memory + // allocation depends on the number of producers and the block size (e.g. if + // the block size is equal to `capacity`, only a single block will be allocated + // up-front, which means only a single producer will be able to enqueue elements + // without an extra allocation -- blocks aren't shared between producers). + // This method is not thread safe -- it is up to the user to ensure that the + // queue is fully constructed before it starts being used by other threads (this + // includes making the memory effects of construction visible, possibly with a + // memory barrier). + explicit ConcurrentQueue(size_t capacity = 32 * BLOCK_SIZE) + : producerListTail(nullptr), + producerCount(0), + initialBlockPoolIndex(0), + nextExplicitConsumerId(0), + globalExplicitConsumerOffset(0) + { + implicitProducerHashResizeInProgress.clear(std::memory_order_relaxed); + populate_initial_implicit_producer_hash(); + populate_initial_block_list(capacity / BLOCK_SIZE + ((capacity & (BLOCK_SIZE - 1)) == 0 ? 0 : 1)); + +#ifdef MOODYCAMEL_QUEUE_INTERNAL_DEBUG + // Track all the producers using a fully-resolved typed list for + // each kind; this makes it possible to debug them starting from + // the root queue object (otherwise wacky casts are needed that + // don't compile in the debugger's expression evaluator). + explicitProducers.store(nullptr, std::memory_order_relaxed); + implicitProducers.store(nullptr, std::memory_order_relaxed); +#endif + } + + // Computes the correct amount of pre-allocated blocks for you based + // on the minimum number of elements you want available at any given + // time, and the maximum concurrent number of each type of producer. + ConcurrentQueue(size_t minCapacity, size_t maxExplicitProducers, size_t maxImplicitProducers) + : producerListTail(nullptr), + producerCount(0), + initialBlockPoolIndex(0), + nextExplicitConsumerId(0), + globalExplicitConsumerOffset(0) + { + implicitProducerHashResizeInProgress.clear(std::memory_order_relaxed); + populate_initial_implicit_producer_hash(); + size_t blocks = (((minCapacity + BLOCK_SIZE - 1) / BLOCK_SIZE) - 1) * (maxExplicitProducers + 1) + 2 * (maxExplicitProducers + maxImplicitProducers); + populate_initial_block_list(blocks); + +#ifdef MOODYCAMEL_QUEUE_INTERNAL_DEBUG + explicitProducers.store(nullptr, std::memory_order_relaxed); + implicitProducers.store(nullptr, std::memory_order_relaxed); +#endif + } + + // Note: The queue should not be accessed concurrently while it's + // being deleted. It's up to the user to synchronize this. + // This method is not thread safe. + ~ConcurrentQueue() + { + // Destroy producers + auto ptr = producerListTail.load(std::memory_order_relaxed); + while (ptr != nullptr) { + auto next = ptr->next_prod(); + if (ptr->token != nullptr) { + ptr->token->producer = nullptr; + } + destroy(ptr); + ptr = next; + } + + // Destroy implicit producer hash tables + MOODYCAMEL_CONSTEXPR_IF (INITIAL_IMPLICIT_PRODUCER_HASH_SIZE != 0) { + auto hash = implicitProducerHash.load(std::memory_order_relaxed); + while (hash != nullptr) { + auto prev = hash->prev; + if (prev != nullptr) { // The last hash is part of this object and was not allocated dynamically + for (size_t i = 0; i != hash->capacity; ++i) { + hash->entries[i].~ImplicitProducerKVP(); + } + hash->~ImplicitProducerHash(); + (Traits::free)(hash); + } + hash = prev; + } + } + + // Destroy global free list + auto block = freeList.head_unsafe(); + while (block != nullptr) { + auto next = block->freeListNext.load(std::memory_order_relaxed); + if (block->dynamicallyAllocated) { + destroy(block); + } + block = next; + } + + // Destroy initial free list + destroy_array(initialBlockPool, initialBlockPoolSize); + } + + // Disable copying and copy assignment + ConcurrentQueue(ConcurrentQueue const&) MOODYCAMEL_DELETE_FUNCTION; + ConcurrentQueue& operator=(ConcurrentQueue const&) MOODYCAMEL_DELETE_FUNCTION; + + // Moving is supported, but note that it is *not* a thread-safe operation. + // Nobody can use the queue while it's being moved, and the memory effects + // of that move must be propagated to other threads before they can use it. + // Note: When a queue is moved, its tokens are still valid but can only be + // used with the destination queue (i.e. semantically they are moved along + // with the queue itself). + ConcurrentQueue(ConcurrentQueue&& other) MOODYCAMEL_NOEXCEPT + : producerListTail(other.producerListTail.load(std::memory_order_relaxed)), + producerCount(other.producerCount.load(std::memory_order_relaxed)), + initialBlockPoolIndex(other.initialBlockPoolIndex.load(std::memory_order_relaxed)), + initialBlockPool(other.initialBlockPool), + initialBlockPoolSize(other.initialBlockPoolSize), + freeList(std::move(other.freeList)), + nextExplicitConsumerId(other.nextExplicitConsumerId.load(std::memory_order_relaxed)), + globalExplicitConsumerOffset(other.globalExplicitConsumerOffset.load(std::memory_order_relaxed)) + { + // Move the other one into this, and leave the other one as an empty queue + implicitProducerHashResizeInProgress.clear(std::memory_order_relaxed); + populate_initial_implicit_producer_hash(); + swap_implicit_producer_hashes(other); + + other.producerListTail.store(nullptr, std::memory_order_relaxed); + other.producerCount.store(0, std::memory_order_relaxed); + other.nextExplicitConsumerId.store(0, std::memory_order_relaxed); + other.globalExplicitConsumerOffset.store(0, std::memory_order_relaxed); + +#ifdef MOODYCAMEL_QUEUE_INTERNAL_DEBUG + explicitProducers.store(other.explicitProducers.load(std::memory_order_relaxed), std::memory_order_relaxed); + other.explicitProducers.store(nullptr, std::memory_order_relaxed); + implicitProducers.store(other.implicitProducers.load(std::memory_order_relaxed), std::memory_order_relaxed); + other.implicitProducers.store(nullptr, std::memory_order_relaxed); +#endif + + other.initialBlockPoolIndex.store(0, std::memory_order_relaxed); + other.initialBlockPoolSize = 0; + other.initialBlockPool = nullptr; + + reown_producers(); + } + + inline ConcurrentQueue& operator=(ConcurrentQueue&& other) MOODYCAMEL_NOEXCEPT + { + return swap_internal(other); + } + + // Swaps this queue's state with the other's. Not thread-safe. + // Swapping two queues does not invalidate their tokens, however + // the tokens that were created for one queue must be used with + // only the swapped queue (i.e. the tokens are tied to the + // queue's movable state, not the object itself). + inline void swap(ConcurrentQueue& other) MOODYCAMEL_NOEXCEPT + { + swap_internal(other); + } + +private: + ConcurrentQueue& swap_internal(ConcurrentQueue& other) + { + if (this == &other) { + return *this; + } + + details::swap_relaxed(producerListTail, other.producerListTail); + details::swap_relaxed(producerCount, other.producerCount); + details::swap_relaxed(initialBlockPoolIndex, other.initialBlockPoolIndex); + std::swap(initialBlockPool, other.initialBlockPool); + std::swap(initialBlockPoolSize, other.initialBlockPoolSize); + freeList.swap(other.freeList); + details::swap_relaxed(nextExplicitConsumerId, other.nextExplicitConsumerId); + details::swap_relaxed(globalExplicitConsumerOffset, other.globalExplicitConsumerOffset); + + swap_implicit_producer_hashes(other); + + reown_producers(); + other.reown_producers(); + +#ifdef MOODYCAMEL_QUEUE_INTERNAL_DEBUG + details::swap_relaxed(explicitProducers, other.explicitProducers); + details::swap_relaxed(implicitProducers, other.implicitProducers); +#endif + + return *this; + } + +public: + // Enqueues a single item (by copying it). + // Allocates memory if required. Only fails if memory allocation fails (or implicit + // production is disabled because Traits::INITIAL_IMPLICIT_PRODUCER_HASH_SIZE is 0, + // or Traits::MAX_SUBQUEUE_SIZE has been defined and would be surpassed). + // Thread-safe. + inline bool enqueue(T const& item) + { + MOODYCAMEL_CONSTEXPR_IF (INITIAL_IMPLICIT_PRODUCER_HASH_SIZE == 0) return false; + else return inner_enqueue(item); + } + + // Enqueues a single item (by moving it, if possible). + // Allocates memory if required. Only fails if memory allocation fails (or implicit + // production is disabled because Traits::INITIAL_IMPLICIT_PRODUCER_HASH_SIZE is 0, + // or Traits::MAX_SUBQUEUE_SIZE has been defined and would be surpassed). + // Thread-safe. + inline bool enqueue(T&& item) + { + MOODYCAMEL_CONSTEXPR_IF (INITIAL_IMPLICIT_PRODUCER_HASH_SIZE == 0) return false; + else return inner_enqueue(std::move(item)); + } + + // Enqueues a single item (by copying it) using an explicit producer token. + // Allocates memory if required. Only fails if memory allocation fails (or + // Traits::MAX_SUBQUEUE_SIZE has been defined and would be surpassed). + // Thread-safe. + inline bool enqueue(producer_token_t const& token, T const& item) + { + return inner_enqueue(token, item); + } + + // Enqueues a single item (by moving it, if possible) using an explicit producer token. + // Allocates memory if required. Only fails if memory allocation fails (or + // Traits::MAX_SUBQUEUE_SIZE has been defined and would be surpassed). + // Thread-safe. + inline bool enqueue(producer_token_t const& token, T&& item) + { + return inner_enqueue(token, std::move(item)); + } + + // Enqueues several items. + // Allocates memory if required. Only fails if memory allocation fails (or + // implicit production is disabled because Traits::INITIAL_IMPLICIT_PRODUCER_HASH_SIZE + // is 0, or Traits::MAX_SUBQUEUE_SIZE has been defined and would be surpassed). + // Note: Use std::make_move_iterator if the elements should be moved instead of copied. + // Thread-safe. + template + bool enqueue_bulk(It itemFirst, size_t count) + { + MOODYCAMEL_CONSTEXPR_IF (INITIAL_IMPLICIT_PRODUCER_HASH_SIZE == 0) return false; + else return inner_enqueue_bulk(itemFirst, count); + } + + // Enqueues several items using an explicit producer token. + // Allocates memory if required. Only fails if memory allocation fails + // (or Traits::MAX_SUBQUEUE_SIZE has been defined and would be surpassed). + // Note: Use std::make_move_iterator if the elements should be moved + // instead of copied. + // Thread-safe. + template + bool enqueue_bulk(producer_token_t const& token, It itemFirst, size_t count) + { + return inner_enqueue_bulk(token, itemFirst, count); + } + + // Enqueues a single item (by copying it). + // Does not allocate memory. Fails if not enough room to enqueue (or implicit + // production is disabled because Traits::INITIAL_IMPLICIT_PRODUCER_HASH_SIZE + // is 0). + // Thread-safe. + inline bool try_enqueue(T const& item) + { + MOODYCAMEL_CONSTEXPR_IF (INITIAL_IMPLICIT_PRODUCER_HASH_SIZE == 0) return false; + else return inner_enqueue(item); + } + + // Enqueues a single item (by moving it, if possible). + // Does not allocate memory (except for one-time implicit producer). + // Fails if not enough room to enqueue (or implicit production is + // disabled because Traits::INITIAL_IMPLICIT_PRODUCER_HASH_SIZE is 0). + // Thread-safe. + inline bool try_enqueue(T&& item) + { + MOODYCAMEL_CONSTEXPR_IF (INITIAL_IMPLICIT_PRODUCER_HASH_SIZE == 0) return false; + else return inner_enqueue(std::move(item)); + } + + // Enqueues a single item (by copying it) using an explicit producer token. + // Does not allocate memory. Fails if not enough room to enqueue. + // Thread-safe. + inline bool try_enqueue(producer_token_t const& token, T const& item) + { + return inner_enqueue(token, item); + } + + // Enqueues a single item (by moving it, if possible) using an explicit producer token. + // Does not allocate memory. Fails if not enough room to enqueue. + // Thread-safe. + inline bool try_enqueue(producer_token_t const& token, T&& item) + { + return inner_enqueue(token, std::move(item)); + } + + // Enqueues several items. + // Does not allocate memory (except for one-time implicit producer). + // Fails if not enough room to enqueue (or implicit production is + // disabled because Traits::INITIAL_IMPLICIT_PRODUCER_HASH_SIZE is 0). + // Note: Use std::make_move_iterator if the elements should be moved + // instead of copied. + // Thread-safe. + template + bool try_enqueue_bulk(It itemFirst, size_t count) + { + MOODYCAMEL_CONSTEXPR_IF (INITIAL_IMPLICIT_PRODUCER_HASH_SIZE == 0) return false; + else return inner_enqueue_bulk(itemFirst, count); + } + + // Enqueues several items using an explicit producer token. + // Does not allocate memory. Fails if not enough room to enqueue. + // Note: Use std::make_move_iterator if the elements should be moved + // instead of copied. + // Thread-safe. + template + bool try_enqueue_bulk(producer_token_t const& token, It itemFirst, size_t count) + { + return inner_enqueue_bulk(token, itemFirst, count); + } + + + + // Attempts to dequeue from the queue. + // Returns false if all producer streams appeared empty at the time they + // were checked (so, the queue is likely but not guaranteed to be empty). + // Never allocates. Thread-safe. + template + bool try_dequeue(U& item) + { + // Instead of simply trying each producer in turn (which could cause needless contention on the first + // producer), we score them heuristically. + size_t nonEmptyCount = 0; + ProducerBase* best = nullptr; + size_t bestSize = 0; + for (auto ptr = producerListTail.load(std::memory_order_acquire); nonEmptyCount < 3 && ptr != nullptr; ptr = ptr->next_prod()) { + auto size = ptr->size_approx(); + if (size > 0) { + if (size > bestSize) { + bestSize = size; + best = ptr; + } + ++nonEmptyCount; + } + } + + // If there was at least one non-empty queue but it appears empty at the time + // we try to dequeue from it, we need to make sure every queue's been tried + if (nonEmptyCount > 0) { + if ((details::likely)(best->dequeue(item))) { + return true; + } + for (auto ptr = producerListTail.load(std::memory_order_acquire); ptr != nullptr; ptr = ptr->next_prod()) { + if (ptr != best && ptr->dequeue(item)) { + return true; + } + } + } + return false; + } + + // Attempts to dequeue from the queue. + // Returns false if all producer streams appeared empty at the time they + // were checked (so, the queue is likely but not guaranteed to be empty). + // This differs from the try_dequeue(item) method in that this one does + // not attempt to reduce contention by interleaving the order that producer + // streams are dequeued from. So, using this method can reduce overall throughput + // under contention, but will give more predictable results in single-threaded + // consumer scenarios. This is mostly only useful for internal unit tests. + // Never allocates. Thread-safe. + template + bool try_dequeue_non_interleaved(U& item) + { + for (auto ptr = producerListTail.load(std::memory_order_acquire); ptr != nullptr; ptr = ptr->next_prod()) { + if (ptr->dequeue(item)) { + return true; + } + } + return false; + } + + // Attempts to dequeue from the queue using an explicit consumer token. + // Returns false if all producer streams appeared empty at the time they + // were checked (so, the queue is likely but not guaranteed to be empty). + // Never allocates. Thread-safe. + template + bool try_dequeue(consumer_token_t& token, U& item) + { + // The idea is roughly as follows: + // Every 256 items from one producer, make everyone rotate (increase the global offset) -> this means the highest efficiency consumer dictates the rotation speed of everyone else, more or less + // If you see that the global offset has changed, you must reset your consumption counter and move to your designated place + // If there's no items where you're supposed to be, keep moving until you find a producer with some items + // If the global offset has not changed but you've run out of items to consume, move over from your current position until you find an producer with something in it + + if (token.desiredProducer == nullptr || token.lastKnownGlobalOffset != globalExplicitConsumerOffset.load(std::memory_order_relaxed)) { + if (!update_current_producer_after_rotation(token)) { + return false; + } + } + + // If there was at least one non-empty queue but it appears empty at the time + // we try to dequeue from it, we need to make sure every queue's been tried + if (static_cast(token.currentProducer)->dequeue(item)) { + if (++token.itemsConsumedFromCurrent == EXPLICIT_CONSUMER_CONSUMPTION_QUOTA_BEFORE_ROTATE) { + globalExplicitConsumerOffset.fetch_add(1, std::memory_order_relaxed); + } + return true; + } + + auto tail = producerListTail.load(std::memory_order_acquire); + auto ptr = static_cast(token.currentProducer)->next_prod(); + if (ptr == nullptr) { + ptr = tail; + } + while (ptr != static_cast(token.currentProducer)) { + if (ptr->dequeue(item)) { + token.currentProducer = ptr; + token.itemsConsumedFromCurrent = 1; + return true; + } + ptr = ptr->next_prod(); + if (ptr == nullptr) { + ptr = tail; + } + } + return false; + } + + // Attempts to dequeue several elements from the queue. + // Returns the number of items actually dequeued. + // Returns 0 if all producer streams appeared empty at the time they + // were checked (so, the queue is likely but not guaranteed to be empty). + // Never allocates. Thread-safe. + template + size_t try_dequeue_bulk(It itemFirst, size_t max) + { + size_t count = 0; + for (auto ptr = producerListTail.load(std::memory_order_acquire); ptr != nullptr; ptr = ptr->next_prod()) { + count += ptr->dequeue_bulk(itemFirst, max - count); + if (count == max) { + break; + } + } + return count; + } + + // Attempts to dequeue several elements from the queue using an explicit consumer token. + // Returns the number of items actually dequeued. + // Returns 0 if all producer streams appeared empty at the time they + // were checked (so, the queue is likely but not guaranteed to be empty). + // Never allocates. Thread-safe. + template + size_t try_dequeue_bulk(consumer_token_t& token, It itemFirst, size_t max) + { + if (token.desiredProducer == nullptr || token.lastKnownGlobalOffset != globalExplicitConsumerOffset.load(std::memory_order_relaxed)) { + if (!update_current_producer_after_rotation(token)) { + return 0; + } + } + + size_t count = static_cast(token.currentProducer)->dequeue_bulk(itemFirst, max); + if (count == max) { + if ((token.itemsConsumedFromCurrent += static_cast(max)) >= EXPLICIT_CONSUMER_CONSUMPTION_QUOTA_BEFORE_ROTATE) { + globalExplicitConsumerOffset.fetch_add(1, std::memory_order_relaxed); + } + return max; + } + token.itemsConsumedFromCurrent += static_cast(count); + max -= count; + + auto tail = producerListTail.load(std::memory_order_acquire); + auto ptr = static_cast(token.currentProducer)->next_prod(); + if (ptr == nullptr) { + ptr = tail; + } + while (ptr != static_cast(token.currentProducer)) { + auto dequeued = ptr->dequeue_bulk(itemFirst, max); + count += dequeued; + if (dequeued != 0) { + token.currentProducer = ptr; + token.itemsConsumedFromCurrent = static_cast(dequeued); + } + if (dequeued == max) { + break; + } + max -= dequeued; + ptr = ptr->next_prod(); + if (ptr == nullptr) { + ptr = tail; + } + } + return count; + } + + + + // Attempts to dequeue from a specific producer's inner queue. + // If you happen to know which producer you want to dequeue from, this + // is significantly faster than using the general-case try_dequeue methods. + // Returns false if the producer's queue appeared empty at the time it + // was checked (so, the queue is likely but not guaranteed to be empty). + // Never allocates. Thread-safe. + template + inline bool try_dequeue_from_producer(producer_token_t const& producer, U& item) + { + return static_cast(producer.producer)->dequeue(item); + } + + // Attempts to dequeue several elements from a specific producer's inner queue. + // Returns the number of items actually dequeued. + // If you happen to know which producer you want to dequeue from, this + // is significantly faster than using the general-case try_dequeue methods. + // Returns 0 if the producer's queue appeared empty at the time it + // was checked (so, the queue is likely but not guaranteed to be empty). + // Never allocates. Thread-safe. + template + inline size_t try_dequeue_bulk_from_producer(producer_token_t const& producer, It itemFirst, size_t max) + { + return static_cast(producer.producer)->dequeue_bulk(itemFirst, max); + } + + + // Returns an estimate of the total number of elements currently in the queue. This + // estimate is only accurate if the queue has completely stabilized before it is called + // (i.e. all enqueue and dequeue operations have completed and their memory effects are + // visible on the calling thread, and no further operations start while this method is + // being called). + // Thread-safe. + size_t size_approx() const + { + size_t size = 0; + for (auto ptr = producerListTail.load(std::memory_order_acquire); ptr != nullptr; ptr = ptr->next_prod()) { + size += ptr->size_approx(); + } + return size; + } + + + // Returns true if the underlying atomic variables used by + // the queue are lock-free (they should be on most platforms). + // Thread-safe. + static constexpr bool is_lock_free() + { + return + details::static_is_lock_free::value == 2 && + details::static_is_lock_free::value == 2 && + details::static_is_lock_free::value == 2 && + details::static_is_lock_free::value == 2 && + details::static_is_lock_free::value == 2 && + details::static_is_lock_free::thread_id_numeric_size_t>::value == 2; + } + + +private: + friend struct ProducerToken; + friend struct ConsumerToken; + struct ExplicitProducer; + friend struct ExplicitProducer; + struct ImplicitProducer; + friend struct ImplicitProducer; + friend class ConcurrentQueueTests; + + enum AllocationMode { CanAlloc, CannotAlloc }; + + + /////////////////////////////// + // Queue methods + /////////////////////////////// + + template + inline bool inner_enqueue(producer_token_t const& token, U&& element) + { + return static_cast(token.producer)->ConcurrentQueue::ExplicitProducer::template enqueue(std::forward(element)); + } + + template + inline bool inner_enqueue(U&& element) + { + auto producer = get_or_add_implicit_producer(); + return producer == nullptr ? false : producer->ConcurrentQueue::ImplicitProducer::template enqueue(std::forward(element)); + } + + template + inline bool inner_enqueue_bulk(producer_token_t const& token, It itemFirst, size_t count) + { + return static_cast(token.producer)->ConcurrentQueue::ExplicitProducer::template enqueue_bulk(itemFirst, count); + } + + template + inline bool inner_enqueue_bulk(It itemFirst, size_t count) + { + auto producer = get_or_add_implicit_producer(); + return producer == nullptr ? false : producer->ConcurrentQueue::ImplicitProducer::template enqueue_bulk(itemFirst, count); + } + + inline bool update_current_producer_after_rotation(consumer_token_t& token) + { + // Ah, there's been a rotation, figure out where we should be! + auto tail = producerListTail.load(std::memory_order_acquire); + if (token.desiredProducer == nullptr && tail == nullptr) { + return false; + } + auto prodCount = producerCount.load(std::memory_order_relaxed); + auto globalOffset = globalExplicitConsumerOffset.load(std::memory_order_relaxed); + if ((details::unlikely)(token.desiredProducer == nullptr)) { + // Aha, first time we're dequeueing anything. + // Figure out our local position + // Note: offset is from start, not end, but we're traversing from end -- subtract from count first + std::uint32_t offset = prodCount - 1 - (token.initialOffset % prodCount); + token.desiredProducer = tail; + for (std::uint32_t i = 0; i != offset; ++i) { + token.desiredProducer = static_cast(token.desiredProducer)->next_prod(); + if (token.desiredProducer == nullptr) { + token.desiredProducer = tail; + } + } + } + + std::uint32_t delta = globalOffset - token.lastKnownGlobalOffset; + if (delta >= prodCount) { + delta = delta % prodCount; + } + for (std::uint32_t i = 0; i != delta; ++i) { + token.desiredProducer = static_cast(token.desiredProducer)->next_prod(); + if (token.desiredProducer == nullptr) { + token.desiredProducer = tail; + } + } + + token.lastKnownGlobalOffset = globalOffset; + token.currentProducer = token.desiredProducer; + token.itemsConsumedFromCurrent = 0; + return true; + } + + + /////////////////////////// + // Free list + /////////////////////////// + + template + struct FreeListNode + { + FreeListNode() : freeListRefs(0), freeListNext(nullptr) { } + + std::atomic freeListRefs; + std::atomic freeListNext; + }; + + // A simple CAS-based lock-free free list. Not the fastest thing in the world under heavy contention, but + // simple and correct (assuming nodes are never freed until after the free list is destroyed), and fairly + // speedy under low contention. + template // N must inherit FreeListNode or have the same fields (and initialization of them) + struct FreeList + { + FreeList() : freeListHead(nullptr) { } + FreeList(FreeList&& other) : freeListHead(other.freeListHead.load(std::memory_order_relaxed)) { other.freeListHead.store(nullptr, std::memory_order_relaxed); } + void swap(FreeList& other) { details::swap_relaxed(freeListHead, other.freeListHead); } + + FreeList(FreeList const&) MOODYCAMEL_DELETE_FUNCTION; + FreeList& operator=(FreeList const&) MOODYCAMEL_DELETE_FUNCTION; + + inline void add(N* node) + { +#ifdef MCDBGQ_NOLOCKFREE_FREELIST + debug::DebugLock lock(mutex); +#endif + // We know that the should-be-on-freelist bit is 0 at this point, so it's safe to + // set it using a fetch_add + if (node->freeListRefs.fetch_add(SHOULD_BE_ON_FREELIST, std::memory_order_acq_rel) == 0) { + // Oh look! We were the last ones referencing this node, and we know + // we want to add it to the free list, so let's do it! + add_knowing_refcount_is_zero(node); + } + } + + inline N* try_get() + { +#ifdef MCDBGQ_NOLOCKFREE_FREELIST + debug::DebugLock lock(mutex); +#endif + auto head = freeListHead.load(std::memory_order_acquire); + while (head != nullptr) { + auto prevHead = head; + auto refs = head->freeListRefs.load(std::memory_order_relaxed); + if ((refs & REFS_MASK) == 0 || !head->freeListRefs.compare_exchange_strong(refs, refs + 1, std::memory_order_acquire, std::memory_order_relaxed)) { + head = freeListHead.load(std::memory_order_acquire); + continue; + } + + // Good, reference count has been incremented (it wasn't at zero), which means we can read the + // next and not worry about it changing between now and the time we do the CAS + auto next = head->freeListNext.load(std::memory_order_relaxed); + if (freeListHead.compare_exchange_strong(head, next, std::memory_order_acquire, std::memory_order_relaxed)) { + // Yay, got the node. This means it was on the list, which means shouldBeOnFreeList must be false no + // matter the refcount (because nobody else knows it's been taken off yet, it can't have been put back on). + assert((head->freeListRefs.load(std::memory_order_relaxed) & SHOULD_BE_ON_FREELIST) == 0); + + // Decrease refcount twice, once for our ref, and once for the list's ref + head->freeListRefs.fetch_sub(2, std::memory_order_release); + return head; + } + + // OK, the head must have changed on us, but we still need to decrease the refcount we increased. + // Note that we don't need to release any memory effects, but we do need to ensure that the reference + // count decrement happens-after the CAS on the head. + refs = prevHead->freeListRefs.fetch_sub(1, std::memory_order_acq_rel); + if (refs == SHOULD_BE_ON_FREELIST + 1) { + add_knowing_refcount_is_zero(prevHead); + } + } + + return nullptr; + } + + // Useful for traversing the list when there's no contention (e.g. to destroy remaining nodes) + N* head_unsafe() const { return freeListHead.load(std::memory_order_relaxed); } + + private: + inline void add_knowing_refcount_is_zero(N* node) + { + // Since the refcount is zero, and nobody can increase it once it's zero (except us, and we run + // only one copy of this method per node at a time, i.e. the single thread case), then we know + // we can safely change the next pointer of the node; however, once the refcount is back above + // zero, then other threads could increase it (happens under heavy contention, when the refcount + // goes to zero in between a load and a refcount increment of a node in try_get, then back up to + // something non-zero, then the refcount increment is done by the other thread) -- so, if the CAS + // to add the node to the actual list fails, decrease the refcount and leave the add operation to + // the next thread who puts the refcount back at zero (which could be us, hence the loop). + auto head = freeListHead.load(std::memory_order_relaxed); + while (true) { + node->freeListNext.store(head, std::memory_order_relaxed); + node->freeListRefs.store(1, std::memory_order_release); + if (!freeListHead.compare_exchange_strong(head, node, std::memory_order_release, std::memory_order_relaxed)) { + // Hmm, the add failed, but we can only try again when the refcount goes back to zero + if (node->freeListRefs.fetch_add(SHOULD_BE_ON_FREELIST - 1, std::memory_order_release) == 1) { + continue; + } + } + return; + } + } + + private: + // Implemented like a stack, but where node order doesn't matter (nodes are inserted out of order under contention) + std::atomic freeListHead; + + static const std::uint32_t REFS_MASK = 0x7FFFFFFF; + static const std::uint32_t SHOULD_BE_ON_FREELIST = 0x80000000; + +#ifdef MCDBGQ_NOLOCKFREE_FREELIST + debug::DebugMutex mutex; +#endif + }; + + + /////////////////////////// + // Block + /////////////////////////// + + enum InnerQueueContext { implicit_context = 0, explicit_context = 1 }; + + struct Block + { + Block() + : next(nullptr), elementsCompletelyDequeued(0), freeListRefs(0), freeListNext(nullptr), dynamicallyAllocated(true) + { +#ifdef MCDBGQ_TRACKMEM + owner = nullptr; +#endif + } + + template + inline bool is_empty() const + { + MOODYCAMEL_CONSTEXPR_IF (context == explicit_context && BLOCK_SIZE <= EXPLICIT_BLOCK_EMPTY_COUNTER_THRESHOLD) { + // Check flags + for (size_t i = 0; i < BLOCK_SIZE; ++i) { + if (!emptyFlags[i].load(std::memory_order_relaxed)) { + return false; + } + } + + // Aha, empty; make sure we have all other memory effects that happened before the empty flags were set + std::atomic_thread_fence(std::memory_order_acquire); + return true; + } + else { + // Check counter + if (elementsCompletelyDequeued.load(std::memory_order_relaxed) == BLOCK_SIZE) { + std::atomic_thread_fence(std::memory_order_acquire); + return true; + } + assert(elementsCompletelyDequeued.load(std::memory_order_relaxed) <= BLOCK_SIZE); + return false; + } + } + + // Returns true if the block is now empty (does not apply in explicit context) + template + inline bool set_empty(MOODYCAMEL_MAYBE_UNUSED index_t i) + { + MOODYCAMEL_CONSTEXPR_IF (context == explicit_context && BLOCK_SIZE <= EXPLICIT_BLOCK_EMPTY_COUNTER_THRESHOLD) { + // Set flag + assert(!emptyFlags[BLOCK_SIZE - 1 - static_cast(i & static_cast(BLOCK_SIZE - 1))].load(std::memory_order_relaxed)); + emptyFlags[BLOCK_SIZE - 1 - static_cast(i & static_cast(BLOCK_SIZE - 1))].store(true, std::memory_order_release); + return false; + } + else { + // Increment counter + auto prevVal = elementsCompletelyDequeued.fetch_add(1, std::memory_order_release); + assert(prevVal < BLOCK_SIZE); + return prevVal == BLOCK_SIZE - 1; + } + } + + // Sets multiple contiguous item statuses to 'empty' (assumes no wrapping and count > 0). + // Returns true if the block is now empty (does not apply in explicit context). + template + inline bool set_many_empty(MOODYCAMEL_MAYBE_UNUSED index_t i, size_t count) + { + MOODYCAMEL_CONSTEXPR_IF (context == explicit_context && BLOCK_SIZE <= EXPLICIT_BLOCK_EMPTY_COUNTER_THRESHOLD) { + // Set flags + std::atomic_thread_fence(std::memory_order_release); + i = BLOCK_SIZE - 1 - static_cast(i & static_cast(BLOCK_SIZE - 1)) - count + 1; + for (size_t j = 0; j != count; ++j) { + assert(!emptyFlags[i + j].load(std::memory_order_relaxed)); + emptyFlags[i + j].store(true, std::memory_order_relaxed); + } + return false; + } + else { + // Increment counter + auto prevVal = elementsCompletelyDequeued.fetch_add(count, std::memory_order_release); + assert(prevVal + count <= BLOCK_SIZE); + return prevVal + count == BLOCK_SIZE; + } + } + + template + inline void set_all_empty() + { + MOODYCAMEL_CONSTEXPR_IF (context == explicit_context && BLOCK_SIZE <= EXPLICIT_BLOCK_EMPTY_COUNTER_THRESHOLD) { + // Set all flags + for (size_t i = 0; i != BLOCK_SIZE; ++i) { + emptyFlags[i].store(true, std::memory_order_relaxed); + } + } + else { + // Reset counter + elementsCompletelyDequeued.store(BLOCK_SIZE, std::memory_order_relaxed); + } + } + + template + inline void reset_empty() + { + MOODYCAMEL_CONSTEXPR_IF (context == explicit_context && BLOCK_SIZE <= EXPLICIT_BLOCK_EMPTY_COUNTER_THRESHOLD) { + // Reset flags + for (size_t i = 0; i != BLOCK_SIZE; ++i) { + emptyFlags[i].store(false, std::memory_order_relaxed); + } + } + else { + // Reset counter + elementsCompletelyDequeued.store(0, std::memory_order_relaxed); + } + } + + inline T* operator[](index_t idx) MOODYCAMEL_NOEXCEPT { return static_cast(static_cast(elements)) + static_cast(idx & static_cast(BLOCK_SIZE - 1)); } + inline T const* operator[](index_t idx) const MOODYCAMEL_NOEXCEPT { return static_cast(static_cast(elements)) + static_cast(idx & static_cast(BLOCK_SIZE - 1)); } + + private: + static_assert(std::alignment_of::value <= sizeof(T), "The queue does not support types with an alignment greater than their size at this time"); + MOODYCAMEL_ALIGNED_TYPE_LIKE(char[sizeof(T) * BLOCK_SIZE], T) elements; + public: + Block* next; + std::atomic elementsCompletelyDequeued; + std::atomic emptyFlags[BLOCK_SIZE <= EXPLICIT_BLOCK_EMPTY_COUNTER_THRESHOLD ? BLOCK_SIZE : 1]; + public: + std::atomic freeListRefs; + std::atomic freeListNext; + bool dynamicallyAllocated; // Perhaps a better name for this would be 'isNotPartOfInitialBlockPool' + +#ifdef MCDBGQ_TRACKMEM + void* owner; +#endif + }; + static_assert(std::alignment_of::value >= std::alignment_of::value, "Internal error: Blocks must be at least as aligned as the type they are wrapping"); + + +#ifdef MCDBGQ_TRACKMEM +public: + struct MemStats; +private: +#endif + + /////////////////////////// + // Producer base + /////////////////////////// + + struct ProducerBase : public details::ConcurrentQueueProducerTypelessBase + { + ProducerBase(ConcurrentQueue* parent_, bool isExplicit_) : + tailIndex(0), + headIndex(0), + dequeueOptimisticCount(0), + dequeueOvercommit(0), + tailBlock(nullptr), + isExplicit(isExplicit_), + parent(parent_) + { + } + + virtual ~ProducerBase() { } + + template + inline bool dequeue(U& element) + { + if (isExplicit) { + return static_cast(this)->dequeue(element); + } + else { + return static_cast(this)->dequeue(element); + } + } + + template + inline size_t dequeue_bulk(It& itemFirst, size_t max) + { + if (isExplicit) { + return static_cast(this)->dequeue_bulk(itemFirst, max); + } + else { + return static_cast(this)->dequeue_bulk(itemFirst, max); + } + } + + inline ProducerBase* next_prod() const { return static_cast(next); } + + inline size_t size_approx() const + { + auto tail = tailIndex.load(std::memory_order_relaxed); + auto head = headIndex.load(std::memory_order_relaxed); + return details::circular_less_than(head, tail) ? static_cast(tail - head) : 0; + } + + inline index_t getTail() const { return tailIndex.load(std::memory_order_relaxed); } + protected: + std::atomic tailIndex; // Where to enqueue to next + std::atomic headIndex; // Where to dequeue from next + + std::atomic dequeueOptimisticCount; + std::atomic dequeueOvercommit; + + Block* tailBlock; + + public: + bool isExplicit; + ConcurrentQueue* parent; + + protected: +#ifdef MCDBGQ_TRACKMEM + friend struct MemStats; +#endif + }; + + + /////////////////////////// + // Explicit queue + /////////////////////////// + + struct ExplicitProducer : public ProducerBase + { + explicit ExplicitProducer(ConcurrentQueue* parent_) : + ProducerBase(parent_, true), + blockIndex(nullptr), + pr_blockIndexSlotsUsed(0), + pr_blockIndexSize(EXPLICIT_INITIAL_INDEX_SIZE >> 1), + pr_blockIndexFront(0), + pr_blockIndexEntries(nullptr), + pr_blockIndexRaw(nullptr) + { + size_t poolBasedIndexSize = details::ceil_to_pow_2(parent_->initialBlockPoolSize) >> 1; + if (poolBasedIndexSize > pr_blockIndexSize) { + pr_blockIndexSize = poolBasedIndexSize; + } + + new_block_index(0); // This creates an index with double the number of current entries, i.e. EXPLICIT_INITIAL_INDEX_SIZE + } + + ~ExplicitProducer() + { + // Destruct any elements not yet dequeued. + // Since we're in the destructor, we can assume all elements + // are either completely dequeued or completely not (no halfways). + if (this->tailBlock != nullptr) { // Note this means there must be a block index too + // First find the block that's partially dequeued, if any + Block* halfDequeuedBlock = nullptr; + if ((this->headIndex.load(std::memory_order_relaxed) & static_cast(BLOCK_SIZE - 1)) != 0) { + // The head's not on a block boundary, meaning a block somewhere is partially dequeued + // (or the head block is the tail block and was fully dequeued, but the head/tail are still not on a boundary) + size_t i = (pr_blockIndexFront - pr_blockIndexSlotsUsed) & (pr_blockIndexSize - 1); + while (details::circular_less_than(pr_blockIndexEntries[i].base + BLOCK_SIZE, this->headIndex.load(std::memory_order_relaxed))) { + i = (i + 1) & (pr_blockIndexSize - 1); + } + assert(details::circular_less_than(pr_blockIndexEntries[i].base, this->headIndex.load(std::memory_order_relaxed))); + halfDequeuedBlock = pr_blockIndexEntries[i].block; + } + + // Start at the head block (note the first line in the loop gives us the head from the tail on the first iteration) + auto block = this->tailBlock; + do { + block = block->next; + if (block->ConcurrentQueue::Block::template is_empty()) { + continue; + } + + size_t i = 0; // Offset into block + if (block == halfDequeuedBlock) { + i = static_cast(this->headIndex.load(std::memory_order_relaxed) & static_cast(BLOCK_SIZE - 1)); + } + + // Walk through all the items in the block; if this is the tail block, we need to stop when we reach the tail index + auto lastValidIndex = (this->tailIndex.load(std::memory_order_relaxed) & static_cast(BLOCK_SIZE - 1)) == 0 ? BLOCK_SIZE : static_cast(this->tailIndex.load(std::memory_order_relaxed) & static_cast(BLOCK_SIZE - 1)); + while (i != BLOCK_SIZE && (block != this->tailBlock || i != lastValidIndex)) { + (*block)[i++]->~T(); + } + } while (block != this->tailBlock); + } + + // Destroy all blocks that we own + if (this->tailBlock != nullptr) { + auto block = this->tailBlock; + do { + auto nextBlock = block->next; + this->parent->add_block_to_free_list(block); + block = nextBlock; + } while (block != this->tailBlock); + } + + // Destroy the block indices + auto header = static_cast(pr_blockIndexRaw); + while (header != nullptr) { + auto prev = static_cast(header->prev); + header->~BlockIndexHeader(); + (Traits::free)(header); + header = prev; + } + } + + template + inline bool enqueue(U&& element) + { + index_t currentTailIndex = this->tailIndex.load(std::memory_order_relaxed); + index_t newTailIndex = 1 + currentTailIndex; + if ((currentTailIndex & static_cast(BLOCK_SIZE - 1)) == 0) { + // We reached the end of a block, start a new one + auto startBlock = this->tailBlock; + auto originalBlockIndexSlotsUsed = pr_blockIndexSlotsUsed; + if (this->tailBlock != nullptr && this->tailBlock->next->ConcurrentQueue::Block::template is_empty()) { + // We can re-use the block ahead of us, it's empty! + this->tailBlock = this->tailBlock->next; + this->tailBlock->ConcurrentQueue::Block::template reset_empty(); + + // We'll put the block on the block index (guaranteed to be room since we're conceptually removing the + // last block from it first -- except instead of removing then adding, we can just overwrite). + // Note that there must be a valid block index here, since even if allocation failed in the ctor, + // it would have been re-attempted when adding the first block to the queue; since there is such + // a block, a block index must have been successfully allocated. + } + else { + // Whatever head value we see here is >= the last value we saw here (relatively), + // and <= its current value. Since we have the most recent tail, the head must be + // <= to it. + auto head = this->headIndex.load(std::memory_order_relaxed); + assert(!details::circular_less_than(currentTailIndex, head)); + if (!details::circular_less_than(head, currentTailIndex + BLOCK_SIZE) + || (MAX_SUBQUEUE_SIZE != details::const_numeric_max::value && (MAX_SUBQUEUE_SIZE == 0 || MAX_SUBQUEUE_SIZE - BLOCK_SIZE < currentTailIndex - head))) { + // We can't enqueue in another block because there's not enough leeway -- the + // tail could surpass the head by the time the block fills up! (Or we'll exceed + // the size limit, if the second part of the condition was true.) + return false; + } + // We're going to need a new block; check that the block index has room + if (pr_blockIndexRaw == nullptr || pr_blockIndexSlotsUsed == pr_blockIndexSize) { + // Hmm, the circular block index is already full -- we'll need + // to allocate a new index. Note pr_blockIndexRaw can only be nullptr if + // the initial allocation failed in the constructor. + + MOODYCAMEL_CONSTEXPR_IF (allocMode == CannotAlloc) { + return false; + } + else if (!new_block_index(pr_blockIndexSlotsUsed)) { + return false; + } + } + + // Insert a new block in the circular linked list + auto newBlock = this->parent->ConcurrentQueue::template requisition_block(); + if (newBlock == nullptr) { + return false; + } +#ifdef MCDBGQ_TRACKMEM + newBlock->owner = this; +#endif + newBlock->ConcurrentQueue::Block::template reset_empty(); + if (this->tailBlock == nullptr) { + newBlock->next = newBlock; + } + else { + newBlock->next = this->tailBlock->next; + this->tailBlock->next = newBlock; + } + this->tailBlock = newBlock; + ++pr_blockIndexSlotsUsed; + } + + MOODYCAMEL_CONSTEXPR_IF (!MOODYCAMEL_NOEXCEPT_CTOR(T, U, new (static_cast(nullptr)) T(std::forward(element)))) { + // The constructor may throw. We want the element not to appear in the queue in + // that case (without corrupting the queue): + MOODYCAMEL_TRY { + new ((*this->tailBlock)[currentTailIndex]) T(std::forward(element)); + } + MOODYCAMEL_CATCH (...) { + // Revert change to the current block, but leave the new block available + // for next time + pr_blockIndexSlotsUsed = originalBlockIndexSlotsUsed; + this->tailBlock = startBlock == nullptr ? this->tailBlock : startBlock; + MOODYCAMEL_RETHROW; + } + } + else { + (void)startBlock; + (void)originalBlockIndexSlotsUsed; + } + + // Add block to block index + auto& entry = blockIndex.load(std::memory_order_relaxed)->entries[pr_blockIndexFront]; + entry.base = currentTailIndex; + entry.block = this->tailBlock; + blockIndex.load(std::memory_order_relaxed)->front.store(pr_blockIndexFront, std::memory_order_release); + pr_blockIndexFront = (pr_blockIndexFront + 1) & (pr_blockIndexSize - 1); + + MOODYCAMEL_CONSTEXPR_IF (!MOODYCAMEL_NOEXCEPT_CTOR(T, U, new (static_cast(nullptr)) T(std::forward(element)))) { + this->tailIndex.store(newTailIndex, std::memory_order_release); + return true; + } + } + + // Enqueue + new ((*this->tailBlock)[currentTailIndex]) T(std::forward(element)); + + this->tailIndex.store(newTailIndex, std::memory_order_release); + return true; + } + + template + bool dequeue(U& element) + { + auto tail = this->tailIndex.load(std::memory_order_relaxed); + auto overcommit = this->dequeueOvercommit.load(std::memory_order_relaxed); + if (details::circular_less_than(this->dequeueOptimisticCount.load(std::memory_order_relaxed) - overcommit, tail)) { + // Might be something to dequeue, let's give it a try + + // Note that this if is purely for performance purposes in the common case when the queue is + // empty and the values are eventually consistent -- we may enter here spuriously. + + // Note that whatever the values of overcommit and tail are, they are not going to change (unless we + // change them) and must be the same value at this point (inside the if) as when the if condition was + // evaluated. + + // We insert an acquire fence here to synchronize-with the release upon incrementing dequeueOvercommit below. + // This ensures that whatever the value we got loaded into overcommit, the load of dequeueOptisticCount in + // the fetch_add below will result in a value at least as recent as that (and therefore at least as large). + // Note that I believe a compiler (signal) fence here would be sufficient due to the nature of fetch_add (all + // read-modify-write operations are guaranteed to work on the latest value in the modification order), but + // unfortunately that can't be shown to be correct using only the C++11 standard. + // See http://stackoverflow.com/questions/18223161/what-are-the-c11-memory-ordering-guarantees-in-this-corner-case + std::atomic_thread_fence(std::memory_order_acquire); + + // Increment optimistic counter, then check if it went over the boundary + auto myDequeueCount = this->dequeueOptimisticCount.fetch_add(1, std::memory_order_relaxed); + + // Note that since dequeueOvercommit must be <= dequeueOptimisticCount (because dequeueOvercommit is only ever + // incremented after dequeueOptimisticCount -- this is enforced in the `else` block below), and since we now + // have a version of dequeueOptimisticCount that is at least as recent as overcommit (due to the release upon + // incrementing dequeueOvercommit and the acquire above that synchronizes with it), overcommit <= myDequeueCount. + // However, we can't assert this since both dequeueOptimisticCount and dequeueOvercommit may (independently) + // overflow; in such a case, though, the logic still holds since the difference between the two is maintained. + + // Note that we reload tail here in case it changed; it will be the same value as before or greater, since + // this load is sequenced after (happens after) the earlier load above. This is supported by read-read + // coherency (as defined in the standard), explained here: http://en.cppreference.com/w/cpp/atomic/memory_order + tail = this->tailIndex.load(std::memory_order_acquire); + if ((details::likely)(details::circular_less_than(myDequeueCount - overcommit, tail))) { + // Guaranteed to be at least one element to dequeue! + + // Get the index. Note that since there's guaranteed to be at least one element, this + // will never exceed tail. We need to do an acquire-release fence here since it's possible + // that whatever condition got us to this point was for an earlier enqueued element (that + // we already see the memory effects for), but that by the time we increment somebody else + // has incremented it, and we need to see the memory effects for *that* element, which is + // in such a case is necessarily visible on the thread that incremented it in the first + // place with the more current condition (they must have acquired a tail that is at least + // as recent). + auto index = this->headIndex.fetch_add(1, std::memory_order_acq_rel); + + + // Determine which block the element is in + + auto localBlockIndex = blockIndex.load(std::memory_order_acquire); + auto localBlockIndexHead = localBlockIndex->front.load(std::memory_order_acquire); + + // We need to be careful here about subtracting and dividing because of index wrap-around. + // When an index wraps, we need to preserve the sign of the offset when dividing it by the + // block size (in order to get a correct signed block count offset in all cases): + auto headBase = localBlockIndex->entries[localBlockIndexHead].base; + auto blockBaseIndex = index & ~static_cast(BLOCK_SIZE - 1); + auto offset = static_cast(static_cast::type>(blockBaseIndex - headBase) / static_cast::type>(BLOCK_SIZE)); + auto block = localBlockIndex->entries[(localBlockIndexHead + offset) & (localBlockIndex->size - 1)].block; + + // Dequeue + auto& el = *((*block)[index]); + if (!MOODYCAMEL_NOEXCEPT_ASSIGN(T, T&&, element = std::move(el))) { + // Make sure the element is still fully dequeued and destroyed even if the assignment + // throws + struct Guard { + Block* block; + index_t index; + + ~Guard() + { + (*block)[index]->~T(); + block->ConcurrentQueue::Block::template set_empty(index); + } + } guard = { block, index }; + + element = std::move(el); // NOLINT + } + else { + element = std::move(el); // NOLINT + el.~T(); // NOLINT + block->ConcurrentQueue::Block::template set_empty(index); + } + + return true; + } + else { + // Wasn't anything to dequeue after all; make the effective dequeue count eventually consistent + this->dequeueOvercommit.fetch_add(1, std::memory_order_release); // Release so that the fetch_add on dequeueOptimisticCount is guaranteed to happen before this write + } + } + + return false; + } + + template + bool MOODYCAMEL_NO_TSAN enqueue_bulk(It itemFirst, size_t count) + { + // First, we need to make sure we have enough room to enqueue all of the elements; + // this means pre-allocating blocks and putting them in the block index (but only if + // all the allocations succeeded). + index_t startTailIndex = this->tailIndex.load(std::memory_order_relaxed); + auto startBlock = this->tailBlock; + auto originalBlockIndexFront = pr_blockIndexFront; + auto originalBlockIndexSlotsUsed = pr_blockIndexSlotsUsed; + + Block* firstAllocatedBlock = nullptr; + + // Figure out how many blocks we'll need to allocate, and do so + size_t blockBaseDiff = ((startTailIndex + count - 1) & ~static_cast(BLOCK_SIZE - 1)) - ((startTailIndex - 1) & ~static_cast(BLOCK_SIZE - 1)); + index_t currentTailIndex = (startTailIndex - 1) & ~static_cast(BLOCK_SIZE - 1); + if (blockBaseDiff > 0) { + // Allocate as many blocks as possible from ahead + while (blockBaseDiff > 0 && this->tailBlock != nullptr && this->tailBlock->next != firstAllocatedBlock && this->tailBlock->next->ConcurrentQueue::Block::template is_empty()) { + blockBaseDiff -= static_cast(BLOCK_SIZE); + currentTailIndex += static_cast(BLOCK_SIZE); + + this->tailBlock = this->tailBlock->next; + firstAllocatedBlock = firstAllocatedBlock == nullptr ? this->tailBlock : firstAllocatedBlock; + + auto& entry = blockIndex.load(std::memory_order_relaxed)->entries[pr_blockIndexFront]; + entry.base = currentTailIndex; + entry.block = this->tailBlock; + pr_blockIndexFront = (pr_blockIndexFront + 1) & (pr_blockIndexSize - 1); + } + + // Now allocate as many blocks as necessary from the block pool + while (blockBaseDiff > 0) { + blockBaseDiff -= static_cast(BLOCK_SIZE); + currentTailIndex += static_cast(BLOCK_SIZE); + + auto head = this->headIndex.load(std::memory_order_relaxed); + assert(!details::circular_less_than(currentTailIndex, head)); + bool full = !details::circular_less_than(head, currentTailIndex + BLOCK_SIZE) || (MAX_SUBQUEUE_SIZE != details::const_numeric_max::value && (MAX_SUBQUEUE_SIZE == 0 || MAX_SUBQUEUE_SIZE - BLOCK_SIZE < currentTailIndex - head)); + if (pr_blockIndexRaw == nullptr || pr_blockIndexSlotsUsed == pr_blockIndexSize || full) { + MOODYCAMEL_CONSTEXPR_IF (allocMode == CannotAlloc) { + // Failed to allocate, undo changes (but keep injected blocks) + pr_blockIndexFront = originalBlockIndexFront; + pr_blockIndexSlotsUsed = originalBlockIndexSlotsUsed; + this->tailBlock = startBlock == nullptr ? firstAllocatedBlock : startBlock; + return false; + } + else if (full || !new_block_index(originalBlockIndexSlotsUsed)) { + // Failed to allocate, undo changes (but keep injected blocks) + pr_blockIndexFront = originalBlockIndexFront; + pr_blockIndexSlotsUsed = originalBlockIndexSlotsUsed; + this->tailBlock = startBlock == nullptr ? firstAllocatedBlock : startBlock; + return false; + } + + // pr_blockIndexFront is updated inside new_block_index, so we need to + // update our fallback value too (since we keep the new index even if we + // later fail) + originalBlockIndexFront = originalBlockIndexSlotsUsed; + } + + // Insert a new block in the circular linked list + auto newBlock = this->parent->ConcurrentQueue::template requisition_block(); + if (newBlock == nullptr) { + pr_blockIndexFront = originalBlockIndexFront; + pr_blockIndexSlotsUsed = originalBlockIndexSlotsUsed; + this->tailBlock = startBlock == nullptr ? firstAllocatedBlock : startBlock; + return false; + } + +#ifdef MCDBGQ_TRACKMEM + newBlock->owner = this; +#endif + newBlock->ConcurrentQueue::Block::template set_all_empty(); + if (this->tailBlock == nullptr) { + newBlock->next = newBlock; + } + else { + newBlock->next = this->tailBlock->next; + this->tailBlock->next = newBlock; + } + this->tailBlock = newBlock; + firstAllocatedBlock = firstAllocatedBlock == nullptr ? this->tailBlock : firstAllocatedBlock; + + ++pr_blockIndexSlotsUsed; + + auto& entry = blockIndex.load(std::memory_order_relaxed)->entries[pr_blockIndexFront]; + entry.base = currentTailIndex; + entry.block = this->tailBlock; + pr_blockIndexFront = (pr_blockIndexFront + 1) & (pr_blockIndexSize - 1); + } + + // Excellent, all allocations succeeded. Reset each block's emptiness before we fill them up, and + // publish the new block index front + auto block = firstAllocatedBlock; + while (true) { + block->ConcurrentQueue::Block::template reset_empty(); + if (block == this->tailBlock) { + break; + } + block = block->next; + } + + MOODYCAMEL_CONSTEXPR_IF (MOODYCAMEL_NOEXCEPT_CTOR(T, decltype(*itemFirst), new (static_cast(nullptr)) T(details::deref_noexcept(itemFirst)))) { + blockIndex.load(std::memory_order_relaxed)->front.store((pr_blockIndexFront - 1) & (pr_blockIndexSize - 1), std::memory_order_release); + } + } + + // Enqueue, one block at a time + index_t newTailIndex = startTailIndex + static_cast(count); + currentTailIndex = startTailIndex; + auto endBlock = this->tailBlock; + this->tailBlock = startBlock; + assert((startTailIndex & static_cast(BLOCK_SIZE - 1)) != 0 || firstAllocatedBlock != nullptr || count == 0); + if ((startTailIndex & static_cast(BLOCK_SIZE - 1)) == 0 && firstAllocatedBlock != nullptr) { + this->tailBlock = firstAllocatedBlock; + } + while (true) { + index_t stopIndex = (currentTailIndex & ~static_cast(BLOCK_SIZE - 1)) + static_cast(BLOCK_SIZE); + if (details::circular_less_than(newTailIndex, stopIndex)) { + stopIndex = newTailIndex; + } + MOODYCAMEL_CONSTEXPR_IF (MOODYCAMEL_NOEXCEPT_CTOR(T, decltype(*itemFirst), new (static_cast(nullptr)) T(details::deref_noexcept(itemFirst)))) { + while (currentTailIndex != stopIndex) { + new ((*this->tailBlock)[currentTailIndex++]) T(*itemFirst++); + } + } + else { + MOODYCAMEL_TRY { + while (currentTailIndex != stopIndex) { + // Must use copy constructor even if move constructor is available + // because we may have to revert if there's an exception. + // Sorry about the horrible templated next line, but it was the only way + // to disable moving *at compile time*, which is important because a type + // may only define a (noexcept) move constructor, and so calls to the + // cctor will not compile, even if they are in an if branch that will never + // be executed + new ((*this->tailBlock)[currentTailIndex]) T(details::nomove_if(nullptr)) T(details::deref_noexcept(itemFirst)))>::eval(*itemFirst)); + ++currentTailIndex; + ++itemFirst; + } + } + MOODYCAMEL_CATCH (...) { + // Oh dear, an exception's been thrown -- destroy the elements that + // were enqueued so far and revert the entire bulk operation (we'll keep + // any allocated blocks in our linked list for later, though). + auto constructedStopIndex = currentTailIndex; + auto lastBlockEnqueued = this->tailBlock; + + pr_blockIndexFront = originalBlockIndexFront; + pr_blockIndexSlotsUsed = originalBlockIndexSlotsUsed; + this->tailBlock = startBlock == nullptr ? firstAllocatedBlock : startBlock; + + if (!details::is_trivially_destructible::value) { + auto block = startBlock; + if ((startTailIndex & static_cast(BLOCK_SIZE - 1)) == 0) { + block = firstAllocatedBlock; + } + currentTailIndex = startTailIndex; + while (true) { + stopIndex = (currentTailIndex & ~static_cast(BLOCK_SIZE - 1)) + static_cast(BLOCK_SIZE); + if (details::circular_less_than(constructedStopIndex, stopIndex)) { + stopIndex = constructedStopIndex; + } + while (currentTailIndex != stopIndex) { + (*block)[currentTailIndex++]->~T(); + } + if (block == lastBlockEnqueued) { + break; + } + block = block->next; + } + } + MOODYCAMEL_RETHROW; + } + } + + if (this->tailBlock == endBlock) { + assert(currentTailIndex == newTailIndex); + break; + } + this->tailBlock = this->tailBlock->next; + } + + MOODYCAMEL_CONSTEXPR_IF (!MOODYCAMEL_NOEXCEPT_CTOR(T, decltype(*itemFirst), new (static_cast(nullptr)) T(details::deref_noexcept(itemFirst)))) { + if (firstAllocatedBlock != nullptr) + blockIndex.load(std::memory_order_relaxed)->front.store((pr_blockIndexFront - 1) & (pr_blockIndexSize - 1), std::memory_order_release); + } + + this->tailIndex.store(newTailIndex, std::memory_order_release); + return true; + } + + template + size_t dequeue_bulk(It& itemFirst, size_t max) + { + auto tail = this->tailIndex.load(std::memory_order_relaxed); + auto overcommit = this->dequeueOvercommit.load(std::memory_order_relaxed); + auto desiredCount = static_cast(tail - (this->dequeueOptimisticCount.load(std::memory_order_relaxed) - overcommit)); + if (details::circular_less_than(0, desiredCount)) { + desiredCount = desiredCount < max ? desiredCount : max; + std::atomic_thread_fence(std::memory_order_acquire); + + auto myDequeueCount = this->dequeueOptimisticCount.fetch_add(desiredCount, std::memory_order_relaxed); + + tail = this->tailIndex.load(std::memory_order_acquire); + auto actualCount = static_cast(tail - (myDequeueCount - overcommit)); + if (details::circular_less_than(0, actualCount)) { + actualCount = desiredCount < actualCount ? desiredCount : actualCount; + if (actualCount < desiredCount) { + this->dequeueOvercommit.fetch_add(desiredCount - actualCount, std::memory_order_release); + } + + // Get the first index. Note that since there's guaranteed to be at least actualCount elements, this + // will never exceed tail. + auto firstIndex = this->headIndex.fetch_add(actualCount, std::memory_order_acq_rel); + + // Determine which block the first element is in + auto localBlockIndex = blockIndex.load(std::memory_order_acquire); + auto localBlockIndexHead = localBlockIndex->front.load(std::memory_order_acquire); + + auto headBase = localBlockIndex->entries[localBlockIndexHead].base; + auto firstBlockBaseIndex = firstIndex & ~static_cast(BLOCK_SIZE - 1); + auto offset = static_cast(static_cast::type>(firstBlockBaseIndex - headBase) / static_cast::type>(BLOCK_SIZE)); + auto indexIndex = (localBlockIndexHead + offset) & (localBlockIndex->size - 1); + + // Iterate the blocks and dequeue + auto index = firstIndex; + do { + auto firstIndexInBlock = index; + index_t endIndex = (index & ~static_cast(BLOCK_SIZE - 1)) + static_cast(BLOCK_SIZE); + endIndex = details::circular_less_than(firstIndex + static_cast(actualCount), endIndex) ? firstIndex + static_cast(actualCount) : endIndex; + auto block = localBlockIndex->entries[indexIndex].block; + if (MOODYCAMEL_NOEXCEPT_ASSIGN(T, T&&, details::deref_noexcept(itemFirst) = std::move((*(*block)[index])))) { + while (index != endIndex) { + auto& el = *((*block)[index]); + *itemFirst++ = std::move(el); + el.~T(); + ++index; + } + } + else { + MOODYCAMEL_TRY { + while (index != endIndex) { + auto& el = *((*block)[index]); + *itemFirst = std::move(el); + ++itemFirst; + el.~T(); + ++index; + } + } + MOODYCAMEL_CATCH (...) { + // It's too late to revert the dequeue, but we can make sure that all + // the dequeued objects are properly destroyed and the block index + // (and empty count) are properly updated before we propagate the exception + do { + block = localBlockIndex->entries[indexIndex].block; + while (index != endIndex) { + (*block)[index++]->~T(); + } + block->ConcurrentQueue::Block::template set_many_empty(firstIndexInBlock, static_cast(endIndex - firstIndexInBlock)); + indexIndex = (indexIndex + 1) & (localBlockIndex->size - 1); + + firstIndexInBlock = index; + endIndex = (index & ~static_cast(BLOCK_SIZE - 1)) + static_cast(BLOCK_SIZE); + endIndex = details::circular_less_than(firstIndex + static_cast(actualCount), endIndex) ? firstIndex + static_cast(actualCount) : endIndex; + } while (index != firstIndex + actualCount); + + MOODYCAMEL_RETHROW; + } + } + block->ConcurrentQueue::Block::template set_many_empty(firstIndexInBlock, static_cast(endIndex - firstIndexInBlock)); + indexIndex = (indexIndex + 1) & (localBlockIndex->size - 1); + } while (index != firstIndex + actualCount); + + return actualCount; + } + else { + // Wasn't anything to dequeue after all; make the effective dequeue count eventually consistent + this->dequeueOvercommit.fetch_add(desiredCount, std::memory_order_release); + } + } + + return 0; + } + + private: + struct BlockIndexEntry + { + index_t base; + Block* block; + }; + + struct BlockIndexHeader + { + size_t size; + std::atomic front; // Current slot (not next, like pr_blockIndexFront) + BlockIndexEntry* entries; + void* prev; + }; + + + bool new_block_index(size_t numberOfFilledSlotsToExpose) + { + auto prevBlockSizeMask = pr_blockIndexSize - 1; + + // Create the new block + pr_blockIndexSize <<= 1; + auto newRawPtr = static_cast((Traits::malloc)(sizeof(BlockIndexHeader) + std::alignment_of::value - 1 + sizeof(BlockIndexEntry) * pr_blockIndexSize)); + if (newRawPtr == nullptr) { + pr_blockIndexSize >>= 1; // Reset to allow graceful retry + return false; + } + + auto newBlockIndexEntries = reinterpret_cast(details::align_for(newRawPtr + sizeof(BlockIndexHeader))); + + // Copy in all the old indices, if any + size_t j = 0; + if (pr_blockIndexSlotsUsed != 0) { + auto i = (pr_blockIndexFront - pr_blockIndexSlotsUsed) & prevBlockSizeMask; + do { + newBlockIndexEntries[j++] = pr_blockIndexEntries[i]; + i = (i + 1) & prevBlockSizeMask; + } while (i != pr_blockIndexFront); + } + + // Update everything + auto header = new (newRawPtr) BlockIndexHeader; + header->size = pr_blockIndexSize; + header->front.store(numberOfFilledSlotsToExpose - 1, std::memory_order_relaxed); + header->entries = newBlockIndexEntries; + header->prev = pr_blockIndexRaw; // we link the new block to the old one so we can free it later + + pr_blockIndexFront = j; + pr_blockIndexEntries = newBlockIndexEntries; + pr_blockIndexRaw = newRawPtr; + blockIndex.store(header, std::memory_order_release); + + return true; + } + + private: + std::atomic blockIndex; + + // To be used by producer only -- consumer must use the ones in referenced by blockIndex + size_t pr_blockIndexSlotsUsed; + size_t pr_blockIndexSize; + size_t pr_blockIndexFront; // Next slot (not current) + BlockIndexEntry* pr_blockIndexEntries; + void* pr_blockIndexRaw; + +#ifdef MOODYCAMEL_QUEUE_INTERNAL_DEBUG + public: + ExplicitProducer* nextExplicitProducer; + private: +#endif + +#ifdef MCDBGQ_TRACKMEM + friend struct MemStats; +#endif + }; + + + ////////////////////////////////// + // Implicit queue + ////////////////////////////////// + + struct ImplicitProducer : public ProducerBase + { + ImplicitProducer(ConcurrentQueue* parent_) : + ProducerBase(parent_, false), + nextBlockIndexCapacity(IMPLICIT_INITIAL_INDEX_SIZE), + blockIndex(nullptr) + { + new_block_index(); + } + + ~ImplicitProducer() + { + // Note that since we're in the destructor we can assume that all enqueue/dequeue operations + // completed already; this means that all undequeued elements are placed contiguously across + // contiguous blocks, and that only the first and last remaining blocks can be only partially + // empty (all other remaining blocks must be completely full). + +#ifdef MOODYCAMEL_CPP11_THREAD_LOCAL_SUPPORTED + // Unregister ourselves for thread termination notification + if (!this->inactive.load(std::memory_order_relaxed)) { + details::ThreadExitNotifier::unsubscribe(&threadExitListener); + } +#endif + + // Destroy all remaining elements! + auto tail = this->tailIndex.load(std::memory_order_relaxed); + auto index = this->headIndex.load(std::memory_order_relaxed); + Block* block = nullptr; + assert(index == tail || details::circular_less_than(index, tail)); + bool forceFreeLastBlock = index != tail; // If we enter the loop, then the last (tail) block will not be freed + while (index != tail) { + if ((index & static_cast(BLOCK_SIZE - 1)) == 0 || block == nullptr) { + if (block != nullptr) { + // Free the old block + this->parent->add_block_to_free_list(block); + } + + block = get_block_index_entry_for_index(index)->value.load(std::memory_order_relaxed); + } + + ((*block)[index])->~T(); + ++index; + } + // Even if the queue is empty, there's still one block that's not on the free list + // (unless the head index reached the end of it, in which case the tail will be poised + // to create a new block). + if (this->tailBlock != nullptr && (forceFreeLastBlock || (tail & static_cast(BLOCK_SIZE - 1)) != 0)) { + this->parent->add_block_to_free_list(this->tailBlock); + } + + // Destroy block index + auto localBlockIndex = blockIndex.load(std::memory_order_relaxed); + if (localBlockIndex != nullptr) { + for (size_t i = 0; i != localBlockIndex->capacity; ++i) { + localBlockIndex->index[i]->~BlockIndexEntry(); + } + do { + auto prev = localBlockIndex->prev; + localBlockIndex->~BlockIndexHeader(); + (Traits::free)(localBlockIndex); + localBlockIndex = prev; + } while (localBlockIndex != nullptr); + } + } + + template + inline bool enqueue(U&& element) + { + index_t currentTailIndex = this->tailIndex.load(std::memory_order_relaxed); + index_t newTailIndex = 1 + currentTailIndex; + if ((currentTailIndex & static_cast(BLOCK_SIZE - 1)) == 0) { + // We reached the end of a block, start a new one + auto head = this->headIndex.load(std::memory_order_relaxed); + assert(!details::circular_less_than(currentTailIndex, head)); + if (!details::circular_less_than(head, currentTailIndex + BLOCK_SIZE) || (MAX_SUBQUEUE_SIZE != details::const_numeric_max::value && (MAX_SUBQUEUE_SIZE == 0 || MAX_SUBQUEUE_SIZE - BLOCK_SIZE < currentTailIndex - head))) { + return false; + } +#ifdef MCDBGQ_NOLOCKFREE_IMPLICITPRODBLOCKINDEX + debug::DebugLock lock(mutex); +#endif + // Find out where we'll be inserting this block in the block index + BlockIndexEntry* idxEntry; + if (!insert_block_index_entry(idxEntry, currentTailIndex)) { + return false; + } + + // Get ahold of a new block + auto newBlock = this->parent->ConcurrentQueue::template requisition_block(); + if (newBlock == nullptr) { + rewind_block_index_tail(); + idxEntry->value.store(nullptr, std::memory_order_relaxed); + return false; + } +#ifdef MCDBGQ_TRACKMEM + newBlock->owner = this; +#endif + newBlock->ConcurrentQueue::Block::template reset_empty(); + + MOODYCAMEL_CONSTEXPR_IF (!MOODYCAMEL_NOEXCEPT_CTOR(T, U, new (static_cast(nullptr)) T(std::forward(element)))) { + // May throw, try to insert now before we publish the fact that we have this new block + MOODYCAMEL_TRY { + new ((*newBlock)[currentTailIndex]) T(std::forward(element)); + } + MOODYCAMEL_CATCH (...) { + rewind_block_index_tail(); + idxEntry->value.store(nullptr, std::memory_order_relaxed); + this->parent->add_block_to_free_list(newBlock); + MOODYCAMEL_RETHROW; + } + } + + // Insert the new block into the index + idxEntry->value.store(newBlock, std::memory_order_relaxed); + + this->tailBlock = newBlock; + + MOODYCAMEL_CONSTEXPR_IF (!MOODYCAMEL_NOEXCEPT_CTOR(T, U, new (static_cast(nullptr)) T(std::forward(element)))) { + this->tailIndex.store(newTailIndex, std::memory_order_release); + return true; + } + } + + // Enqueue + new ((*this->tailBlock)[currentTailIndex]) T(std::forward(element)); + + this->tailIndex.store(newTailIndex, std::memory_order_release); + return true; + } + + template + bool dequeue(U& element) + { + // See ExplicitProducer::dequeue for rationale and explanation + index_t tail = this->tailIndex.load(std::memory_order_relaxed); + index_t overcommit = this->dequeueOvercommit.load(std::memory_order_relaxed); + if (details::circular_less_than(this->dequeueOptimisticCount.load(std::memory_order_relaxed) - overcommit, tail)) { + std::atomic_thread_fence(std::memory_order_acquire); + + index_t myDequeueCount = this->dequeueOptimisticCount.fetch_add(1, std::memory_order_relaxed); + tail = this->tailIndex.load(std::memory_order_acquire); + if ((details::likely)(details::circular_less_than(myDequeueCount - overcommit, tail))) { + index_t index = this->headIndex.fetch_add(1, std::memory_order_acq_rel); + + // Determine which block the element is in + auto entry = get_block_index_entry_for_index(index); + + // Dequeue + auto block = entry->value.load(std::memory_order_relaxed); + auto& el = *((*block)[index]); + + if (!MOODYCAMEL_NOEXCEPT_ASSIGN(T, T&&, element = std::move(el))) { +#ifdef MCDBGQ_NOLOCKFREE_IMPLICITPRODBLOCKINDEX + // Note: Acquiring the mutex with every dequeue instead of only when a block + // is released is very sub-optimal, but it is, after all, purely debug code. + debug::DebugLock lock(producer->mutex); +#endif + struct Guard { + Block* block; + index_t index; + BlockIndexEntry* entry; + ConcurrentQueue* parent; + + ~Guard() + { + (*block)[index]->~T(); + if (block->ConcurrentQueue::Block::template set_empty(index)) { + entry->value.store(nullptr, std::memory_order_relaxed); + parent->add_block_to_free_list(block); + } + } + } guard = { block, index, entry, this->parent }; + + element = std::move(el); // NOLINT + } + else { + element = std::move(el); // NOLINT + el.~T(); // NOLINT + + if (block->ConcurrentQueue::Block::template set_empty(index)) { + { +#ifdef MCDBGQ_NOLOCKFREE_IMPLICITPRODBLOCKINDEX + debug::DebugLock lock(mutex); +#endif + // Add the block back into the global free pool (and remove from block index) + entry->value.store(nullptr, std::memory_order_relaxed); + } + this->parent->add_block_to_free_list(block); // releases the above store + } + } + + return true; + } + else { + this->dequeueOvercommit.fetch_add(1, std::memory_order_release); + } + } + + return false; + } + +#ifdef _MSC_VER +#pragma warning(push) +#pragma warning(disable: 4706) // assignment within conditional expression +#endif + template + bool enqueue_bulk(It itemFirst, size_t count) + { + // First, we need to make sure we have enough room to enqueue all of the elements; + // this means pre-allocating blocks and putting them in the block index (but only if + // all the allocations succeeded). + + // Note that the tailBlock we start off with may not be owned by us any more; + // this happens if it was filled up exactly to the top (setting tailIndex to + // the first index of the next block which is not yet allocated), then dequeued + // completely (putting it on the free list) before we enqueue again. + + index_t startTailIndex = this->tailIndex.load(std::memory_order_relaxed); + auto startBlock = this->tailBlock; + Block* firstAllocatedBlock = nullptr; + auto endBlock = this->tailBlock; + + // Figure out how many blocks we'll need to allocate, and do so + size_t blockBaseDiff = ((startTailIndex + count - 1) & ~static_cast(BLOCK_SIZE - 1)) - ((startTailIndex - 1) & ~static_cast(BLOCK_SIZE - 1)); + index_t currentTailIndex = (startTailIndex - 1) & ~static_cast(BLOCK_SIZE - 1); + if (blockBaseDiff > 0) { +#ifdef MCDBGQ_NOLOCKFREE_IMPLICITPRODBLOCKINDEX + debug::DebugLock lock(mutex); +#endif + do { + blockBaseDiff -= static_cast(BLOCK_SIZE); + currentTailIndex += static_cast(BLOCK_SIZE); + + // Find out where we'll be inserting this block in the block index + BlockIndexEntry* idxEntry = nullptr; // initialization here unnecessary but compiler can't always tell + Block* newBlock; + bool indexInserted = false; + auto head = this->headIndex.load(std::memory_order_relaxed); + assert(!details::circular_less_than(currentTailIndex, head)); + bool full = !details::circular_less_than(head, currentTailIndex + BLOCK_SIZE) || (MAX_SUBQUEUE_SIZE != details::const_numeric_max::value && (MAX_SUBQUEUE_SIZE == 0 || MAX_SUBQUEUE_SIZE - BLOCK_SIZE < currentTailIndex - head)); + + if (full || !(indexInserted = insert_block_index_entry(idxEntry, currentTailIndex)) || (newBlock = this->parent->ConcurrentQueue::template requisition_block()) == nullptr) { + // Index allocation or block allocation failed; revert any other allocations + // and index insertions done so far for this operation + if (indexInserted) { + rewind_block_index_tail(); + idxEntry->value.store(nullptr, std::memory_order_relaxed); + } + currentTailIndex = (startTailIndex - 1) & ~static_cast(BLOCK_SIZE - 1); + for (auto block = firstAllocatedBlock; block != nullptr; block = block->next) { + currentTailIndex += static_cast(BLOCK_SIZE); + idxEntry = get_block_index_entry_for_index(currentTailIndex); + idxEntry->value.store(nullptr, std::memory_order_relaxed); + rewind_block_index_tail(); + } + this->parent->add_blocks_to_free_list(firstAllocatedBlock); + this->tailBlock = startBlock; + + return false; + } + +#ifdef MCDBGQ_TRACKMEM + newBlock->owner = this; +#endif + newBlock->ConcurrentQueue::Block::template reset_empty(); + newBlock->next = nullptr; + + // Insert the new block into the index + idxEntry->value.store(newBlock, std::memory_order_relaxed); + + // Store the chain of blocks so that we can undo if later allocations fail, + // and so that we can find the blocks when we do the actual enqueueing + if ((startTailIndex & static_cast(BLOCK_SIZE - 1)) != 0 || firstAllocatedBlock != nullptr) { + assert(this->tailBlock != nullptr); + this->tailBlock->next = newBlock; + } + this->tailBlock = newBlock; + endBlock = newBlock; + firstAllocatedBlock = firstAllocatedBlock == nullptr ? newBlock : firstAllocatedBlock; + } while (blockBaseDiff > 0); + } + + // Enqueue, one block at a time + index_t newTailIndex = startTailIndex + static_cast(count); + currentTailIndex = startTailIndex; + this->tailBlock = startBlock; + assert((startTailIndex & static_cast(BLOCK_SIZE - 1)) != 0 || firstAllocatedBlock != nullptr || count == 0); + if ((startTailIndex & static_cast(BLOCK_SIZE - 1)) == 0 && firstAllocatedBlock != nullptr) { + this->tailBlock = firstAllocatedBlock; + } + while (true) { + index_t stopIndex = (currentTailIndex & ~static_cast(BLOCK_SIZE - 1)) + static_cast(BLOCK_SIZE); + if (details::circular_less_than(newTailIndex, stopIndex)) { + stopIndex = newTailIndex; + } + MOODYCAMEL_CONSTEXPR_IF (MOODYCAMEL_NOEXCEPT_CTOR(T, decltype(*itemFirst), new (static_cast(nullptr)) T(details::deref_noexcept(itemFirst)))) { + while (currentTailIndex != stopIndex) { + new ((*this->tailBlock)[currentTailIndex++]) T(*itemFirst++); + } + } + else { + MOODYCAMEL_TRY { + while (currentTailIndex != stopIndex) { + new ((*this->tailBlock)[currentTailIndex]) T(details::nomove_if(nullptr)) T(details::deref_noexcept(itemFirst)))>::eval(*itemFirst)); + ++currentTailIndex; + ++itemFirst; + } + } + MOODYCAMEL_CATCH (...) { + auto constructedStopIndex = currentTailIndex; + auto lastBlockEnqueued = this->tailBlock; + + if (!details::is_trivially_destructible::value) { + auto block = startBlock; + if ((startTailIndex & static_cast(BLOCK_SIZE - 1)) == 0) { + block = firstAllocatedBlock; + } + currentTailIndex = startTailIndex; + while (true) { + stopIndex = (currentTailIndex & ~static_cast(BLOCK_SIZE - 1)) + static_cast(BLOCK_SIZE); + if (details::circular_less_than(constructedStopIndex, stopIndex)) { + stopIndex = constructedStopIndex; + } + while (currentTailIndex != stopIndex) { + (*block)[currentTailIndex++]->~T(); + } + if (block == lastBlockEnqueued) { + break; + } + block = block->next; + } + } + + currentTailIndex = (startTailIndex - 1) & ~static_cast(BLOCK_SIZE - 1); + for (auto block = firstAllocatedBlock; block != nullptr; block = block->next) { + currentTailIndex += static_cast(BLOCK_SIZE); + auto idxEntry = get_block_index_entry_for_index(currentTailIndex); + idxEntry->value.store(nullptr, std::memory_order_relaxed); + rewind_block_index_tail(); + } + this->parent->add_blocks_to_free_list(firstAllocatedBlock); + this->tailBlock = startBlock; + MOODYCAMEL_RETHROW; + } + } + + if (this->tailBlock == endBlock) { + assert(currentTailIndex == newTailIndex); + break; + } + this->tailBlock = this->tailBlock->next; + } + this->tailIndex.store(newTailIndex, std::memory_order_release); + return true; + } +#ifdef _MSC_VER +#pragma warning(pop) +#endif + + template + size_t dequeue_bulk(It& itemFirst, size_t max) + { + auto tail = this->tailIndex.load(std::memory_order_relaxed); + auto overcommit = this->dequeueOvercommit.load(std::memory_order_relaxed); + auto desiredCount = static_cast(tail - (this->dequeueOptimisticCount.load(std::memory_order_relaxed) - overcommit)); + if (details::circular_less_than(0, desiredCount)) { + desiredCount = desiredCount < max ? desiredCount : max; + std::atomic_thread_fence(std::memory_order_acquire); + + auto myDequeueCount = this->dequeueOptimisticCount.fetch_add(desiredCount, std::memory_order_relaxed); + + tail = this->tailIndex.load(std::memory_order_acquire); + auto actualCount = static_cast(tail - (myDequeueCount - overcommit)); + if (details::circular_less_than(0, actualCount)) { + actualCount = desiredCount < actualCount ? desiredCount : actualCount; + if (actualCount < desiredCount) { + this->dequeueOvercommit.fetch_add(desiredCount - actualCount, std::memory_order_release); + } + + // Get the first index. Note that since there's guaranteed to be at least actualCount elements, this + // will never exceed tail. + auto firstIndex = this->headIndex.fetch_add(actualCount, std::memory_order_acq_rel); + + // Iterate the blocks and dequeue + auto index = firstIndex; + BlockIndexHeader* localBlockIndex; + auto indexIndex = get_block_index_index_for_index(index, localBlockIndex); + do { + auto blockStartIndex = index; + index_t endIndex = (index & ~static_cast(BLOCK_SIZE - 1)) + static_cast(BLOCK_SIZE); + endIndex = details::circular_less_than(firstIndex + static_cast(actualCount), endIndex) ? firstIndex + static_cast(actualCount) : endIndex; + + auto entry = localBlockIndex->index[indexIndex]; + auto block = entry->value.load(std::memory_order_relaxed); + if (MOODYCAMEL_NOEXCEPT_ASSIGN(T, T&&, details::deref_noexcept(itemFirst) = std::move((*(*block)[index])))) { + while (index != endIndex) { + auto& el = *((*block)[index]); + *itemFirst++ = std::move(el); + el.~T(); + ++index; + } + } + else { + MOODYCAMEL_TRY { + while (index != endIndex) { + auto& el = *((*block)[index]); + *itemFirst = std::move(el); + ++itemFirst; + el.~T(); + ++index; + } + } + MOODYCAMEL_CATCH (...) { + do { + entry = localBlockIndex->index[indexIndex]; + block = entry->value.load(std::memory_order_relaxed); + while (index != endIndex) { + (*block)[index++]->~T(); + } + + if (block->ConcurrentQueue::Block::template set_many_empty(blockStartIndex, static_cast(endIndex - blockStartIndex))) { +#ifdef MCDBGQ_NOLOCKFREE_IMPLICITPRODBLOCKINDEX + debug::DebugLock lock(mutex); +#endif + entry->value.store(nullptr, std::memory_order_relaxed); + this->parent->add_block_to_free_list(block); + } + indexIndex = (indexIndex + 1) & (localBlockIndex->capacity - 1); + + blockStartIndex = index; + endIndex = (index & ~static_cast(BLOCK_SIZE - 1)) + static_cast(BLOCK_SIZE); + endIndex = details::circular_less_than(firstIndex + static_cast(actualCount), endIndex) ? firstIndex + static_cast(actualCount) : endIndex; + } while (index != firstIndex + actualCount); + + MOODYCAMEL_RETHROW; + } + } + if (block->ConcurrentQueue::Block::template set_many_empty(blockStartIndex, static_cast(endIndex - blockStartIndex))) { + { +#ifdef MCDBGQ_NOLOCKFREE_IMPLICITPRODBLOCKINDEX + debug::DebugLock lock(mutex); +#endif + // Note that the set_many_empty above did a release, meaning that anybody who acquires the block + // we're about to free can use it safely since our writes (and reads!) will have happened-before then. + entry->value.store(nullptr, std::memory_order_relaxed); + } + this->parent->add_block_to_free_list(block); // releases the above store + } + indexIndex = (indexIndex + 1) & (localBlockIndex->capacity - 1); + } while (index != firstIndex + actualCount); + + return actualCount; + } + else { + this->dequeueOvercommit.fetch_add(desiredCount, std::memory_order_release); + } + } + + return 0; + } + + private: + // The block size must be > 1, so any number with the low bit set is an invalid block base index + static const index_t INVALID_BLOCK_BASE = 1; + + struct BlockIndexEntry + { + std::atomic key; + std::atomic value; + }; + + struct BlockIndexHeader + { + size_t capacity; + std::atomic tail; + BlockIndexEntry* entries; + BlockIndexEntry** index; + BlockIndexHeader* prev; + }; + + template + inline bool insert_block_index_entry(BlockIndexEntry*& idxEntry, index_t blockStartIndex) + { + auto localBlockIndex = blockIndex.load(std::memory_order_relaxed); // We're the only writer thread, relaxed is OK + if (localBlockIndex == nullptr) { + return false; // this can happen if new_block_index failed in the constructor + } + size_t newTail = (localBlockIndex->tail.load(std::memory_order_relaxed) + 1) & (localBlockIndex->capacity - 1); + idxEntry = localBlockIndex->index[newTail]; + if (idxEntry->key.load(std::memory_order_relaxed) == INVALID_BLOCK_BASE || + idxEntry->value.load(std::memory_order_relaxed) == nullptr) { + + idxEntry->key.store(blockStartIndex, std::memory_order_relaxed); + localBlockIndex->tail.store(newTail, std::memory_order_release); + return true; + } + + // No room in the old block index, try to allocate another one! + MOODYCAMEL_CONSTEXPR_IF (allocMode == CannotAlloc) { + return false; + } + else if (!new_block_index()) { + return false; + } + else { + localBlockIndex = blockIndex.load(std::memory_order_relaxed); + newTail = (localBlockIndex->tail.load(std::memory_order_relaxed) + 1) & (localBlockIndex->capacity - 1); + idxEntry = localBlockIndex->index[newTail]; + assert(idxEntry->key.load(std::memory_order_relaxed) == INVALID_BLOCK_BASE); + idxEntry->key.store(blockStartIndex, std::memory_order_relaxed); + localBlockIndex->tail.store(newTail, std::memory_order_release); + return true; + } + } + + inline void rewind_block_index_tail() + { + auto localBlockIndex = blockIndex.load(std::memory_order_relaxed); + localBlockIndex->tail.store((localBlockIndex->tail.load(std::memory_order_relaxed) - 1) & (localBlockIndex->capacity - 1), std::memory_order_relaxed); + } + + inline BlockIndexEntry* get_block_index_entry_for_index(index_t index) const + { + BlockIndexHeader* localBlockIndex; + auto idx = get_block_index_index_for_index(index, localBlockIndex); + return localBlockIndex->index[idx]; + } + + inline size_t get_block_index_index_for_index(index_t index, BlockIndexHeader*& localBlockIndex) const + { +#ifdef MCDBGQ_NOLOCKFREE_IMPLICITPRODBLOCKINDEX + debug::DebugLock lock(mutex); +#endif + index &= ~static_cast(BLOCK_SIZE - 1); + localBlockIndex = blockIndex.load(std::memory_order_acquire); + auto tail = localBlockIndex->tail.load(std::memory_order_acquire); + auto tailBase = localBlockIndex->index[tail]->key.load(std::memory_order_relaxed); + assert(tailBase != INVALID_BLOCK_BASE); + // Note: Must use division instead of shift because the index may wrap around, causing a negative + // offset, whose negativity we want to preserve + auto offset = static_cast(static_cast::type>(index - tailBase) / static_cast::type>(BLOCK_SIZE)); + size_t idx = (tail + offset) & (localBlockIndex->capacity - 1); + assert(localBlockIndex->index[idx]->key.load(std::memory_order_relaxed) == index && localBlockIndex->index[idx]->value.load(std::memory_order_relaxed) != nullptr); + return idx; + } + + bool new_block_index() + { + auto prev = blockIndex.load(std::memory_order_relaxed); + size_t prevCapacity = prev == nullptr ? 0 : prev->capacity; + auto entryCount = prev == nullptr ? nextBlockIndexCapacity : prevCapacity; + auto raw = static_cast((Traits::malloc)( + sizeof(BlockIndexHeader) + + std::alignment_of::value - 1 + sizeof(BlockIndexEntry) * entryCount + + std::alignment_of::value - 1 + sizeof(BlockIndexEntry*) * nextBlockIndexCapacity)); + if (raw == nullptr) { + return false; + } + + auto header = new (raw) BlockIndexHeader; + auto entries = reinterpret_cast(details::align_for(raw + sizeof(BlockIndexHeader))); + auto index = reinterpret_cast(details::align_for(reinterpret_cast(entries) + sizeof(BlockIndexEntry) * entryCount)); + if (prev != nullptr) { + auto prevTail = prev->tail.load(std::memory_order_relaxed); + auto prevPos = prevTail; + size_t i = 0; + do { + prevPos = (prevPos + 1) & (prev->capacity - 1); + index[i++] = prev->index[prevPos]; + } while (prevPos != prevTail); + assert(i == prevCapacity); + } + for (size_t i = 0; i != entryCount; ++i) { + new (entries + i) BlockIndexEntry; + entries[i].key.store(INVALID_BLOCK_BASE, std::memory_order_relaxed); + index[prevCapacity + i] = entries + i; + } + header->prev = prev; + header->entries = entries; + header->index = index; + header->capacity = nextBlockIndexCapacity; + header->tail.store((prevCapacity - 1) & (nextBlockIndexCapacity - 1), std::memory_order_relaxed); + + blockIndex.store(header, std::memory_order_release); + + nextBlockIndexCapacity <<= 1; + + return true; + } + + private: + size_t nextBlockIndexCapacity; + std::atomic blockIndex; + +#ifdef MOODYCAMEL_CPP11_THREAD_LOCAL_SUPPORTED + public: + details::ThreadExitListener threadExitListener; + private: +#endif + +#ifdef MOODYCAMEL_QUEUE_INTERNAL_DEBUG + public: + ImplicitProducer* nextImplicitProducer; + private: +#endif + +#ifdef MCDBGQ_NOLOCKFREE_IMPLICITPRODBLOCKINDEX + mutable debug::DebugMutex mutex; +#endif +#ifdef MCDBGQ_TRACKMEM + friend struct MemStats; +#endif + }; + + + ////////////////////////////////// + // Block pool manipulation + ////////////////////////////////// + + void populate_initial_block_list(size_t blockCount) + { + initialBlockPoolSize = blockCount; + if (initialBlockPoolSize == 0) { + initialBlockPool = nullptr; + return; + } + + initialBlockPool = create_array(blockCount); + if (initialBlockPool == nullptr) { + initialBlockPoolSize = 0; + } + for (size_t i = 0; i < initialBlockPoolSize; ++i) { + initialBlockPool[i].dynamicallyAllocated = false; + } + } + + inline Block* try_get_block_from_initial_pool() + { + if (initialBlockPoolIndex.load(std::memory_order_relaxed) >= initialBlockPoolSize) { + return nullptr; + } + + auto index = initialBlockPoolIndex.fetch_add(1, std::memory_order_relaxed); + + return index < initialBlockPoolSize ? (initialBlockPool + index) : nullptr; + } + + inline void add_block_to_free_list(Block* block) + { +#ifdef MCDBGQ_TRACKMEM + block->owner = nullptr; +#endif + if (!Traits::RECYCLE_ALLOCATED_BLOCKS && block->dynamicallyAllocated) { + destroy(block); + } + else { + freeList.add(block); + } + } + + inline void add_blocks_to_free_list(Block* block) + { + while (block != nullptr) { + auto next = block->next; + add_block_to_free_list(block); + block = next; + } + } + + inline Block* try_get_block_from_free_list() + { + return freeList.try_get(); + } + + // Gets a free block from one of the memory pools, or allocates a new one (if applicable) + template + Block* requisition_block() + { + auto block = try_get_block_from_initial_pool(); + if (block != nullptr) { + return block; + } + + block = try_get_block_from_free_list(); + if (block != nullptr) { + return block; + } + + MOODYCAMEL_CONSTEXPR_IF (canAlloc == CanAlloc) { + return create(); + } + else { + return nullptr; + } + } + + +#ifdef MCDBGQ_TRACKMEM + public: + struct MemStats { + size_t allocatedBlocks; + size_t usedBlocks; + size_t freeBlocks; + size_t ownedBlocksExplicit; + size_t ownedBlocksImplicit; + size_t implicitProducers; + size_t explicitProducers; + size_t elementsEnqueued; + size_t blockClassBytes; + size_t queueClassBytes; + size_t implicitBlockIndexBytes; + size_t explicitBlockIndexBytes; + + friend class ConcurrentQueue; + + private: + static MemStats getFor(ConcurrentQueue* q) + { + MemStats stats = { 0 }; + + stats.elementsEnqueued = q->size_approx(); + + auto block = q->freeList.head_unsafe(); + while (block != nullptr) { + ++stats.allocatedBlocks; + ++stats.freeBlocks; + block = block->freeListNext.load(std::memory_order_relaxed); + } + + for (auto ptr = q->producerListTail.load(std::memory_order_acquire); ptr != nullptr; ptr = ptr->next_prod()) { + bool implicit = dynamic_cast(ptr) != nullptr; + stats.implicitProducers += implicit ? 1 : 0; + stats.explicitProducers += implicit ? 0 : 1; + + if (implicit) { + auto prod = static_cast(ptr); + stats.queueClassBytes += sizeof(ImplicitProducer); + auto head = prod->headIndex.load(std::memory_order_relaxed); + auto tail = prod->tailIndex.load(std::memory_order_relaxed); + auto hash = prod->blockIndex.load(std::memory_order_relaxed); + if (hash != nullptr) { + for (size_t i = 0; i != hash->capacity; ++i) { + if (hash->index[i]->key.load(std::memory_order_relaxed) != ImplicitProducer::INVALID_BLOCK_BASE && hash->index[i]->value.load(std::memory_order_relaxed) != nullptr) { + ++stats.allocatedBlocks; + ++stats.ownedBlocksImplicit; + } + } + stats.implicitBlockIndexBytes += hash->capacity * sizeof(typename ImplicitProducer::BlockIndexEntry); + for (; hash != nullptr; hash = hash->prev) { + stats.implicitBlockIndexBytes += sizeof(typename ImplicitProducer::BlockIndexHeader) + hash->capacity * sizeof(typename ImplicitProducer::BlockIndexEntry*); + } + } + for (; details::circular_less_than(head, tail); head += BLOCK_SIZE) { + //auto block = prod->get_block_index_entry_for_index(head); + ++stats.usedBlocks; + } + } + else { + auto prod = static_cast(ptr); + stats.queueClassBytes += sizeof(ExplicitProducer); + auto tailBlock = prod->tailBlock; + bool wasNonEmpty = false; + if (tailBlock != nullptr) { + auto block = tailBlock; + do { + ++stats.allocatedBlocks; + if (!block->ConcurrentQueue::Block::template is_empty() || wasNonEmpty) { + ++stats.usedBlocks; + wasNonEmpty = wasNonEmpty || block != tailBlock; + } + ++stats.ownedBlocksExplicit; + block = block->next; + } while (block != tailBlock); + } + auto index = prod->blockIndex.load(std::memory_order_relaxed); + while (index != nullptr) { + stats.explicitBlockIndexBytes += sizeof(typename ExplicitProducer::BlockIndexHeader) + index->size * sizeof(typename ExplicitProducer::BlockIndexEntry); + index = static_cast(index->prev); + } + } + } + + auto freeOnInitialPool = q->initialBlockPoolIndex.load(std::memory_order_relaxed) >= q->initialBlockPoolSize ? 0 : q->initialBlockPoolSize - q->initialBlockPoolIndex.load(std::memory_order_relaxed); + stats.allocatedBlocks += freeOnInitialPool; + stats.freeBlocks += freeOnInitialPool; + + stats.blockClassBytes = sizeof(Block) * stats.allocatedBlocks; + stats.queueClassBytes += sizeof(ConcurrentQueue); + + return stats; + } + }; + + // For debugging only. Not thread-safe. + MemStats getMemStats() + { + return MemStats::getFor(this); + } + private: + friend struct MemStats; +#endif + + + ////////////////////////////////// + // Producer list manipulation + ////////////////////////////////// + + ProducerBase* recycle_or_create_producer(bool isExplicit) + { +#ifdef MCDBGQ_NOLOCKFREE_IMPLICITPRODHASH + debug::DebugLock lock(implicitProdMutex); +#endif + // Try to re-use one first + for (auto ptr = producerListTail.load(std::memory_order_acquire); ptr != nullptr; ptr = ptr->next_prod()) { + if (ptr->inactive.load(std::memory_order_relaxed) && ptr->isExplicit == isExplicit) { + bool expected = true; + if (ptr->inactive.compare_exchange_strong(expected, /* desired */ false, std::memory_order_acquire, std::memory_order_relaxed)) { + // We caught one! It's been marked as activated, the caller can have it + return ptr; + } + } + } + + return add_producer(isExplicit ? static_cast(create(this)) : create(this)); + } + + ProducerBase* add_producer(ProducerBase* producer) + { + // Handle failed memory allocation + if (producer == nullptr) { + return nullptr; + } + + producerCount.fetch_add(1, std::memory_order_relaxed); + + // Add it to the lock-free list + auto prevTail = producerListTail.load(std::memory_order_relaxed); + do { + producer->next = prevTail; + } while (!producerListTail.compare_exchange_weak(prevTail, producer, std::memory_order_release, std::memory_order_relaxed)); + +#ifdef MOODYCAMEL_QUEUE_INTERNAL_DEBUG + if (producer->isExplicit) { + auto prevTailExplicit = explicitProducers.load(std::memory_order_relaxed); + do { + static_cast(producer)->nextExplicitProducer = prevTailExplicit; + } while (!explicitProducers.compare_exchange_weak(prevTailExplicit, static_cast(producer), std::memory_order_release, std::memory_order_relaxed)); + } + else { + auto prevTailImplicit = implicitProducers.load(std::memory_order_relaxed); + do { + static_cast(producer)->nextImplicitProducer = prevTailImplicit; + } while (!implicitProducers.compare_exchange_weak(prevTailImplicit, static_cast(producer), std::memory_order_release, std::memory_order_relaxed)); + } +#endif + + return producer; + } + + void reown_producers() + { + // After another instance is moved-into/swapped-with this one, all the + // producers we stole still think their parents are the other queue. + // So fix them up! + for (auto ptr = producerListTail.load(std::memory_order_relaxed); ptr != nullptr; ptr = ptr->next_prod()) { + ptr->parent = this; + } + } + + + ////////////////////////////////// + // Implicit producer hash + ////////////////////////////////// + + struct ImplicitProducerKVP + { + std::atomic key; + ImplicitProducer* value; // No need for atomicity since it's only read by the thread that sets it in the first place + + ImplicitProducerKVP() : value(nullptr) { } + + ImplicitProducerKVP(ImplicitProducerKVP&& other) MOODYCAMEL_NOEXCEPT + { + key.store(other.key.load(std::memory_order_relaxed), std::memory_order_relaxed); + value = other.value; + } + + inline ImplicitProducerKVP& operator=(ImplicitProducerKVP&& other) MOODYCAMEL_NOEXCEPT + { + swap(other); + return *this; + } + + inline void swap(ImplicitProducerKVP& other) MOODYCAMEL_NOEXCEPT + { + if (this != &other) { + details::swap_relaxed(key, other.key); + std::swap(value, other.value); + } + } + }; + + template + friend void moodycamel::swap(typename ConcurrentQueue::ImplicitProducerKVP&, typename ConcurrentQueue::ImplicitProducerKVP&) MOODYCAMEL_NOEXCEPT; + + struct ImplicitProducerHash + { + size_t capacity; + ImplicitProducerKVP* entries; + ImplicitProducerHash* prev; + }; + + inline void populate_initial_implicit_producer_hash() + { + MOODYCAMEL_CONSTEXPR_IF (INITIAL_IMPLICIT_PRODUCER_HASH_SIZE == 0) { + return; + } + else { + implicitProducerHashCount.store(0, std::memory_order_relaxed); + auto hash = &initialImplicitProducerHash; + hash->capacity = INITIAL_IMPLICIT_PRODUCER_HASH_SIZE; + hash->entries = &initialImplicitProducerHashEntries[0]; + for (size_t i = 0; i != INITIAL_IMPLICIT_PRODUCER_HASH_SIZE; ++i) { + initialImplicitProducerHashEntries[i].key.store(details::invalid_thread_id, std::memory_order_relaxed); + } + hash->prev = nullptr; + implicitProducerHash.store(hash, std::memory_order_relaxed); + } + } + + void swap_implicit_producer_hashes(ConcurrentQueue& other) + { + MOODYCAMEL_CONSTEXPR_IF (INITIAL_IMPLICIT_PRODUCER_HASH_SIZE == 0) { + return; + } + else { + // Swap (assumes our implicit producer hash is initialized) + initialImplicitProducerHashEntries.swap(other.initialImplicitProducerHashEntries); + initialImplicitProducerHash.entries = &initialImplicitProducerHashEntries[0]; + other.initialImplicitProducerHash.entries = &other.initialImplicitProducerHashEntries[0]; + + details::swap_relaxed(implicitProducerHashCount, other.implicitProducerHashCount); + + details::swap_relaxed(implicitProducerHash, other.implicitProducerHash); + if (implicitProducerHash.load(std::memory_order_relaxed) == &other.initialImplicitProducerHash) { + implicitProducerHash.store(&initialImplicitProducerHash, std::memory_order_relaxed); + } + else { + ImplicitProducerHash* hash; + for (hash = implicitProducerHash.load(std::memory_order_relaxed); hash->prev != &other.initialImplicitProducerHash; hash = hash->prev) { + continue; + } + hash->prev = &initialImplicitProducerHash; + } + if (other.implicitProducerHash.load(std::memory_order_relaxed) == &initialImplicitProducerHash) { + other.implicitProducerHash.store(&other.initialImplicitProducerHash, std::memory_order_relaxed); + } + else { + ImplicitProducerHash* hash; + for (hash = other.implicitProducerHash.load(std::memory_order_relaxed); hash->prev != &initialImplicitProducerHash; hash = hash->prev) { + continue; + } + hash->prev = &other.initialImplicitProducerHash; + } + } + } + + // Only fails (returns nullptr) if memory allocation fails + ImplicitProducer* get_or_add_implicit_producer() + { + // Note that since the data is essentially thread-local (key is thread ID), + // there's a reduced need for fences (memory ordering is already consistent + // for any individual thread), except for the current table itself. + + // Start by looking for the thread ID in the current and all previous hash tables. + // If it's not found, it must not be in there yet, since this same thread would + // have added it previously to one of the tables that we traversed. + + // Code and algorithm adapted from http://preshing.com/20130605/the-worlds-simplest-lock-free-hash-table + +#ifdef MCDBGQ_NOLOCKFREE_IMPLICITPRODHASH + debug::DebugLock lock(implicitProdMutex); +#endif + + auto id = details::thread_id(); + auto hashedId = details::hash_thread_id(id); + + auto mainHash = implicitProducerHash.load(std::memory_order_acquire); + assert(mainHash != nullptr); // silence clang-tidy and MSVC warnings (hash cannot be null) + for (auto hash = mainHash; hash != nullptr; hash = hash->prev) { + // Look for the id in this hash + auto index = hashedId; + while (true) { // Not an infinite loop because at least one slot is free in the hash table + index &= hash->capacity - 1u; + + auto probedKey = hash->entries[index].key.load(std::memory_order_relaxed); + if (probedKey == id) { + // Found it! If we had to search several hashes deep, though, we should lazily add it + // to the current main hash table to avoid the extended search next time. + // Note there's guaranteed to be room in the current hash table since every subsequent + // table implicitly reserves space for all previous tables (there's only one + // implicitProducerHashCount). + auto value = hash->entries[index].value; + if (hash != mainHash) { + index = hashedId; + while (true) { + index &= mainHash->capacity - 1u; + auto empty = details::invalid_thread_id; +#ifdef MOODYCAMEL_CPP11_THREAD_LOCAL_SUPPORTED + auto reusable = details::invalid_thread_id2; + if (mainHash->entries[index].key.compare_exchange_strong(empty, id, std::memory_order_seq_cst, std::memory_order_relaxed) || + mainHash->entries[index].key.compare_exchange_strong(reusable, id, std::memory_order_seq_cst, std::memory_order_relaxed)) { +#else + if (mainHash->entries[index].key.compare_exchange_strong(empty, id, std::memory_order_seq_cst, std::memory_order_relaxed)) { +#endif + mainHash->entries[index].value = value; + break; + } + ++index; + } + } + + return value; + } + if (probedKey == details::invalid_thread_id) { + break; // Not in this hash table + } + ++index; + } + } + + // Insert! + auto newCount = 1 + implicitProducerHashCount.fetch_add(1, std::memory_order_relaxed); + while (true) { + // NOLINTNEXTLINE(clang-analyzer-core.NullDereference) + if (newCount >= (mainHash->capacity >> 1) && !implicitProducerHashResizeInProgress.test_and_set(std::memory_order_acquire)) { + // We've acquired the resize lock, try to allocate a bigger hash table. + // Note the acquire fence synchronizes with the release fence at the end of this block, and hence when + // we reload implicitProducerHash it must be the most recent version (it only gets changed within this + // locked block). + mainHash = implicitProducerHash.load(std::memory_order_acquire); + if (newCount >= (mainHash->capacity >> 1)) { + size_t newCapacity = mainHash->capacity << 1; + while (newCount >= (newCapacity >> 1)) { + newCapacity <<= 1; + } + auto raw = static_cast((Traits::malloc)(sizeof(ImplicitProducerHash) + std::alignment_of::value - 1 + sizeof(ImplicitProducerKVP) * newCapacity)); + if (raw == nullptr) { + // Allocation failed + implicitProducerHashCount.fetch_sub(1, std::memory_order_relaxed); + implicitProducerHashResizeInProgress.clear(std::memory_order_relaxed); + return nullptr; + } + + auto newHash = new (raw) ImplicitProducerHash; + newHash->capacity = static_cast(newCapacity); + newHash->entries = reinterpret_cast(details::align_for(raw + sizeof(ImplicitProducerHash))); + for (size_t i = 0; i != newCapacity; ++i) { + new (newHash->entries + i) ImplicitProducerKVP; + newHash->entries[i].key.store(details::invalid_thread_id, std::memory_order_relaxed); + } + newHash->prev = mainHash; + implicitProducerHash.store(newHash, std::memory_order_release); + implicitProducerHashResizeInProgress.clear(std::memory_order_release); + mainHash = newHash; + } + else { + implicitProducerHashResizeInProgress.clear(std::memory_order_release); + } + } + + // If it's < three-quarters full, add to the old one anyway so that we don't have to wait for the next table + // to finish being allocated by another thread (and if we just finished allocating above, the condition will + // always be true) + if (newCount < (mainHash->capacity >> 1) + (mainHash->capacity >> 2)) { + auto producer = static_cast(recycle_or_create_producer(false)); + if (producer == nullptr) { + implicitProducerHashCount.fetch_sub(1, std::memory_order_relaxed); + return nullptr; + } + +#ifdef MOODYCAMEL_CPP11_THREAD_LOCAL_SUPPORTED + producer->threadExitListener.callback = &ConcurrentQueue::implicit_producer_thread_exited_callback; + producer->threadExitListener.userData = producer; + details::ThreadExitNotifier::subscribe(&producer->threadExitListener); +#endif + + auto index = hashedId; + while (true) { + index &= mainHash->capacity - 1u; + auto empty = details::invalid_thread_id; +#ifdef MOODYCAMEL_CPP11_THREAD_LOCAL_SUPPORTED + auto reusable = details::invalid_thread_id2; + if (mainHash->entries[index].key.compare_exchange_strong(reusable, id, std::memory_order_seq_cst, std::memory_order_relaxed)) { + implicitProducerHashCount.fetch_sub(1, std::memory_order_relaxed); // already counted as a used slot + mainHash->entries[index].value = producer; + break; + } +#endif + if (mainHash->entries[index].key.compare_exchange_strong(empty, id, std::memory_order_seq_cst, std::memory_order_relaxed)) { + mainHash->entries[index].value = producer; + break; + } + ++index; + } + return producer; + } + + // Hmm, the old hash is quite full and somebody else is busy allocating a new one. + // We need to wait for the allocating thread to finish (if it succeeds, we add, if not, + // we try to allocate ourselves). + mainHash = implicitProducerHash.load(std::memory_order_acquire); + } + } + +#ifdef MOODYCAMEL_CPP11_THREAD_LOCAL_SUPPORTED + void implicit_producer_thread_exited(ImplicitProducer* producer) + { + // Remove from hash +#ifdef MCDBGQ_NOLOCKFREE_IMPLICITPRODHASH + debug::DebugLock lock(implicitProdMutex); +#endif + auto hash = implicitProducerHash.load(std::memory_order_acquire); + assert(hash != nullptr); // The thread exit listener is only registered if we were added to a hash in the first place + auto id = details::thread_id(); + auto hashedId = details::hash_thread_id(id); + details::thread_id_t probedKey; + + // We need to traverse all the hashes just in case other threads aren't on the current one yet and are + // trying to add an entry thinking there's a free slot (because they reused a producer) + for (; hash != nullptr; hash = hash->prev) { + auto index = hashedId; + do { + index &= hash->capacity - 1u; + probedKey = id; + if (hash->entries[index].key.compare_exchange_strong(probedKey, details::invalid_thread_id2, std::memory_order_seq_cst, std::memory_order_relaxed)) { + break; + } + ++index; + } while (probedKey != details::invalid_thread_id); // Can happen if the hash has changed but we weren't put back in it yet, or if we weren't added to this hash in the first place + } + + // Mark the queue as being recyclable + producer->inactive.store(true, std::memory_order_release); + } + + static void implicit_producer_thread_exited_callback(void* userData) + { + auto producer = static_cast(userData); + auto queue = producer->parent; + queue->implicit_producer_thread_exited(producer); + } +#endif + + ////////////////////////////////// + // Utility functions + ////////////////////////////////// + + template + static inline void* aligned_malloc(size_t size) + { + MOODYCAMEL_CONSTEXPR_IF (std::alignment_of::value <= std::alignment_of::value) + return (Traits::malloc)(size); + else { + size_t alignment = std::alignment_of::value; + void* raw = (Traits::malloc)(size + alignment - 1 + sizeof(void*)); + if (!raw) + return nullptr; + char* ptr = details::align_for(reinterpret_cast(raw) + sizeof(void*)); + *(reinterpret_cast(ptr) - 1) = raw; + return ptr; + } + } + + template + static inline void aligned_free(void* ptr) + { + MOODYCAMEL_CONSTEXPR_IF (std::alignment_of::value <= std::alignment_of::value) + return (Traits::free)(ptr); + else + (Traits::free)(ptr ? *(reinterpret_cast(ptr) - 1) : nullptr); + } + + template + static inline U* create_array(size_t count) + { + assert(count > 0); + U* p = static_cast(aligned_malloc(sizeof(U) * count)); + if (p == nullptr) + return nullptr; + + for (size_t i = 0; i != count; ++i) + new (p + i) U(); + return p; + } + + template + static inline void destroy_array(U* p, size_t count) + { + if (p != nullptr) { + assert(count > 0); + for (size_t i = count; i != 0; ) + (p + --i)->~U(); + } + aligned_free(p); + } + + template + static inline U* create() + { + void* p = aligned_malloc(sizeof(U)); + return p != nullptr ? new (p) U : nullptr; + } + + template + static inline U* create(A1&& a1) + { + void* p = aligned_malloc(sizeof(U)); + return p != nullptr ? new (p) U(std::forward(a1)) : nullptr; + } + + template + static inline void destroy(U* p) + { + if (p != nullptr) + p->~U(); + aligned_free(p); + } + +private: + std::atomic producerListTail; + std::atomic producerCount; + + std::atomic initialBlockPoolIndex; + Block* initialBlockPool; + size_t initialBlockPoolSize; + +#ifndef MCDBGQ_USEDEBUGFREELIST + FreeList freeList; +#else + debug::DebugFreeList freeList; +#endif + + std::atomic implicitProducerHash; + std::atomic implicitProducerHashCount; // Number of slots logically used + ImplicitProducerHash initialImplicitProducerHash; + std::array initialImplicitProducerHashEntries; + std::atomic_flag implicitProducerHashResizeInProgress; + + std::atomic nextExplicitConsumerId; + std::atomic globalExplicitConsumerOffset; + +#ifdef MCDBGQ_NOLOCKFREE_IMPLICITPRODHASH + debug::DebugMutex implicitProdMutex; +#endif + +#ifdef MOODYCAMEL_QUEUE_INTERNAL_DEBUG + std::atomic explicitProducers; + std::atomic implicitProducers; +#endif +}; + + +template +ProducerToken::ProducerToken(ConcurrentQueue& queue) + : producer(queue.recycle_or_create_producer(true)) +{ + if (producer != nullptr) { + producer->token = this; + } +} + +template +ProducerToken::ProducerToken(BlockingConcurrentQueue& queue) + : producer(reinterpret_cast*>(&queue)->recycle_or_create_producer(true)) +{ + if (producer != nullptr) { + producer->token = this; + } +} + +template +ConsumerToken::ConsumerToken(ConcurrentQueue& queue) + : itemsConsumedFromCurrent(0), currentProducer(nullptr), desiredProducer(nullptr) +{ + initialOffset = queue.nextExplicitConsumerId.fetch_add(1, std::memory_order_release); + lastKnownGlobalOffset = static_cast(-1); +} + +template +ConsumerToken::ConsumerToken(BlockingConcurrentQueue& queue) + : itemsConsumedFromCurrent(0), currentProducer(nullptr), desiredProducer(nullptr) +{ + initialOffset = reinterpret_cast*>(&queue)->nextExplicitConsumerId.fetch_add(1, std::memory_order_release); + lastKnownGlobalOffset = static_cast(-1); +} + +template +inline void swap(ConcurrentQueue& a, ConcurrentQueue& b) MOODYCAMEL_NOEXCEPT +{ + a.swap(b); +} + +inline void swap(ProducerToken& a, ProducerToken& b) MOODYCAMEL_NOEXCEPT +{ + a.swap(b); +} + +inline void swap(ConsumerToken& a, ConsumerToken& b) MOODYCAMEL_NOEXCEPT +{ + a.swap(b); +} + +template +inline void swap(typename ConcurrentQueue::ImplicitProducerKVP& a, typename ConcurrentQueue::ImplicitProducerKVP& b) MOODYCAMEL_NOEXCEPT +{ + a.swap(b); +} + +} + +#if defined(_MSC_VER) && (!defined(_HAS_CXX17) || !_HAS_CXX17) +#pragma warning(pop) +#endif + +#if defined(__GNUC__) && !defined(__INTEL_COMPILER) +#pragma GCC diagnostic pop +#endif diff --git a/Sources/TetraConcurrentQueueShim/include/TetraConcurrentQueueShim.h b/Sources/TetraConcurrentQueueShim/include/TetraConcurrentQueueShim.h new file mode 100644 index 0000000..6078821 --- /dev/null +++ b/Sources/TetraConcurrentQueueShim/include/TetraConcurrentQueueShim.h @@ -0,0 +1,13 @@ +// +// Header.h +// Tetra +// +// Created by 박병관 on 1/20/25. +// + +#ifndef TetraConcurrentQueue_h +#define TetraConcurrentQueue_h + +#include "sim.h" + +#endif /* Header_h */ diff --git a/Sources/TetraConcurrentQueueShim/include/sim.h b/Sources/TetraConcurrentQueueShim/include/sim.h new file mode 100644 index 0000000..04629de --- /dev/null +++ b/Sources/TetraConcurrentQueueShim/include/sim.h @@ -0,0 +1,62 @@ +// +// Header.h +// Tetra +// +// Created by 박병관 on 1/20/25. +// + +#ifndef ConcurrentQueue_shim_h +#define ConcurrentQueue_shim_h +//#include +//#include "concurrentqueue.hpp" +#include +#include + + +CF_ASSUME_NONNULL_BEGIN + +typedef struct { + void(*perform)(CFTypeRef state, CFArrayRef job); + void(* _Nullable schedule)(CFTypeRef state, CFRunLoopRef rl, CFRunLoopMode mode); + void(* _Nullable cancel)(CFTypeRef state, CFRunLoopRef rl, CFRunLoopMode mode); +} TetraContextData; + +//class +//SWIFT_NONCOPYABLE +//SWIFT_SHARED_REFERENCE(retainSharedObject, releaseSharedObject) +//MyBookQueue { +//public: +// static MyBookQueue* create(); +// +// bool enqueue(CFTypeRef ref); +// CF_RETURNS_NOT_RETAINED _Nullable CFTypeRef try_dequeue(); +//private: +// MyBookQueue(); +// CFDataRef data; +//// class MyActualType; +//// std::unique_ptr impl; +//}; + +CF_EXTERN_C_BEGIN + +bool enqueue_ref_concurrent_queue(void* queue, CFTypeRef ref); + +CF_RETURNS_RETAINED CFRunLoopSourceRef create_tetra_runLoop_executor( + CFTypeRef initialState, + const TetraContextData *tetraContext +); + +CF_RETURNS_RETAINED CFDictionaryRef copy_tetra_runLoop_registry(CFRunLoopSourceRef source); + +bool tetra_enqueue_and_signal(CFRunLoopSourceRef source, CFTypeRef ref); + +CF_EXTERN_C_END + +//void retainSharedObject(MyBookQueue* ref); +//void releaseSharedObject(MyBookQueue* _Nullable ref); + +CF_ASSUME_NONNULL_END + + + +#endif /* ConcurrentQueue_shim_h */ diff --git a/Sources/TetraConcurrentQueueShim/sim.cpp b/Sources/TetraConcurrentQueueShim/sim.cpp new file mode 100644 index 0000000..2a8c6f6 --- /dev/null +++ b/Sources/TetraConcurrentQueueShim/sim.cpp @@ -0,0 +1,318 @@ +// +// sim.m +// Tetra +// +// Created by 박병관 on 1/20/25. +// + +#define MOODYCAMEL_NO_THREAD_LOCAL +#include "concurrentqueue.h" +#include "sim.h" +#include +#if __APPLE__ +#include +#endif +//#undef __APPLE__ + +struct CFQueueTrait: moodycamel::ConcurrentQueueDefaultTraits { + CF_INLINE void* malloc(size_t size) { + return CFAllocatorAllocate(kCFAllocatorDefault, size, 0); + } + + CF_INLINE void free(void *ptr) { + return CFAllocatorDeallocate(kCFAllocatorDefault, ptr); + } +}; + + + +typedef std::shared_ptr CFCppRef; +typedef moodycamel::ConcurrentQueue MyConcurrentQueue; + + +bool enqueue_ref_concurrent_queue(void* queue, CFTypeRef ref) { + auto q = reinterpret_cast(queue); + auto ptr = CFCppRef(CFRetain(ref), CFRelease); + + return q->enqueue(std::move(ptr)); +} + +CFTypeRef dequeue_ref_concurrent_queue(void* queue) { + auto q = reinterpret_cast(queue); + CFCppRef ptr; + + if (q->try_dequeue(ptr)) { + return ptr.get(); + } + return nullptr; +} + +CF_INLINE CFAllocatorContext create_defaultContext(void) { + CFAllocatorContext context = { + 0, + (void*)(kCFAllocatorDefault), + [](CFTypeRef ref) { return ref ? CFRetain(ref) : ref; }, + [](CFTypeRef ref) { ref ? CFRelease(ref) : void(); }, + [](CFTypeRef ref) { return ref ? CFCopyDescription(ref) : nullptr; }, + [](CFIndex allocSize, CFOptionFlags hint, void *info) { return CFAllocatorAllocate(kCFAllocatorDefault, allocSize, hint); }, + [](void *ptr, CFIndex newsize, CFOptionFlags hint, void *info) { return CFAllocatorReallocate(kCFAllocatorDefault, ptr, newsize, hint); }, + [](void *ptr, void *info) { CFAllocatorDeallocate(kCFAllocatorDefault, ptr); }, + [](CFIndex size, CFOptionFlags hint, void *info) { return CFAllocatorGetPreferredSizeForSize(kCFAllocatorDefault, size, hint); } + }; + return context; +} + +CF_INLINE CFDataRef create_wrapped_queue() { + constexpr std::size_t queueSize = sizeof(MyConcurrentQueue); + auto queueBuffer = CFQueueTrait::malloc(queueSize); + CFAllocatorContext context = {}; + context.deallocate = [](void *ptr, void *info) { + auto q = reinterpret_cast(ptr); + const auto count = q->size_approx(); + assert(count == 0); + q->~ConcurrentQueue(); + CFQueueTrait::free(q); + }; + auto queue = new (queueBuffer) MyConcurrentQueue(); + CFAllocatorRef deallocator = CFAllocatorCreate(kCFAllocatorDefault, &context); + CFDataRef queueWrapper = CFDataCreateWithBytesNoCopy(kCFAllocatorDefault, reinterpret_cast(queueBuffer), queueSize, deallocator); + CFRelease(deallocator); + return queueWrapper; +} + +CF_INLINE CFDataRef create_wrapped_token(MyConcurrentQueue& queue) { + constexpr std::size_t tokenSize = sizeof(moodycamel::ConsumerToken); + auto tokenBuffer = CFQueueTrait::malloc(tokenSize); + CFAllocatorContext context = {}; + auto token = new (tokenBuffer) moodycamel::ConsumerToken(queue); + context.deallocate = [](void *ptr, void *info) { + auto t = reinterpret_cast(ptr); + t->~ConsumerToken(); + CFQueueTrait::free(t); + }; + CFAllocatorRef deallocator = CFAllocatorCreate(kCFAllocatorDefault, &context); + CFDataRef tokenWrapper = CFDataCreateWithBytesNoCopy(kCFAllocatorDefault, reinterpret_cast(tokenBuffer), tokenSize, deallocator); + CFRelease(deallocator); + return tokenWrapper; +} + + + +CFRunLoopSourceRef create_tetra_runLoop_executor( + CFTypeRef initialState, + const TetraContextData *tetraContext +) { + CFDataRef queueWrapper = create_wrapped_queue(); + CFDataRef tokenWrapper = create_wrapped_token( + *reinterpret_cast(const_cast(CFDataGetBytePtr(queueWrapper))) + ); + CFDataRef contextStorage = CFDataCreate(kCFAllocatorDefault, (UInt8 *)tetraContext, sizeof(TetraContextData)); + auto registryContext = create_defaultContext(); + registryContext.retain = [](CFTypeRef ref) -> CFTypeRef { +#if __APPLE__ + constexpr size_t size = sizeof(os_unfair_lock_s); +#else + constexpr size_t size = sizeof(std::mutex); +#endif + auto buffer = CFAllocatorAllocate(kCFAllocatorDefault, size, 0); + +#if __APPLE__ + os_unfair_lock_t mutex = new (buffer) os_unfair_lock_s(OS_UNFAIR_LOCK_INIT); +#else + auto mutex = new (buffer) std::mutex; +#endif + + return buffer; + }; + registryContext.copyDescription = nullptr; + registryContext.release = [](CFTypeRef ref) { +#if __APPLE__ + auto lock = reinterpret_cast(const_cast(ref)); + lock->~os_unfair_lock_s(); +#else + auto mutex = reinterpret_cast(const_cast(ref)); + mutex->~mutex(); +#endif + CFAllocatorDeallocate(kCFAllocatorDefault, const_cast(ref)); + }; + CFAllocatorRef registryDeallocator = CFAllocatorCreate(kCFAllocatorDefault, ®istryContext); + CFMutableDictionaryRef runLoopRegistry = CFDictionaryCreateMutable(registryDeallocator, 0, &kCFTypeDictionaryKeyCallBacks, &kCFTypeDictionaryValueCallBacks); + CFArrayRef array = CFArrayCreate(kCFAllocatorDefault, (CFTypeRef []){queueWrapper, tokenWrapper, initialState, contextStorage, runLoopRegistry}, 5, &kCFTypeArrayCallBacks); + CFRelease(queueWrapper); + CFRelease(tokenWrapper); + CFRelease(contextStorage); + CFRelease(runLoopRegistry); + CFRelease(registryDeallocator); +// CFRelease(initialState); + /** + [ + queue, + consumerToken, + userDefinedState, + contextCallbackStorage + ] + **/ + CFRunLoopSourceContext soureContext = { + 0, + (void*)array, + CFRetain, + CFRelease, + [](CFTypeRef ref) -> CFStringRef { + CFArrayRef array = reinterpret_cast(ref); + CFTypeRef buffer[] = { + CFStringCreateWithCString(kCFAllocatorDefault, "moody::camel::ConcurrentQueue", kCFStringEncodingUTF8), + CFStringCreateWithCString(kCFAllocatorDefault, "moody::camel::ConsumerToken", kCFStringEncodingUTF8), + CFArrayGetValueAtIndex(array, 2), + CFStringCreateWithCString(kCFAllocatorDefault, "TetraContextStorage", kCFStringEncodingUTF8), + CFStringCreateWithCString(kCFAllocatorDefault, "RunLoopRegistry", kCFStringEncodingUTF8), + }; + CFArrayRef temp = CFArrayCreate(kCFAllocatorDefault, buffer, 5, &kCFTypeArrayCallBacks); + CFStringRef description = CFCopyDescription(temp); + CFRelease(temp); + return description; + }, + CFEqual, + CFHash, + [](void * info, CFRunLoopRef runLoop, CFRunLoopMode mode) { + //schedule + auto array = static_cast(info); + CFTypeRef state = CFArrayGetValueAtIndex(array, 2); + CFDataRef context = (CFDataRef) CFArrayGetValueAtIndex(array, 3); + auto tetraContext = (TetraContextData *)CFDataGetBytePtr(context); + CFRunLoopWakeUp(runLoop); + { + CFMutableDictionaryRef registry = (CFMutableDictionaryRef)CFArrayGetValueAtIndex(array, 4); + CFAllocatorContext allocContext = {}; + CFAllocatorGetContext(CFGetAllocator(registry),&allocContext); +#if __APPLE__ + os_unfair_lock_lock((os_unfair_lock_t)allocContext.info); +#else + std::lock_guard lock(*(std::mutex *)allocContext.info); +#endif + CFMutableSetRef set = (CFMutableSetRef)CFDictionaryGetValue(registry, runLoop); + if (!set) { + set = CFSetCreateMutable(kCFAllocatorDefault, 0, &kCFTypeSetCallBacks); + CFDictionarySetValue(registry, runLoop, set); + CFRelease(set); + } +#if __APPLE__ + os_unfair_lock_unlock((os_unfair_lock_t)allocContext.info); +#endif + CFSetAddValue(set, mode); + } + if (tetraContext->schedule) { + tetraContext->schedule(state, runLoop, mode); + } + }, + [](void * info, CFRunLoopRef runLoop, CFRunLoopMode mode) { + // cancel + auto array = static_cast(info); + CFTypeRef state = CFArrayGetValueAtIndex(array, 2); + CFTypeRef context = CFArrayGetValueAtIndex(array, 3); + auto tetraContext = (TetraContextData *)CFDataGetBytePtr((CFDataRef)context); + CFRunLoopWakeUp(runLoop); + { + CFMutableDictionaryRef registry = (CFMutableDictionaryRef)CFArrayGetValueAtIndex(array, 4); + CFAllocatorContext allocContext = {}; + CFAllocatorGetContext(CFGetAllocator(registry),&allocContext); +#if __APPLE__ + os_unfair_lock_lock((os_unfair_lock_t)allocContext.info); +#else + std::lock_guard lock(*(std::mutex *)allocContext.info); +#endif + CFMutableSetRef set = (CFMutableSetRef)CFDictionaryGetValue(registry, runLoop); + + CFSetRemoveValue(set, mode); + + if (CFSetGetCount(set) == 0) { + CFDictionaryRemoveValue(registry, runLoop); + } +#if __APPLE__ + os_unfair_lock_unlock((os_unfair_lock_t)allocContext.info); +#endif + } + if (tetraContext->cancel) { + tetraContext->cancel(state, runLoop, mode); + } + + }, + [](void *info) { + auto array = static_cast(info); + auto queue = reinterpret_cast((void *)CFDataGetBytePtr((CFDataRef)CFArrayGetValueAtIndex(array, 0))); + auto token = reinterpret_cast((void *)CFDataGetBytePtr((CFDataRef)CFArrayGetValueAtIndex(array, 1))); + CFTypeRef state = CFArrayGetValueAtIndex(array, 2); + CFTypeRef context = CFArrayGetValueAtIndex(array, 3); + auto tetraContext = (TetraContextData *)CFDataGetBytePtr((CFDataRef)context); + CFMutableArrayRef dequeue = CFArrayCreateMutable(kCFAllocatorDefault, 0, &kCFTypeArrayCallBacks); + constexpr size_t buffer_size = 10; + CFCppRef result[buffer_size]; + size_t size = 0; + while ((size = queue->try_dequeue_bulk(*token, result, buffer_size)) > 0) { + for (int i = 0; i < size; i++) { + CFCppRef ref = std::move(result[i]); + CFArrayAppendValue(dequeue, ref.get()); + } + } + tetraContext->perform(state, dequeue); + CFRelease(dequeue); + } + }; + + CFRunLoopSourceRef source = CFRunLoopSourceCreate(kCFAllocatorDefault, 0, &soureContext); + CFRelease(array); + return source; +} + +CFDictionaryRef copy_tetra_runLoop_registry(CFRunLoopSourceRef source) { + CFMutableDictionaryRef registry_source; + CFDictionaryRef registry; + { + CFRunLoopSourceContext context = {}; + CFRunLoopSourceGetContext(source, &context); + CFArrayRef array = reinterpret_cast(context.info); + assert(CFGetTypeID(array) == CFArrayGetTypeID()); + CFTypeRef ref = CFArrayGetValueAtIndex(array, 4); +// assert(CFGetTypeID(registry_source) == CFDictionaryGetTypeID()); + registry_source = reinterpret_cast(const_cast(ref)); + } + { + CFAllocatorContext allocContext = {}; + CFAllocatorGetContext(CFGetAllocator(registry_source),&allocContext); +#if __APPLE__ + os_unfair_lock_lock((os_unfair_lock_t)allocContext.info); +#else + std::lock_guard lock(*(std::mutex *)allocContext.info); +#endif + registry = CFDictionaryCreateCopy(kCFAllocatorDefault, registry_source); +#if __APPLE__ + os_unfair_lock_unlock((os_unfair_lock_t)allocContext.info); +#endif + } + return registry; +} + +bool tetra_enqueue_and_signal(CFRunLoopSourceRef source, CFTypeRef ref) { + CFArrayRef array; + MyConcurrentQueue* queue; + { + CFRunLoopSourceContext context = {}; + CFRunLoopSourceGetContext(source, &context); + array = static_cast(context.info); + } + { + CFDataRef queueWrapper = (CFDataRef)CFArrayGetValueAtIndex(array, 0); + queue = reinterpret_cast(const_cast(CFDataGetBytePtr(queueWrapper))); + } + auto ptr = CFCppRef(CFRetain(ref), CFRelease); + const bool success = queue->enqueue(std::move(ptr)); + if (!success) { + return false; + } + CFRunLoopSourceSignal(source); + CFDictionaryRef registry = copy_tetra_runLoop_registry(source); + CFDictionaryApplyFunction(registry, [](CFTypeRef key, CFTypeRef value, void * info) { + CFRunLoopWakeUp((CFRunLoopRef) key); + }, nullptr); + CFRelease(registry); + return true; +} diff --git a/Sources/TetraRunLoopConcurrency/TetraRunLoopExecutor.swift b/Sources/TetraRunLoopConcurrency/TetraRunLoopExecutor.swift new file mode 100644 index 0000000..65b1699 --- /dev/null +++ b/Sources/TetraRunLoopConcurrency/TetraRunLoopExecutor.swift @@ -0,0 +1,198 @@ +// +// File.swift +// Tetra +// +// Created by 박병관 on 1/27/25. +// + +import Foundation +@preconcurrency import CoreFoundation +@_implementationOnly private import TetraConcurrentQueueShim +//private import CriticalSection +//private import Atomics + +struct StateStorage { + +// fileprivate let lock: some UnfairLockProtocol = createUnfairLock() +// var registry = [CFRunLoop: Set]() + var serialRef: UnownedSerialExecutor +// fileprivate let reference = ManagedAtomicLazyReference() + + var taskRef:AnyObject? = nil + +} + + + +final class RunLoopStorageBufferHolder {} + + + +final package class TetraRunLoopExecutor: NSObject { + + + let source: CFRunLoopSource + + deinit { + CFRunLoopSourceInvalidate(source) +// runLoop.perform {} + + } + + @objc + package override convenience init() { + self.init(name: nil) + } + + @nonobjc + package init( + name: String? + ) { + + + var bufferPtr = ManagedBufferPointer(bufferClass: RunLoopStorageBufferHolder.self, minimumCapacity: 0) { buffer, capacity in + .init(serialRef: MainActor.sharedUnownedExecutor) + } + + self.source = withUnsafePointer(to: TetraContextData( + perform: tetra_runLoop_drainSource, + schedule: tetra_runLoop_schedule_cb, + cancel: tetra_runLoop_cancel_cb + )) { + create_tetra_runLoop_executor(bufferPtr.buffer, $0) + } + super.init() + // override serialExecutor to the correct value + if #available(macOS 15.0, iOS 18.0, watchOS 11.0, tvOS 18.0, visionOS 2.0, *) { + bufferPtr.header.taskRef = ManagedBufferPointer(bufferClass: RunLoopStorageBufferHolder.self, minimumCapacity: 0, makingHeaderWith: { buffer, capacity in + return asUnownedTaskExecutor() + }).buffer + } + bufferPtr.header.serialRef = asUnownedSerialExecutor() + let thread = Thread(block: runLoopThreadRun) + thread.threadDictionary["source"] = self.source + if let name = name { + thread.name = name + } + thread.qualityOfService = .default + thread.start() + + } + +} + + + + +extension TetraRunLoopExecutor : SerialExecutor {} + +extension TetraRunLoopExecutor: TaskExecutor {} + +package extension TetraRunLoopExecutor { + + @objc + nonisolated func copyCurrentRegistry() -> [CFRunLoop:Set] { + return copy_tetra_runLoop_registry(source) as! [CFRunLoop:Set] + } + + @available(macOS 14.0, iOS 17.0, watchOS 10.0, tvOS 17.0, *) + nonisolated func enqueue(_ job: consuming ExecutorJob) { + enqueue(UnownedJob(job)) + } + + nonisolated func enqueue(_ job: UnownedJob) { + tetra_enqueue_and_signal(source, job as AnyObject) + } + + nonisolated func asUnownedSerialExecutor() -> UnownedSerialExecutor { + if #available(macOS 14.0, iOS 17.0, watchOS 10.0, tvOS 17.0, *) { + return .init(complexEquality: self) + } + return .init(ordinary: self) + } + + nonisolated func checkIsolated() { +// let currentMode = RunLoop.current.currentMode.flatMap{ $0.rawValue +// as CFString }.flatMap{ CFRunLoopMode($0)} + precondition( +// currentMode != nil && + CFRunLoopContainsSource(CFRunLoopGetCurrent(), source, CFRunLoopMode.defaultMode), + "TetraRunLoopExecutor must be called from the RunLoop that it was created on." + ) + } + + + +} + +extension TetraRunLoopExecutor { + + nonisolated package func register(_ runLoop: CFRunLoop) { + CFRunLoopAddSource(runLoop, source, .defaultMode) + } + + nonisolated package func register(_ runLoop:RunLoop) { + CFRunLoopAddSource(runLoop.getCFRunLoop(), source, .defaultMode) + } + + @available(swift, obsoleted: 1.0) + @objc + package func schedule( _ block: @convention(block) () -> Void) { + block() + } + +} + + +fileprivate nonisolated func runLoopThreadRun() { + let source = Thread.current.threadDictionary["source"] as! CFRunLoopSource + Thread.current.threadDictionary["source"] = nil + CFRunLoopAddSource(CFRunLoopGetCurrent(), source, CFRunLoopMode.defaultMode) + while CFRunLoopSourceIsValid(source), RunLoop.current.run(mode: .default, before: .distantFuture) { + + } + +} + +private func tetra_runLoop_drainSource(_ stateRef:CFTypeRef, _ jobRef:CFArray) { + let buffPtr = ManagedBufferPointer(unsafeBufferObject: stateRef) + let serials = buffPtr.header.serialRef + let jobArray = jobRef as! [UnownedJob] + if #available(macOS 15.0, iOS 18.0, watchOS 11.0, tvOS 18.0, visionOS 2.0, *), let taskRef = buffPtr.header.taskRef { + let taskExecutor = ManagedBufferPointer(unsafeBufferObject: taskRef).header + jobArray.forEach{ + $0.runSynchronously(isolatedTo: serials, taskExecutor: taskExecutor) + } + } else { + jobArray.forEach{ + $0.runSynchronously(on: serials) + } + } +} + +private func tetra_runLoop_schedule_cb(_ state:CFTypeRef, _ runLoop:CFRunLoop, _ mode:CFRunLoopMode) { +// let buffPtr = ManagedBufferPointer(unsafeBufferObject: state) +// let lock = buffPtr.header.lock +// let _ = buffPtr.header.reference.storeIfNilThenLoad(runLoop) +// lock.withLockUnchecked { +// buffPtr.withUnsafeMutablePointerToHeader{ +// if ($0.pointee.registry[runLoop] == nil) { +// $0.pointee.registry[runLoop] = [mode.rawValue as String] +// } else { +// $0.pointee.registry[runLoop]?.insert(mode.rawValue as String) +// } +// } +// } +} + +private func tetra_runLoop_cancel_cb(_ state:CFTypeRef, _ runLoop:CFRunLoop, _ mode:CFRunLoopMode) { +// let buffPtr = ManagedBufferPointer(unsafeBufferObject: state) +// let lock = buffPtr.header.lock +// lock.withLockUnchecked { +// buffPtr.withUnsafeMutablePointerToHeader{ +// if ($0.pointee.registry[runLoop] != nil) { +// $0.pointee.registry[runLoop]?.remove(mode.rawValue as String) +// } +// } +// } +} From 62ef0bb385563a77fa0a7b46bbd935364a01c0d9 Mon Sep 17 00:00:00 2001 From: park-byeong-gwan Date: Tue, 28 Jan 2025 23:33:10 +0900 Subject: [PATCH 58/63] migrate some test to Swift-Testing --- .../NSManagedObjectContextTests.swift | 81 ++++++++++--------- .../NotificationSequenceTests.swift | 12 ++- Tests/TetraTests/TetraTests.swift | 26 ++---- 3 files changed, 56 insertions(+), 63 deletions(-) diff --git a/Tests/TetraTests/NSManagedObjectContextTests.swift b/Tests/TetraTests/NSManagedObjectContextTests.swift index 7c0504a..0420029 100644 --- a/Tests/TetraTests/NSManagedObjectContextTests.swift +++ b/Tests/TetraTests/NSManagedObjectContextTests.swift @@ -5,42 +5,44 @@ // Created by 박병관 on 6/5/24. // -import XCTest +import Testing #if canImport(CoreData) import CoreData @testable import Tetra internal import NamespaceExtension -final class NSManagedObjectContextTests: XCTestCase { +@Suite +struct NSManagedObjectContextTests { + @Test func testBlocking() throws { let context = NSManagedObjectContext(concurrencyType: .privateQueueConcurrencyType) let uuid = UUID() let returnValue = context.tetra._performAndWait { return uuid } - XCTAssertEqual(uuid, returnValue) + #expect(uuid == returnValue) let result = Result { try context.tetra._performAndWait { throw CancellationError() } } - XCTAssertThrowsError(try result.get()) { - XCTAssertTrue($0 is CancellationError, "\($0) is not \(CancellationError.self)") - - } + #expect(throws: CancellationError.self, performing: { + try result.get() + }) } + @Test func testMainAsync() async throws { let context = NSManagedObjectContext(concurrencyType: .mainQueueConcurrencyType) let uuid = UUID() let returnValue = await context.tetra._performEnqueue { - XCTAssertTrue(Thread.isMainThread) + #expect(Thread.isMainThread) return uuid } - XCTAssertEqual(uuid, returnValue) + #expect(uuid == returnValue) let result:Result do { try await context.tetra._performEnqueue{ @@ -50,19 +52,20 @@ final class NSManagedObjectContextTests: XCTestCase { } catch { result = .failure(error) } - XCTAssertThrowsError(try result.get()) { - XCTAssertTrue($0 is CancellationError, "\($0) is not \(CancellationError.self)") - } + #expect(throws: CancellationError.self, performing: { + try result.get() + }) } + @Test func testBackgroundAsync() async throws { let context = NSManagedObjectContext(concurrencyType: .privateQueueConcurrencyType) let uuid = UUID() let returnValue = await context.tetra._performEnqueue { - XCTAssertFalse(Thread.isMainThread) + #expect(!Thread.isMainThread) return uuid } - XCTAssertEqual(uuid, returnValue) + #expect(uuid == returnValue) let result:Result do { try await context.tetra._performEnqueue{ @@ -72,41 +75,39 @@ final class NSManagedObjectContextTests: XCTestCase { } catch { result = .failure(error) } - XCTAssertThrowsError(try result.get()) { - XCTAssertTrue($0 is CancellationError, "\($0) is not \(CancellationError.self)") + #expect(throws: CancellationError.self) { + try result.get() } } - - func testImmediate() throws { + @Test + func testImmediate() async throws { let context = NSManagedObjectContext(concurrencyType: .privateQueueConcurrencyType) - let completion = expectation(description: "completion") - context.perform { - defer { completion.fulfill() } - let uuid = UUID() - let expectValue = context.tetra._performImmediate { - return uuid - } - XCTAssertEqual(expectValue, .success(uuid)) - let expectThrow = Result { - try context.tetra._performImmediate { - throw CancellationError() + let _:Void = await withUnsafeContinuation{ continuation in + context.perform { + defer { + continuation.resume() } - } - func checkError() { - XCTAssertThrowsError(try expectThrow.get()) { - XCTAssertTrue($0 is CancellationError, "\($0) is not \(CancellationError.self)") + let uuid = UUID() + let expectValue = context.tetra._performImmediate { + return uuid + } + #expect(expectValue == .success(uuid)) + let expectThrow = Result { + try context.tetra._performImmediate { + throw CancellationError() + } + } + #expect(throws: CancellationError.self) { + try expectThrow.get() } } - checkError() - } - - let expectNil = context.tetra._performImmediate { + let expectNil = context.tetra._performImmediate { + + } + #expect(expectNil == nil) } - XCTAssertNil(expectNil) - - wait(for: [completion], timeout: 0.1) } diff --git a/Tests/TetraTests/NotificationSequenceTests.swift b/Tests/TetraTests/NotificationSequenceTests.swift index d4c6253..78a02db 100644 --- a/Tests/TetraTests/NotificationSequenceTests.swift +++ b/Tests/TetraTests/NotificationSequenceTests.swift @@ -5,11 +5,14 @@ // Created by pbk on 2023/01/27. // -import XCTest +import Testing +import Foundation @testable import Tetra -final class NotificationSequenceTests: XCTestCase { +@Suite +struct NotificationSequenceTests { + @Test func testNotificationSequence() async throws { let name = Notification.Name(UUID().uuidString) let object = NSObject() @@ -34,9 +37,10 @@ final class NotificationSequenceTests: XCTestCase { task.cancel() NotificationCenter.default.post(name: name, object: object, userInfo: ["":""]) let count = await task.value - XCTAssertEqual(count, 2) + #expect(count == 2) } + @Test func testAlreadyCancelled() async throws { let name = Notification.Name(UUID().uuidString) let object = NSObject() @@ -63,7 +67,7 @@ final class NotificationSequenceTests: XCTestCase { } task.cancel() let count = await task.value - XCTAssertEqual(count, 0) + #expect(count == 0) } } diff --git a/Tests/TetraTests/TetraTests.swift b/Tests/TetraTests/TetraTests.swift index 2eb2a41..c9302a7 100644 --- a/Tests/TetraTests/TetraTests.swift +++ b/Tests/TetraTests/TetraTests.swift @@ -5,30 +5,16 @@ // Created by iquest1127 on 2022/12/19. // -import XCTest +import Testing import os @testable import Tetra import Combine import CriticalSection -final class TetraTests: XCTestCase { - - override func setUp() async throws { - // Put setup code here. This method is called before the invocation of each test method in the class. - } - - override func tearDown() async throws { - // Put teardown code here. This method is called after the invocation of each test method in the class. - } - - func testExample() throws { - // This is an example of a functional test case. - // Use XCTAssert and related functions to verify your tests produce the correct results. - // Any test you write for XCTest can be annotated as throws and async. - // Mark your test throws to produce an unexpected failure when your test encounters an uncaught error. - // Mark your test async to allow awaiting for asynchronous code to complete. Check the results with assertions afterwards. - } +@Suite +struct TetraTests { + @Test func testUnfairLockPrecondition() throws { if #available(iOS 16.0, tvOS 16.0, macCatalyst 16.0, macOS 13.0, watchOS 9.0, *) { let lock = OSAllocatedUnfairLock() @@ -45,5 +31,7 @@ final class TetraTests: XCTestCase { lock.precondition(.notOwner) } } - + } + + From 8f68ec4f9af1fb070b01aced0bba6212e2f50971 Mon Sep 17 00:00:00 2001 From: park-byeong-gwan Date: Thu, 30 Jan 2025 15:47:48 +0900 Subject: [PATCH 59/63] update RunLoopExecutor --- .../TetraConcurrentQueueShim/include/sim.h | 3 + Sources/TetraConcurrentQueueShim/sim.cpp | 312 +++++++----------- .../TetraRunLoopExecutor.swift | 27 +- 3 files changed, 146 insertions(+), 196 deletions(-) diff --git a/Sources/TetraConcurrentQueueShim/include/sim.h b/Sources/TetraConcurrentQueueShim/include/sim.h index 04629de..b393865 100644 --- a/Sources/TetraConcurrentQueueShim/include/sim.h +++ b/Sources/TetraConcurrentQueueShim/include/sim.h @@ -50,6 +50,9 @@ CF_RETURNS_RETAINED CFDictionaryRef copy_tetra_runLoop_registry(CFRunLoopSourceR bool tetra_enqueue_and_signal(CFRunLoopSourceRef source, CFTypeRef ref); +CF_RETURNS_NOT_RETAINED +CFTypeRef tetra_get_stateInfo(CFRunLoopSourceRef source); + CF_EXTERN_C_END //void retainSharedObject(MyBookQueue* ref); diff --git a/Sources/TetraConcurrentQueueShim/sim.cpp b/Sources/TetraConcurrentQueueShim/sim.cpp index 2a8c6f6..f4bf0d0 100644 --- a/Sources/TetraConcurrentQueueShim/sim.cpp +++ b/Sources/TetraConcurrentQueueShim/sim.cpp @@ -12,7 +12,6 @@ #if __APPLE__ #include #endif -//#undef __APPLE__ struct CFQueueTrait: moodycamel::ConcurrentQueueDefaultTraits { CF_INLINE void* malloc(size_t size) { @@ -25,169 +24,78 @@ struct CFQueueTrait: moodycamel::ConcurrentQueueDefaultTraits { }; - +//#undef __APPLE__ typedef std::shared_ptr CFCppRef; typedef moodycamel::ConcurrentQueue MyConcurrentQueue; - -bool enqueue_ref_concurrent_queue(void* queue, CFTypeRef ref) { - auto q = reinterpret_cast(queue); - auto ptr = CFCppRef(CFRetain(ref), CFRelease); - - return q->enqueue(std::move(ptr)); -} - -CFTypeRef dequeue_ref_concurrent_queue(void* queue) { - auto q = reinterpret_cast(queue); - CFCppRef ptr; +typedef struct { + MyConcurrentQueue queue; + moodycamel::ConsumerToken token; + TetraContextData context; + CFTypeRef state; + CFMutableDictionaryRef runLoopRegistry; +#if __APPLE__ + os_unfair_lock_s lock; +#else + std::mutex* lock; +#endif - if (q->try_dequeue(ptr)) { - return ptr.get(); - } - return nullptr; -} - -CF_INLINE CFAllocatorContext create_defaultContext(void) { - CFAllocatorContext context = { - 0, - (void*)(kCFAllocatorDefault), - [](CFTypeRef ref) { return ref ? CFRetain(ref) : ref; }, - [](CFTypeRef ref) { ref ? CFRelease(ref) : void(); }, - [](CFTypeRef ref) { return ref ? CFCopyDescription(ref) : nullptr; }, - [](CFIndex allocSize, CFOptionFlags hint, void *info) { return CFAllocatorAllocate(kCFAllocatorDefault, allocSize, hint); }, - [](void *ptr, CFIndex newsize, CFOptionFlags hint, void *info) { return CFAllocatorReallocate(kCFAllocatorDefault, ptr, newsize, hint); }, - [](void *ptr, void *info) { CFAllocatorDeallocate(kCFAllocatorDefault, ptr); }, - [](CFIndex size, CFOptionFlags hint, void *info) { return CFAllocatorGetPreferredSizeForSize(kCFAllocatorDefault, size, hint); } - }; - return context; -} - -CF_INLINE CFDataRef create_wrapped_queue() { - constexpr std::size_t queueSize = sizeof(MyConcurrentQueue); - auto queueBuffer = CFQueueTrait::malloc(queueSize); - CFAllocatorContext context = {}; - context.deallocate = [](void *ptr, void *info) { - auto q = reinterpret_cast(ptr); - const auto count = q->size_approx(); - assert(count == 0); - q->~ConcurrentQueue(); - CFQueueTrait::free(q); - }; - auto queue = new (queueBuffer) MyConcurrentQueue(); - CFAllocatorRef deallocator = CFAllocatorCreate(kCFAllocatorDefault, &context); - CFDataRef queueWrapper = CFDataCreateWithBytesNoCopy(kCFAllocatorDefault, reinterpret_cast(queueBuffer), queueSize, deallocator); - CFRelease(deallocator); - return queueWrapper; -} - -CF_INLINE CFDataRef create_wrapped_token(MyConcurrentQueue& queue) { - constexpr std::size_t tokenSize = sizeof(moodycamel::ConsumerToken); - auto tokenBuffer = CFQueueTrait::malloc(tokenSize); - CFAllocatorContext context = {}; - auto token = new (tokenBuffer) moodycamel::ConsumerToken(queue); - context.deallocate = [](void *ptr, void *info) { - auto t = reinterpret_cast(ptr); - t->~ConsumerToken(); - CFQueueTrait::free(t); - }; - CFAllocatorRef deallocator = CFAllocatorCreate(kCFAllocatorDefault, &context); - CFDataRef tokenWrapper = CFDataCreateWithBytesNoCopy(kCFAllocatorDefault, reinterpret_cast(tokenBuffer), tokenSize, deallocator); - CFRelease(deallocator); - return tokenWrapper; -} - - +} RunLoopContextInfo; CFRunLoopSourceRef create_tetra_runLoop_executor( CFTypeRef initialState, const TetraContextData *tetraContext ) { - CFDataRef queueWrapper = create_wrapped_queue(); - CFDataRef tokenWrapper = create_wrapped_token( - *reinterpret_cast(const_cast(CFDataGetBytePtr(queueWrapper))) - ); - CFDataRef contextStorage = CFDataCreate(kCFAllocatorDefault, (UInt8 *)tetraContext, sizeof(TetraContextData)); - auto registryContext = create_defaultContext(); - registryContext.retain = [](CFTypeRef ref) -> CFTypeRef { -#if __APPLE__ - constexpr size_t size = sizeof(os_unfair_lock_s); -#else - constexpr size_t size = sizeof(std::mutex); -#endif - auto buffer = CFAllocatorAllocate(kCFAllocatorDefault, size, 0); - -#if __APPLE__ - os_unfair_lock_t mutex = new (buffer) os_unfair_lock_s(OS_UNFAIR_LOCK_INIT); -#else - auto mutex = new (buffer) std::mutex; -#endif - - return buffer; - }; - registryContext.copyDescription = nullptr; - registryContext.release = [](CFTypeRef ref) { + auto queue = MyConcurrentQueue(); + auto token = moodycamel::ConsumerToken(queue); + + RunLoopContextInfo stackInfo = RunLoopContextInfo{ + std::move(queue), + std::move(token), + *tetraContext, + initialState, + CFDictionaryCreateMutable(kCFAllocatorDefault, 0, &kCFTypeDictionaryKeyCallBacks, &kCFTypeDictionaryValueCallBacks), #if __APPLE__ - auto lock = reinterpret_cast(const_cast(ref)); - lock->~os_unfair_lock_s(); + OS_UNFAIR_LOCK_INIT, #else - auto mutex = reinterpret_cast(const_cast(ref)); - mutex->~mutex(); + new (CFQueueTrait::malloc(sizeof(std::mutex))) std::mutex(), #endif - CFAllocatorDeallocate(kCFAllocatorDefault, const_cast(ref)); }; - CFAllocatorRef registryDeallocator = CFAllocatorCreate(kCFAllocatorDefault, ®istryContext); - CFMutableDictionaryRef runLoopRegistry = CFDictionaryCreateMutable(registryDeallocator, 0, &kCFTypeDictionaryKeyCallBacks, &kCFTypeDictionaryValueCallBacks); - CFArrayRef array = CFArrayCreate(kCFAllocatorDefault, (CFTypeRef []){queueWrapper, tokenWrapper, initialState, contextStorage, runLoopRegistry}, 5, &kCFTypeArrayCallBacks); - CFRelease(queueWrapper); - CFRelease(tokenWrapper); - CFRelease(contextStorage); - CFRelease(runLoopRegistry); - CFRelease(registryDeallocator); -// CFRelease(initialState); - /** - [ - queue, - consumerToken, - userDefinedState, - contextCallbackStorage - ] - **/ CFRunLoopSourceContext soureContext = { 0, - (void*)array, - CFRetain, - CFRelease, - [](CFTypeRef ref) -> CFStringRef { - CFArrayRef array = reinterpret_cast(ref); - CFTypeRef buffer[] = { - CFStringCreateWithCString(kCFAllocatorDefault, "moody::camel::ConcurrentQueue", kCFStringEncodingUTF8), - CFStringCreateWithCString(kCFAllocatorDefault, "moody::camel::ConsumerToken", kCFStringEncodingUTF8), - CFArrayGetValueAtIndex(array, 2), - CFStringCreateWithCString(kCFAllocatorDefault, "TetraContextStorage", kCFStringEncodingUTF8), - CFStringCreateWithCString(kCFAllocatorDefault, "RunLoopRegistry", kCFStringEncodingUTF8), - }; - CFArrayRef temp = CFArrayCreate(kCFAllocatorDefault, buffer, 5, &kCFTypeArrayCallBacks); - CFStringRef description = CFCopyDescription(temp); - CFRelease(temp); - return description; + (void*)&stackInfo, + [](const void * stackRawInfo) -> const void * { + void * buffer = CFAllocatorAllocate(kCFAllocatorDefault, sizeof(RunLoopContextInfo), 0); + RunLoopContextInfo* stackInfo = reinterpret_cast(const_cast(stackRawInfo)); + + auto myInfo = new (buffer) RunLoopContextInfo(std::move(*stackInfo)); + myInfo->state = CFRetain(stackInfo->state); + return myInfo; + }, + [](const void * heapRawInfo) { + auto info = reinterpret_cast(const_cast(heapRawInfo)); +#if !__APPLE__ + info->lock->~mutex(); + CFQueueTrait::free(info->lock); +#endif + auto stack = std::move(*info); + CFAllocatorDeallocate(kCFAllocatorDefault, info); + CFRelease(stack.runLoopRegistry); + CFRelease(stack.state); }, - CFEqual, - CFHash, + nullptr, + nullptr, + nullptr, [](void * info, CFRunLoopRef runLoop, CFRunLoopMode mode) { //schedule - auto array = static_cast(info); - CFTypeRef state = CFArrayGetValueAtIndex(array, 2); - CFDataRef context = (CFDataRef) CFArrayGetValueAtIndex(array, 3); - auto tetraContext = (TetraContextData *)CFDataGetBytePtr(context); + auto &sourceInfo = *reinterpret_cast(info); CFRunLoopWakeUp(runLoop); { - CFMutableDictionaryRef registry = (CFMutableDictionaryRef)CFArrayGetValueAtIndex(array, 4); - CFAllocatorContext allocContext = {}; - CFAllocatorGetContext(CFGetAllocator(registry),&allocContext); + CFMutableDictionaryRef registry = sourceInfo.runLoopRegistry; #if __APPLE__ - os_unfair_lock_lock((os_unfair_lock_t)allocContext.info); + os_unfair_lock_lock(&sourceInfo.lock); #else - std::lock_guard lock(*(std::mutex *)allocContext.info); + std::lock_guard lock(*sourceInfo.lock); #endif CFMutableSetRef set = (CFMutableSetRef)CFDictionaryGetValue(registry, runLoop); if (!set) { @@ -196,29 +104,24 @@ CFRunLoopSourceRef create_tetra_runLoop_executor( CFRelease(set); } #if __APPLE__ - os_unfair_lock_unlock((os_unfair_lock_t)allocContext.info); + os_unfair_lock_unlock(&sourceInfo.lock); #endif CFSetAddValue(set, mode); } - if (tetraContext->schedule) { - tetraContext->schedule(state, runLoop, mode); + if (sourceInfo.context.schedule) { + sourceInfo.context.schedule(sourceInfo.state, runLoop, mode); } }, [](void * info, CFRunLoopRef runLoop, CFRunLoopMode mode) { // cancel - auto array = static_cast(info); - CFTypeRef state = CFArrayGetValueAtIndex(array, 2); - CFTypeRef context = CFArrayGetValueAtIndex(array, 3); - auto tetraContext = (TetraContextData *)CFDataGetBytePtr((CFDataRef)context); + auto &sourceInfo = *reinterpret_cast(info); CFRunLoopWakeUp(runLoop); { - CFMutableDictionaryRef registry = (CFMutableDictionaryRef)CFArrayGetValueAtIndex(array, 4); - CFAllocatorContext allocContext = {}; - CFAllocatorGetContext(CFGetAllocator(registry),&allocContext); + CFMutableDictionaryRef registry = sourceInfo.runLoopRegistry; #if __APPLE__ - os_unfair_lock_lock((os_unfair_lock_t)allocContext.info); + os_unfair_lock_lock(&sourceInfo.lock); #else - std::lock_guard lock(*(std::mutex *)allocContext.info); + std::lock_guard lock(*sourceInfo.lock); #endif CFMutableSetRef set = (CFMutableSetRef)CFDictionaryGetValue(registry, runLoop); @@ -228,91 +131,124 @@ CFRunLoopSourceRef create_tetra_runLoop_executor( CFDictionaryRemoveValue(registry, runLoop); } #if __APPLE__ - os_unfair_lock_unlock((os_unfair_lock_t)allocContext.info); + os_unfair_lock_unlock(&sourceInfo.lock); #endif } - if (tetraContext->cancel) { - tetraContext->cancel(state, runLoop, mode); + if (sourceInfo.context.schedule) { + sourceInfo.context.schedule(sourceInfo.state, runLoop, mode); } }, [](void *info) { - auto array = static_cast(info); - auto queue = reinterpret_cast((void *)CFDataGetBytePtr((CFDataRef)CFArrayGetValueAtIndex(array, 0))); - auto token = reinterpret_cast((void *)CFDataGetBytePtr((CFDataRef)CFArrayGetValueAtIndex(array, 1))); - CFTypeRef state = CFArrayGetValueAtIndex(array, 2); - CFTypeRef context = CFArrayGetValueAtIndex(array, 3); - auto tetraContext = (TetraContextData *)CFDataGetBytePtr((CFDataRef)context); + auto &sourceInfo = *reinterpret_cast(info); + + CFMutableArrayRef dequeue = CFArrayCreateMutable(kCFAllocatorDefault, 0, &kCFTypeArrayCallBacks); constexpr size_t buffer_size = 10; CFCppRef result[buffer_size]; size_t size = 0; - while ((size = queue->try_dequeue_bulk(*token, result, buffer_size)) > 0) { + while ((size = sourceInfo.queue.try_dequeue_bulk(sourceInfo.token, result, buffer_size)) > 0) { for (int i = 0; i < size; i++) { CFCppRef ref = std::move(result[i]); CFArrayAppendValue(dequeue, ref.get()); } } - tetraContext->perform(state, dequeue); + sourceInfo.context.perform(sourceInfo.state, dequeue); CFRelease(dequeue); } }; CFRunLoopSourceRef source = CFRunLoopSourceCreate(kCFAllocatorDefault, 0, &soureContext); - CFRelease(array); return source; } CFDictionaryRef copy_tetra_runLoop_registry(CFRunLoopSourceRef source) { - CFMutableDictionaryRef registry_source; - CFDictionaryRef registry; + RunLoopContextInfo* info; { CFRunLoopSourceContext context = {}; CFRunLoopSourceGetContext(source, &context); - CFArrayRef array = reinterpret_cast(context.info); - assert(CFGetTypeID(array) == CFArrayGetTypeID()); - CFTypeRef ref = CFArrayGetValueAtIndex(array, 4); -// assert(CFGetTypeID(registry_source) == CFDictionaryGetTypeID()); - registry_source = reinterpret_cast(const_cast(ref)); + info = reinterpret_cast(context.info); } + RunLoopContextInfo& variable = *info; + CFDictionaryRef registry; { - CFAllocatorContext allocContext = {}; - CFAllocatorGetContext(CFGetAllocator(registry_source),&allocContext); #if __APPLE__ - os_unfair_lock_lock((os_unfair_lock_t)allocContext.info); + os_unfair_lock_lock(&variable.lock); #else - std::lock_guard lock(*(std::mutex *)allocContext.info); + std::lock_guard lock(*variable.lock); #endif - registry = CFDictionaryCreateCopy(kCFAllocatorDefault, registry_source); + registry = CFDictionaryCreateCopy(kCFAllocatorDefault, variable.runLoopRegistry); #if __APPLE__ - os_unfair_lock_unlock((os_unfair_lock_t)allocContext.info); + os_unfair_lock_unlock(&variable.lock); #endif } return registry; } -bool tetra_enqueue_and_signal(CFRunLoopSourceRef source, CFTypeRef ref) { - CFArrayRef array; - MyConcurrentQueue* queue; +CF_INLINE CFDictionaryRef try_copy_tetra_runLoop_registry(CFRunLoopSourceRef source) { + RunLoopContextInfo* info; { CFRunLoopSourceContext context = {}; CFRunLoopSourceGetContext(source, &context); - array = static_cast(context.info); + info = reinterpret_cast(context.info); } + RunLoopContextInfo& variable = *info; + CFDictionaryRef registry; { - CFDataRef queueWrapper = (CFDataRef)CFArrayGetValueAtIndex(array, 0); - queue = reinterpret_cast(const_cast(CFDataGetBytePtr(queueWrapper))); +#if __APPLE__ + if (os_unfair_lock_trylock(&variable.lock) == false) { + return nullptr; + } +#else + std::unique_lock lock(*variable.lock, std::try_to_lock); + if(!lock.owns_lock()){ + return nullptr; + } +#endif + registry = CFDictionaryCreateCopy(kCFAllocatorDefault, variable.runLoopRegistry); +#if __APPLE__ + os_unfair_lock_unlock(&variable.lock); +#endif } + return registry; +} + + +bool tetra_enqueue_and_signal(CFRunLoopSourceRef source, CFTypeRef ref) { + RunLoopContextInfo* info; + { + CFRunLoopSourceContext context = {}; + CFRunLoopSourceGetContext(source, &context); + info = reinterpret_cast(context.info); + } + RunLoopContextInfo& variable = *info; auto ptr = CFCppRef(CFRetain(ref), CFRelease); - const bool success = queue->enqueue(std::move(ptr)); + const bool success = variable.queue.enqueue(std::move(ptr)); if (!success) { return false; } CFRunLoopSourceSignal(source); - CFDictionaryRef registry = copy_tetra_runLoop_registry(source); + CFDictionaryRef registry = try_copy_tetra_runLoop_registry(source); + // somebody is already waking up the runloop + if (registry == nullptr) { + + return true; + } CFDictionaryApplyFunction(registry, [](CFTypeRef key, CFTypeRef value, void * info) { - CFRunLoopWakeUp((CFRunLoopRef) key); + if (CFRunLoopIsWaiting((CFRunLoopRef) key)) { + CFRunLoopWakeUp((CFRunLoopRef) key); + } }, nullptr); CFRelease(registry); return true; } + +CFTypeRef tetra_get_stateInfo(CFRunLoopSourceRef source) { + RunLoopContextInfo* info; + { + CFRunLoopSourceContext context = {}; + CFRunLoopSourceGetContext(source, &context); + info = reinterpret_cast(context.info); + } + return info->state; +} diff --git a/Sources/TetraRunLoopConcurrency/TetraRunLoopExecutor.swift b/Sources/TetraRunLoopConcurrency/TetraRunLoopExecutor.swift index 65b1699..fddd3d3 100644 --- a/Sources/TetraRunLoopConcurrency/TetraRunLoopExecutor.swift +++ b/Sources/TetraRunLoopConcurrency/TetraRunLoopExecutor.swift @@ -23,11 +23,18 @@ struct StateStorage { } +fileprivate actor DummyActor { + let unownedExecutor: UnownedSerialExecutor + init(unownedExecutor: UnownedSerialExecutor) { + self.unownedExecutor = unownedExecutor + } + + func run(_ block: @convention(block) () -> Void) { block() } + +} final class RunLoopStorageBufferHolder {} - - final package class TetraRunLoopExecutor: NSObject { @@ -102,6 +109,7 @@ package extension TetraRunLoopExecutor { nonisolated func enqueue(_ job: UnownedJob) { tetra_enqueue_and_signal(source, job as AnyObject) + let _ = ManagedBufferPointer(unsafeBufferObject: tetra_get_stateInfo(source)) } nonisolated func asUnownedSerialExecutor() -> UnownedSerialExecutor { @@ -121,7 +129,9 @@ package extension TetraRunLoopExecutor { ) } - + func isSameExclusiveExecutionContext(other: TetraRunLoopExecutor) -> Bool { + CFEqual(source, other.source) + } } @@ -135,11 +145,12 @@ extension TetraRunLoopExecutor { CFRunLoopAddSource(runLoop.getCFRunLoop(), source, .defaultMode) } - @available(swift, obsoleted: 1.0) - @objc - package func schedule( _ block: @convention(block) () -> Void) { - block() - } +// @available(swift, obsoleted: 1.0) +// @objc +// package func schedule( _ block: @convention(block) () -> Void) async { +// await DummyActor(unownedExecutor: asUnownedSerialExecutor()) +// .run(block) +// } } From d4fc8b96d7727432782da3a15072b07125b74f0b Mon Sep 17 00:00:00 2001 From: pbk Date: Wed, 11 Feb 2026 13:12:36 +0900 Subject: [PATCH 60/63] update UnfairLock with new api --- Package.resolved | 19 +++-- Package.swift | 20 ++++-- .../CriticalSection/ManagedUnfairLock.swift | 71 +++++++++++++++++++ 3 files changed, 100 insertions(+), 10 deletions(-) diff --git a/Package.resolved b/Package.resolved index bb910e4..ad25430 100644 --- a/Package.resolved +++ b/Package.resolved @@ -1,13 +1,22 @@ { - "originHash" : "2fc989cc1b67d91eb25b0319fdf3b974c0dd91e42699a64bf0c693dd7dde9fab", + "originHash" : "fc820e12ad9e80aff7472f03ecba201f0d811b13c43b9639e488f590adb48628", "pins" : [ + { + "identity" : "swift-async-algorithms", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-async-algorithms", + "state" : { + "revision" : "6c050d5ef8e1aa6342528460db614e9770d7f804", + "version" : "1.1.1" + } + }, { "identity" : "swift-atomics", "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-atomics.git", "state" : { - "revision" : "cd142fd2f64be2100422d658e7411e39489da985", - "version" : "1.2.0" + "revision" : "b601256eab081c0f92f059e12818ac1d4f178ff7", + "version" : "1.3.0" } }, { @@ -15,8 +24,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-collections.git", "state" : { - "revision" : "94cf62b3ba8d4bed62680a282d4c25f9c63c2efb", - "version" : "1.1.0" + "revision" : "7b847a3b7008b2dc2f47ca3110d8c782fb2e5c7e", + "version" : "1.3.0" } } ], diff --git a/Package.swift b/Package.swift index e1f7e34..8d22da0 100644 --- a/Package.swift +++ b/Package.swift @@ -1,4 +1,4 @@ -// swift-tools-version: 6.0 +// swift-tools-version: 6.2 // The swift-tools-version declares the minimum version of Swift required to build this package. import PackageDescription @@ -23,12 +23,21 @@ let package = Package( dependencies: [ // Dependencies declare other packages that this package depends on. // .package(url: /* package url */, from: "1.0.0"), - .package(url: "https://github.com/apple/swift-collections.git", .upToNextMajor(from: "1.1.0")), + .package( + url: "https://github.com/apple/swift-collections.git", + .upToNextMajor(from: "1.3.0"), + traits: [ + .defaults, +// .trait(name: "UnstableContainersPreview") + ], + ), .package( url: "https://github.com/apple/swift-atomics.git", - .upToNextMajor(from: "1.2.0") // or `.upToNextMinor + .upToNextMajor(from: "1.3.0"), + traits: [.defaults], ), - + .package(url: "https://github.com/apple/swift-async-algorithms", from: "1.1.1"), + ], targets: [ // Targets are the basic building blocks of a package. A target can define a module or a test suite. @@ -83,7 +92,8 @@ let package = Package( "CriticalSection", "BackportDiscardingTaskGroup", "Namespace", - "NamespaceExtension" + "NamespaceExtension", + .product(name: "AsyncAlgorithms", package: "swift-async-algorithms"), ], swiftSettings: [ .enableUpcomingFeature("FullTypedThrows"), diff --git a/Sources/CriticalSection/ManagedUnfairLock.swift b/Sources/CriticalSection/ManagedUnfairLock.swift index d7dee90..6232457 100644 --- a/Sources/CriticalSection/ManagedUnfairLock.swift +++ b/Sources/CriticalSection/ManagedUnfairLock.swift @@ -6,7 +6,9 @@ // import Foundation +#if canImport(os) && canImport(Darwin) import os +import Darwin @usableFromInline internal final class LockBuffer: ManagedBuffer { @@ -67,6 +69,16 @@ public struct ManagedUnfairLock: @unchecked Sendable { } } + @available(iOS 18.0, tvOS 18.0, macOS 15.0, macCatalyst 18.0, watchOS 11.0, visionOS 2.0, *) + @inlinable + func withLockUnchecked(flags: UnfairLockFlags, _ body: (inout State) throws(Failure) -> R) throws(Failure) -> R { + try __lock.withUnsafeMutablePointers { state, lock throws(Failure) in + os_unfair_lock_lock_with_flags(lock, flags.unwrapped) + defer { os_unfair_lock_unlock(lock) } + return try body(&state.pointee) + } + } + /// Perform a sendable closure while holding this lock. /// /// @@ -79,6 +91,12 @@ public struct ManagedUnfairLock: @unchecked Sendable { try withLockUnchecked(body) } + @available(iOS 18.0, tvOS 18.0, macOS 15.0, macCatalyst 18.0, watchOS 11.0, visionOS 2.0, *) + @inlinable + func withLock(flags:UnfairLockFlags ,_ body: @Sendable (inout State) throws(Failure) -> R) throws(Failure) -> R where R : Sendable { + try withLockUnchecked(body) + } + /// Attempt to acquire the lock, if successful, perform a closure while /// holding the lock. /// This method does not enforce sendability requirement @@ -164,6 +182,15 @@ public extension ManagedUnfairLock where State == Void { } } + @_unavailableFromAsync(message: "Use async-safe scoped locking instead") + @available(iOS 18.0, tvOS 18.0, macOS 15.0, macCatalyst 18.0, watchOS 11.0, visionOS 2.0, *) + @inlinable + func lock(flags: UnfairLockFlags) { + __lock.withUnsafeMutablePointerToElements { + os_unfair_lock_lock_with_flags($0, flags.unwrapped) + } + } + /// Unlock this lock. @_unavailableFromAsync(message: "Use async-safe scoped locking instead") @inlinable @@ -182,6 +209,13 @@ public extension ManagedUnfairLock where State == Void { try withLockUnchecked(body) } + @available(iOS 18.0, tvOS 18.0, macOS 15.0, macCatalyst 18.0, watchOS 11.0, visionOS 2.0, *) + @inlinable + func withLock(flags:UnfairLockFlags ,_ body: @Sendable () throws(Failure) -> R) throws(Failure) -> R where R : Sendable { + try withLockUnchecked(body) + } + + /// Perform a closure while holding this lock. /// This method does not enforce sendability requirement /// on closure body and its return type. @@ -201,6 +235,16 @@ public extension ManagedUnfairLock where State == Void { } } + @available(iOS 18.0, tvOS 18.0, macOS 15.0, macCatalyst 18.0, watchOS 11.0, visionOS 2.0, *) + @inlinable + func withLockUnchecked(flags: UnfairLockFlags, _ body: () throws(Failure) -> R) throws(Failure) -> R { + try __lock.withUnsafeMutablePointerToElements { lock throws(Failure) in + os_unfair_lock_lock_with_flags(lock, flags.unwrapped) + defer { os_unfair_lock_unlock(lock) } + return try body() + } + } + /// Attempt to acquire the lock if it is not already locked. /// /// - Returns: `true` if the lock was succesfully locked, and @@ -247,6 +291,30 @@ public extension ManagedUnfairLock where State == Void { } + +public extension ManagedUnfairLock { + + struct UnfairLockFlags: OptionSet, BitwiseCopyable, Hashable { + + public var rawValue: __os_unfair_lock_flags_t.RawValue + + public init(rawValue: RawValue) { + self.rawValue = rawValue + } + + @available(iOS 18.0, tvOS 18.0, macOS 15.0, macCatalyst 18.0, watchOS 11.0, visionOS 2.0, *) + static var adaptiveSpin:Self { .init(rawValue: os.OSAllocatedUnfairLockFlags.adaptiveSpin.rawValue) } + + @usableFromInline + internal var unwrapped: __os_unfair_lock_flags_t { + .init(rawValue: self.rawValue) + } + } + +} + + + public extension ManagedUnfairLock { /// Initialize an SwiftUnfairLock with a lock-protected sendable @@ -335,3 +403,6 @@ package func createUnfairLock() -> some UnfairLockProtocol { return ManagedUnfairLock() } } + + +#endif // canImport(os) && canImport(Darwin) From 65b4a299de4c4e6e0347a28746bcbf76ab02bed6 Mon Sep 17 00:00:00 2001 From: pbk Date: Wed, 18 Feb 2026 16:28:06 +0900 Subject: [PATCH 61/63] implementing new RunLoopExecutor --- Package.swift | 21 +- Sources/CriticalSection/AtomicStore.swift | 295 ++ Sources/CriticalSection/BackportedCell.swift | 138 + Sources/CriticalSection/Cell.swift | 41 - Sources/CriticalSection/DarwinImpl.swift | 4 +- .../Tetra/Combine/ExperimentalMapTask.swift | 4 +- Sources/Tetra/Concurrency/BroadCast2.swift | 8 - Sources/Tetra/Concurrency/JobBlock.swift | 71 - .../Tetra/Concurrency/PriorityRunLoop.swift | 438 -- .../concurrentqueue.h | 3747 ----------------- .../include/TetraConcurrentQueueShim.h | 13 - .../TetraConcurrentQueueShim/include/sim.h | 65 - Sources/TetraConcurrentQueueShim/sim.cpp | 254 -- .../MPMCBoundedQueue.swift | 150 + .../SlicedJobQueue.swift | 730 ++++ .../TetraRunLoopExecutor.swift | 209 - .../TetraRunLoopConcurrency/_MPSCQueue.swift | 162 + .../_SPMCBoundedQueue.swift | 159 + 18 files changed, 1650 insertions(+), 4859 deletions(-) create mode 100644 Sources/CriticalSection/AtomicStore.swift create mode 100644 Sources/CriticalSection/BackportedCell.swift delete mode 100644 Sources/CriticalSection/Cell.swift delete mode 100644 Sources/Tetra/Concurrency/BroadCast2.swift delete mode 100644 Sources/Tetra/Concurrency/JobBlock.swift delete mode 100644 Sources/Tetra/Concurrency/PriorityRunLoop.swift delete mode 100644 Sources/TetraConcurrentQueueShim/concurrentqueue.h delete mode 100644 Sources/TetraConcurrentQueueShim/include/TetraConcurrentQueueShim.h delete mode 100644 Sources/TetraConcurrentQueueShim/include/sim.h delete mode 100644 Sources/TetraConcurrentQueueShim/sim.cpp create mode 100644 Sources/TetraRunLoopConcurrency/MPMCBoundedQueue.swift create mode 100644 Sources/TetraRunLoopConcurrency/SlicedJobQueue.swift delete mode 100644 Sources/TetraRunLoopConcurrency/TetraRunLoopExecutor.swift create mode 100644 Sources/TetraRunLoopConcurrency/_MPSCQueue.swift create mode 100644 Sources/TetraRunLoopConcurrency/_SPMCBoundedQueue.swift diff --git a/Package.swift b/Package.swift index 8d22da0..c20bb37 100644 --- a/Package.swift +++ b/Package.swift @@ -68,6 +68,8 @@ let package = Package( .enableExperimentalFeature("StaticExclusiveOnly"), .enableExperimentalFeature("RawLayout"), .enableExperimentalFeature("BuiltinModule"), + .enableExperimentalFeature("Lifetimes"), + .enableExperimentalFeature("LifetimeDependence"), ] ), .target( @@ -92,7 +94,7 @@ let package = Package( "CriticalSection", "BackportDiscardingTaskGroup", "Namespace", - "NamespaceExtension", + "NamespaceExtension", "TetraRunLoopConcurrency", .product(name: "AsyncAlgorithms", package: "swift-async-algorithms"), ], swiftSettings: [ @@ -101,19 +103,19 @@ let package = Package( .swiftLanguageMode(.v6) ] ), - .target( - name: "TetraConcurrentQueueShim", - linkerSettings: [ - .linkedFramework("CoreFoundation") - ] - ), + .target( name: "TetraRunLoopConcurrency", dependencies: [ - "TetraConcurrentQueueShim", + "CriticalSection", + .product(name: "Atomics", package: "swift-atomics"), + .product(name: "HeapModule", package: "swift-collections"), + .product(name: "BasicContainers", package: "swift-collections"), + .product(name: "ContainersPreview", package: "swift-collections"), ], swiftSettings: [ - .swiftLanguageMode(.v6) + .swiftLanguageMode(.v6), + .enableExperimentalFeature("BuiltinModule"), ] ), .target( @@ -134,5 +136,4 @@ let package = Package( ] ) ], - cxxLanguageStandard: .cxx17 ) diff --git a/Sources/CriticalSection/AtomicStore.swift b/Sources/CriticalSection/AtomicStore.swift new file mode 100644 index 0000000..bf77a13 --- /dev/null +++ b/Sources/CriticalSection/AtomicStore.swift @@ -0,0 +1,295 @@ +// +// AtomicStore.swift +// Tetra +// +// Created by 박병관 on 2/14/26. +// +package import Atomics +package import Builtin + +@_staticExclusiveOnly +@_rawLayout(like: Value.AtomicRepresentation, movesAsLike) +package struct AtomicStore:~Copyable where Value.AtomicRepresentation.Value == Value { + + @_transparent + @usableFromInline + package var _address: UnsafeMutablePointer { + UnsafeMutablePointer(_rawAddress) + } + + @_transparent + @inline(__always) + @usableFromInline + internal var _rawAddress: Builtin.RawPointer { + Builtin.addressOfRawLayout(self) + } + + @_transparent + @usableFromInline + package init(_ initialValue: consuming Value) { + _address.initialize(to: Value.AtomicRepresentation(initialValue)) + } + + @inlinable + deinit { + let _ = _address.pointee.dispose() + } + + + @usableFromInline + @inline(__always) + @_transparent + var _ptr:UnsafeMutablePointer<_Storage> { + _address + } + + + @usableFromInline + @inline(__always) + @_transparent + var store:UnsafeAtomic { + .init(at: _address) + } + + @usableFromInline + typealias _Storage = Value.AtomicRepresentation + +} + + + + + + + + + extension AtomicStore { + + + +// typealias Atomic + + /// Atomically loads and returns the current value, applying the specified + /// memory ordering. + /// + /// - Parameter ordering: The memory ordering to apply on this operation. + /// - Returns: The current value. + @_semantics("atomics.requires_constant_orderings") + @inlinable + @_transparent +// @_alwaysEmitIntoClient + public func load( + ordering: AtomicLoadOrdering + ) -> Value { +// _Storage.atomicLoad(at: _ptr, ordering: ordering) + store.load(ordering: ordering) + } + + /// Atomically sets the current value to `desired`, applying the specified + /// memory ordering. + /// + /// - Parameter desired: The desired new value. + /// - Parameter ordering: The memory ordering to apply on this operation. + @_semantics("atomics.requires_constant_orderings") + @_transparent + //@_alwaysEmitIntoClient + public func store( + _ desired: consuming Value, + ordering: AtomicStoreOrdering + ) { + _Storage.atomicStore(desired, at: _ptr, ordering: ordering) + } + + /// Atomically sets the current value to `desired` and returns the original + /// value, applying the specified memory ordering. + /// + /// - Parameter desired: The desired new value. + /// - Parameter ordering: The memory ordering to apply on this operation. + /// - Returns: The original value. + @_semantics("atomics.requires_constant_orderings") + @_transparent + //@_alwaysEmitIntoClient + public func exchange( + _ desired: consuming Value, + ordering: AtomicUpdateOrdering + ) -> Value { + _Storage.atomicExchange(desired, at: _ptr, ordering: ordering) + } + + /// Perform an atomic compare and exchange operation on the current value, + /// applying the specified memory ordering. + /// + /// This operation performs the following algorithm as a single atomic + /// transaction: + /// + /// ``` + /// atomic(self) { currentValue in + /// let original = currentValue + /// guard original == expected else { return (false, original) } + /// currentValue = desired + /// return (true, original) + /// } + /// ``` + /// + /// This method implements a "strong" compare and exchange operation + /// that does not permit spurious failures. + /// + /// - Parameter expected: The expected current value. + /// - Parameter desired: The desired new value. + /// - Parameter ordering: The memory ordering to apply on this operation. + /// - Returns: A tuple `(exchanged, original)`, where `exchanged` is true if + /// the exchange was successful, and `original` is the original value. + @_semantics("atomics.requires_constant_orderings") + @_transparent + //@_alwaysEmitIntoClient + public func compareExchange( + expected:consuming Value, + desired: consuming Value, + ordering: AtomicUpdateOrdering + ) -> (exchanged: Bool, original: Value) { + _Storage.atomicCompareExchange( + expected: expected, + desired: desired, + at: _ptr, + ordering: ordering) + } + + /// Perform an atomic compare and exchange operation on the current value, + /// applying the specified success/failure memory orderings. + /// + /// This operation performs the following algorithm as a single atomic + /// transaction: + /// + /// ``` + /// atomic(self) { currentValue in + /// let original = currentValue + /// guard original == expected else { return (false, original) } + /// currentValue = desired + /// return (true, original) + /// } + /// ``` + /// + /// The `successOrdering` argument specifies the memory ordering to use when + /// the operation manages to update the current value, while `failureOrdering` + /// will be used when the operation leaves the value intact. + /// + /// This method implements a "strong" compare and exchange operation + /// that does not permit spurious failures. + /// + /// - Parameter expected: The expected current value. + /// - Parameter desired: The desired new value. + /// - Parameter successOrdering: The memory ordering to apply if this + /// operation performs the exchange. + /// - Parameter failureOrdering: The memory ordering to apply on this + /// operation does not perform the exchange. + /// - Returns: A tuple `(exchanged, original)`, where `exchanged` is true if + /// the exchange was successful, and `original` is the original value. + @_semantics("atomics.requires_constant_orderings") + @_transparent + //@_alwaysEmitIntoClient + public func compareExchange( + expected: consuming Value, + desired: consuming Value, + successOrdering: AtomicUpdateOrdering, + failureOrdering: AtomicLoadOrdering + ) -> (exchanged: Bool, original: Value) { + _Storage.atomicCompareExchange( + expected: expected, + desired: desired, + at: _ptr, + successOrdering: successOrdering, + failureOrdering: failureOrdering) + } + + /// Perform an atomic weak compare and exchange operation on the current + /// value, applying the memory ordering. This compare-exchange variant is + /// allowed to spuriously fail; it is designed to be called in a loop until + /// it indicates a successful exchange has happened. + /// + /// This operation performs the following algorithm as a single atomic + /// transaction: + /// + /// ``` + /// atomic(self) { currentValue in + /// let original = currentValue + /// guard original == expected else { return (false, original) } + /// currentValue = desired + /// return (true, original) + /// } + /// ``` + /// + /// (In this weak form, transient conditions may cause the `original == + /// expected` check to sometimes return false when the two values are in fact + /// the same.) + /// + /// - Parameter expected: The expected current value. + /// - Parameter desired: The desired new value. + /// - Parameter ordering: The memory ordering to apply on this operation. + /// - Returns: A tuple `(exchanged, original)`, where `exchanged` is true if + /// the exchange was successful, and `original` is the original value. + @_semantics("atomics.requires_constant_orderings") + @_transparent + //@_alwaysEmitIntoClient + public func weakCompareExchange( + expected: consuming Value, + desired: __owned Value, + ordering: AtomicUpdateOrdering + ) -> (exchanged: Bool, original: Value) { + _Storage.atomicWeakCompareExchange( + expected: expected, + desired: desired, + at: _ptr, + ordering: ordering) + } + + /// Perform an atomic weak compare and exchange operation on the current + /// value, applying the specified success/failure memory orderings. This + /// compare-exchange variant is allowed to spuriously fail; it is designed to + /// be called in a loop until it indicates a successful exchange has happened. + /// + /// This operation performs the following algorithm as a single atomic + /// transaction: + /// + /// ``` + /// atomic(self) { currentValue in + /// let original = currentValue + /// guard original == expected else { return (false, original) } + /// currentValue = desired + /// return (true, original) + /// } + /// ``` + /// + /// (In this weak form, transient conditions may cause the `original == + /// expected` check to sometimes return false when the two values are in fact + /// the same.) + /// + /// The `ordering` argument specifies the memory ordering to use when the + /// operation manages to update the current value, while `failureOrdering` + /// will be used when the operation leaves the value intact. + /// + /// - Parameter expected: The expected current value. + /// - Parameter desired: The desired new value. + /// - Parameter successOrdering: The memory ordering to apply if this + /// operation performs the exchange. + /// - Parameter failureOrdering: The memory ordering to apply on this + /// operation does not perform the exchange. + /// - Returns: A tuple `(exchanged, original)`, where `exchanged` is true if + /// the exchange was successful, and `original` is the original value. + @_semantics("atomics.requires_constant_orderings") + @_transparent + //@_alwaysEmitIntoClient + public func weakCompareExchange( + expected:consuming Value, + desired: consuming Value, + successOrdering: AtomicUpdateOrdering, + failureOrdering: AtomicLoadOrdering + ) -> (exchanged: Bool, original: Value) { + _Storage.atomicWeakCompareExchange( + expected: expected, + desired: desired, + at: _ptr, + successOrdering: successOrdering, + failureOrdering: failureOrdering) + } +} + diff --git a/Sources/CriticalSection/BackportedCell.swift b/Sources/CriticalSection/BackportedCell.swift new file mode 100644 index 0000000..accc74d --- /dev/null +++ b/Sources/CriticalSection/BackportedCell.swift @@ -0,0 +1,138 @@ +// +// BackportedCell.swift +// +// +// Created by 박병관 on 6/26/24. +// +package import Builtin + +//@available(macOS 26.0.0, *) +@_rawLayout(likeArrayOf: T, count: 5, movesAsLike) +package struct FiveArray:~Copyable { + @_transparent + @usableFromInline + package var _rawAddress: Builtin.RawPointer { + Builtin.addressOfRawLayout(self) + } + @_transparent + @usableFromInline + package var _address: UnsafeMutableBufferPointer { + .init(start: .init(_rawAddress), count: 5) + } + + public init(initializingWith initializer: (inout OutputSpan) throws(E) -> Void) throws(E) where E : Error { + var span = unsafe OutputSpan(buffer: _address, initializedCount: 0) + try initializer(&span) + let count = span.finalize(for: _address) + + precondition(5 == count) + } + + deinit { + _address.deinitialize() + } + public + var span:Span { + @_lifetime(borrow self) + borrowing get { + _overrideLifetime(Span(_unsafeStart: UnsafePointer(_rawAddress), count: 5), borrowing: self) + } + } + public + var mutableSpan:MutableSpan { + @_lifetime(&self) + mutating get { + _overrideLifetime(MutableSpan(_unsafeStart: UnsafeMutablePointer(_rawAddress), count:5), mutating: &self) + } + } + + package subscript (index: Int) -> T { + borrowing _read { + yield _address[index] + } + mutating _modify { + yield &_address[index] + } + } + +} + +@_rawLayout(likeArrayOf: T, count: 3, movesAsLike) +package struct ThreeArray:~Copyable { + @_transparent + @usableFromInline + package var _rawAddress: Builtin.RawPointer { + Builtin.addressOfRawLayout(self) + } + @_transparent + @usableFromInline + package var _address: UnsafeMutableBufferPointer { + .init(start: .init(_rawAddress), count: 3) + } + + public init(initializingWith initializer: (inout OutputSpan) throws(E) -> Void) throws(E) where E : Error { + var span = unsafe OutputSpan(buffer: _address, initializedCount: 0) + try initializer(&span) + let count = span.finalize(for: _address) + + precondition(5 == count) + } + + deinit { + _address.deinitialize() + } + public + var span:Span { + @_lifetime(borrow self) + borrowing get { + _overrideLifetime(Span(_unsafeStart: UnsafePointer(_rawAddress), count: 3), borrowing: self) + } + } + public + var mutableSpan:MutableSpan { + @_lifetime(&self) + mutating get { + _overrideLifetime(MutableSpan(_unsafeStart: UnsafeMutablePointer(_rawAddress), count:3), mutating: &self) + } + } + + package subscript (index: Int) -> T { + borrowing _read { + yield _address[index] + } + mutating _modify { + yield &_address[index] + } + } + +} + +@frozen +@usableFromInline +@_rawLayout(like: Value, movesAsLike) +package struct BackportedCell: ~Copyable { + + @_transparent + @usableFromInline + package var _address: UnsafeMutablePointer { + UnsafeMutablePointer(_rawAddress) + } + + @_transparent +// @usableFromInline + public var _rawAddress: Builtin.RawPointer { + Builtin.addressOfRawLayout(self) + } + + @_transparent + @usableFromInline + package init(_ initialValue: consuming Value) { + _address.initialize(to: initialValue) + } + + @inlinable + deinit { + _address.deinitialize(count: 1) + } + +} diff --git a/Sources/CriticalSection/Cell.swift b/Sources/CriticalSection/Cell.swift deleted file mode 100644 index 3b298d0..0000000 --- a/Sources/CriticalSection/Cell.swift +++ /dev/null @@ -1,41 +0,0 @@ -// -// Cell.swift -// -// -// Created by 박병관 on 6/26/24. -// -#if $BuiltinAddressOfRawLayout -import Builtin -import Synchronization - -@frozen -@usableFromInline -@_rawLayout(like: Value, movesAsLike) -internal struct _Cell: ~Copyable { - - @_transparent - @usableFromInline - internal var _address: UnsafeMutablePointer { - UnsafeMutablePointer(_rawAddress) - } - - @_transparent - @usableFromInline - internal var _rawAddress: Builtin.RawPointer { - Builtin.addressOfRawLayout(self) - } - - @_transparent - @usableFromInline - internal init(_ initialValue: consuming Value) { - _address.initialize(to: initialValue) - } - - @inlinable - deinit { - _address.deinitialize(count: 1) - } - -} - -#endif diff --git a/Sources/CriticalSection/DarwinImpl.swift b/Sources/CriticalSection/DarwinImpl.swift index dc2cd16..65c8457 100644 --- a/Sources/CriticalSection/DarwinImpl.swift +++ b/Sources/CriticalSection/DarwinImpl.swift @@ -17,11 +17,11 @@ import Darwin @_staticExclusiveOnly public struct _MutexHandle: ~Copyable { @usableFromInline - let value: _Cell + let value: BackportedCell @_transparent public init() { - value = _Cell(os_unfair_lock()) + value = BackportedCell(os_unfair_lock()) } @_transparent diff --git a/Sources/Tetra/Combine/ExperimentalMapTask.swift b/Sources/Tetra/Combine/ExperimentalMapTask.swift index 8858757..3d46296 100644 --- a/Sources/Tetra/Combine/ExperimentalMapTask.swift +++ b/Sources/Tetra/Combine/ExperimentalMapTask.swift @@ -30,7 +30,9 @@ public struct MultiMapTask: Publisher where Upstream public func receive(subscriber: S) where S : Subscriber, Upstream.Failure == S.Failure, Output == S.Input { let processor = Inner(maxTasks: maxTasks, subscriber: subscriber, transform: transform) - let task = if #available(macOS 15.0, iOS 18.0, watchOS 11.0, tvOS 18.0, visionOS 2.0, *), let executor = taskExecutor as? (any TaskExecutor) { + let task = if #available(macOS 26.0, iOS 26.0, watchOS 26.0, tvOS 26.0, *) { + Task.immediate(priority: priority, executorPreference: taskExecutor as? (any TaskExecutor), operation: processor.run) + } else if #available(macOS 15.0, iOS 18.0, watchOS 11.0, tvOS 18.0, visionOS 2.0, *), let executor = taskExecutor as? (any TaskExecutor) { Task(executorPreference: executor, priority: priority, operation: processor.run) } else { Task(priority: priority, operation: processor.run) diff --git a/Sources/Tetra/Concurrency/BroadCast2.swift b/Sources/Tetra/Concurrency/BroadCast2.swift deleted file mode 100644 index 953c54a..0000000 --- a/Sources/Tetra/Concurrency/BroadCast2.swift +++ /dev/null @@ -1,8 +0,0 @@ -// -// File.swift -// Tetra -// -// Created by 박병관 on 1/4/25. -// - -import Foundation diff --git a/Sources/Tetra/Concurrency/JobBlock.swift b/Sources/Tetra/Concurrency/JobBlock.swift deleted file mode 100644 index b04755a..0000000 --- a/Sources/Tetra/Concurrency/JobBlock.swift +++ /dev/null @@ -1,71 +0,0 @@ -// -// JobBlock.swift -// -// -// Created by 박병관 on 6/30/24. -// -import Darwin -import Dispatch - -// three word is max size to use stack allocation -@usableFromInline -struct JobBlock: Hashable, Comparable, Sendable { - - @usableFromInline - let id:Int - @usableFromInline - let jobImp:UnownedJob - nonisolated(unsafe) - var token:pthread_override_t? - - @usableFromInline - var priority:UInt8 { - if #available(macOS 14.0, iOS 17.0, watchOS 10.0, tvOS 17.0, *) { - jobImp.priority.rawValue - } else { - 0 - } - } - - @usableFromInline - @available(macOS 14.0, iOS 17.0, watchOS 10.0, tvOS 17.0, *) - init(id: Int, job: consuming ExecutorJob) { - self.id = id - self.jobImp = UnownedJob(job) - - } - - @usableFromInline - init(id: Int, jobRef: UnownedJob) { - self.id = id - self.jobImp = jobRef - } - - @usableFromInline - func hash(into hasher: inout Hasher) { - hasher.combine(priority) - } - - @usableFromInline - static func == (lhs: Self, rhs: Self) -> Bool { - lhs.id == rhs.id - } - - @usableFromInline - static func < (lhs: Self, rhs: Self) -> Bool { - if lhs.priority == rhs.priority { - return lhs.id < rhs.id - } - return lhs.priority < rhs.priority - } - - @usableFromInline - static func > (lhs:Self, rhs:Self) -> Bool { - if lhs.priority == rhs.priority { - return lhs.id > rhs.id - } - return lhs.priority > rhs.priority - } - -} - diff --git a/Sources/Tetra/Concurrency/PriorityRunLoop.swift b/Sources/Tetra/Concurrency/PriorityRunLoop.swift deleted file mode 100644 index b6f5ac3..0000000 --- a/Sources/Tetra/Concurrency/PriorityRunLoop.swift +++ /dev/null @@ -1,438 +0,0 @@ -// -// PriorityRunLoop2.swift -// -// -// Created by 박병관 on 6/30/24. -// - -import HeapModule -import Foundation -import CoreFoundation -public import CriticalSection - -@usableFromInline -struct RunLoopPriorityQueue: ~Copyable, Sendable { - - @usableFromInline - internal let heaps: some UnfairStateLock> = createCheckedStateLock(checkedState: .init()) - - @usableFromInline - nonisolated(unsafe) - internal let runLoop:CFRunLoop - - @usableFromInline - nonisolated(unsafe) - internal let source:CFRunLoopSource - - @usableFromInline - internal let isMain:Bool - - nonisolated(unsafe) - internal let thread:pthread_t - - - @usableFromInline - init( - runLoop: RunLoop, - threadId: pthread_t, - execute: @escaping (consuming UnownedJob) -> Void - ) { - self.runLoop = runLoop.getCFRunLoop() - self.thread = threadId - self.isMain = CFEqual(runLoop.getCFRunLoop(), CFRunLoopGetMain()) - if isMain { - self.source = CFRunLoopSourceCreate(nil, 0, nil) - CFRunLoopSourceInvalidate(source) - } else { - - self.source = RunLoopSourceCreateWithHandler { [heaps] in - - guard $0 == .perform else { return } - - var queue = heaps.withLock{ - var next = Heap() - swap(&next, &$0) - return next - } - let currentQos:DispatchQoS - do { - let thread_qos = qos_class_self() - var priority:Int32 = 0 - pthread_get_qos_class_np(pthread_self(), nil, &priority) - currentQos = .init(qosClass: .init(rawValue: thread_qos)!, relativePriority: Int(priority)) - } - var qos: DispatchQoS = currentQos - - defer { - if qos.qosClass != currentQos.qosClass { - let result = pthread_set_qos_class_self_np(currentQos.qosClass.rawValue, Int32(currentQos.relativePriority)) - assert(result == 0, "\(result)") - } - } - - /* - Managing QoS, boost CPU instructions about 5% and decrease CPU cycles by 5% when root task is about `low` priority. and enqueing about 500 random priority tasks at the same time. - - */ - var jobPriority = currentQos.evaluateTaskPriority() - while let block = queue.popMax() { - let job = block.jobImp - defer { execute(job) } - if #available(macOS 14.0, iOS 17.0, watchOS 10.0, tvOS 17.0, *), let newTaskPriority = TaskPriority(job.priority), jobPriority != newTaskPriority { - let newQos = newTaskPriority.evaluateQos() - jobPriority = newTaskPriority - let result = pthread_set_qos_class_self_np(newQos.qosClass.rawValue, Int32(newQos.relativePriority)) - qos = newQos - assert(result == 0, "\(result)") - - } - if let ref = block.token { - let result = pthread_override_qos_class_end_np(ref) - assert(result == 0, "\(result) pthread_override_qos_class_end_np failed") - } - } - } - CFRunLoopAddSource(self.runLoop, source, .commonModes) - } - } - - @inlinable - deinit { - if isMain { - return - } - CFRunLoopSourceInvalidate(source) - if CFRunLoopCopyCurrentMode(runLoop) != nil{ - var arrays = CFRunLoopCopyAllModes(runLoop) as! [CFString] - arrays.append(CFRunLoopMode.commonModes.rawValue) - let timer = CFRunLoopTimerCreate(nil, CFAbsoluteTimeGetCurrent(), 0, 0, 0, nil, nil) - arrays.forEach{ - CFRunLoopAddTimer(runLoop, timer, .init($0)) - } - } else { - CFRunLoopWakeUp(runLoop) - } - heaps.withLock{ - precondition($0.count == 0) - } - } - - @usableFromInline - internal func evaluateCommonModes() -> [CFRunLoopMode] { - var arrys = [CFRunLoopMode]() - withUnsafeMutablePointer(to: &arrys) { ptr in - var context = CFRunLoopSourceContext() - context.info = .init(ptr) - context.schedule = { info, _ , mode in - let arrayPtr = info!.assumingMemoryBound(to: [CFRunLoopMode].self) - arrayPtr.pointee.append(mode!) - } - let emptySource = CFRunLoopSourceCreate(nil, 0, &context)! - CFRunLoopAddSource(runLoop, emptySource, .commonModes) - CFRunLoopSourceInvalidate(source) - } - return arrys - } - - - @usableFromInline - nonisolated - internal func schedule(_ job: consuming UnownedJob) { - if isMain { - MainActor.shared.enqueue(job) - return - } - let qos:DispatchQoS? - if #available(macOS 14.0, iOS 17.0, watchOS 10.0, tvOS 17.0, *) { - let value = TaskPriority(job.priority)?.evaluateQos() - if value?.qosClass != .unspecified { - qos = value - } else { - qos = nil - } - } else { - qos = nil - } - let threadId = thread - var qos_class = QOS_CLASS_UNSPECIFIED - pthread_get_qos_class_np(threadId, &qos_class, nil) -// print(DispatchQoS.QoSClass(rawValue: qos_class)!) - heaps.withLockUnchecked{ [job] in - // lastest has the lower id which results lower priority - let id = -$0.count - let item = JobBlock(id: id, jobRef: job) - $0.insert(item) - if #available(macOS 14.0, iOS 17.0, watchOS 10.0, tvOS 17.0, *), var max = $0.popMax() { - if max.token == nil, let qos, qos.qosClass.rawValue != qos_class, item.priority == max.priority { - let override:pthread_override_t? = pthread_override_qos_class_start_np(threadId, qos.qosClass.rawValue, Int32(qos.relativePriority)) - max.token = override - } - $0.insert(max) - } - } - CFRunLoopSourceSignal(source) - } - - @inlinable - nonisolated - internal func add(_ mode:CFRunLoopMode) { - if isMain { - return - } - CFRunLoopAddSource(runLoop, source, mode) - } - - // you can not remove common mode - @inlinable - nonisolated - internal func remove(_ mode:CFRunLoopMode) { - if isMain { - return - } - if mode == .commonModes || mode == .defaultMode || evaluateCommonModes().contains(mode) { - return - } - CFRunLoopRemoveSource(runLoop, source, mode) - } - -} - - -package final class RunLoopPriorityExecutor { - - - // cache for faster comparsion, RunLoop comparsion trigger creating extra RunLoop - // I'm not sure storing pthread_t as bitpattern is a good idea - @usableFromInline - let threadId:Int - - @usableFromInline - internal let queue:RunLoopPriorityQueue - - @usableFromInline - nonisolated(unsafe) - internal let _runLoop:RunLoop - - @inlinable - internal init() { - self._runLoop = .current - var serialRef: UnownedSerialExecutor! = nil - self.threadId = .init(bitPattern: pthread_self()) - if #available(macOS 15.0, iOS 18.0, watchOS 11.0, tvOS 18.0, visionOS 2.0, *) { - var taskRef:UnownedTaskExecutor! = nil - self.queue = RunLoopPriorityQueue(runLoop: .current, threadId: pthread_self()) { - $0.runSynchronously(isolatedTo: serialRef, taskExecutor: taskRef) - } - taskRef = asUnownedTaskExecutor() - } else { - self.queue = RunLoopPriorityQueue(runLoop: .current, threadId: pthread_self()) { - $0.runSynchronously(on: serialRef) - } - } - serialRef = asUnownedSerialExecutor() - - } - - @inlinable - deinit { - let key = ObjectIdentifier(queue.runLoop) - let _ = Self.cache.withLockUnchecked{ - $0.removeValue(forKey: key) - } - - - } - - @inlinable - nonisolated - public var inRunLoop: Bool { - let this = pthread_t(bitPattern: threadId) - let current = pthread_self() - let check = pthread_equal(this, current) - - return check != 0 - } - - // if you access the runLoop while not isolated, it will trigger assert - @inlinable - public var runLoop: RunLoop { - assert(inRunLoop, "can not access \(#function) outside of isolation") - return _runLoop - } - - @inlinable - nonisolated - public func add(_ mode:RunLoop.Mode) { - queue.add(.init(mode.rawValue as CFString)) - } - - // you can not remove common mode - @inlinable - nonisolated - public func remove(_ mode:RunLoop.Mode) { - queue.remove(.init(mode.rawValue as CFString)) - } - - @usableFromInline - internal var source:CFRunLoopSource { - queue.source - } - -} - -extension RunLoopPriorityExecutor: SerialExecutor { - - @inlinable - nonisolated - public func isSameExclusiveExecutionContext(other: borrowing RunLoopPriorityExecutor) -> Bool { - return CFEqual(queue.runLoop, other.queue.runLoop) - } - - @inlinable - nonisolated - public func asUnownedSerialExecutor() -> UnownedSerialExecutor { - if queue.isMain { - return MainActor.sharedUnownedExecutor - } else if #available(macOS 14.0, iOS 17.0, watchOS 10.0, tvOS 17.0, *) { - return .init(complexEquality: self) - } else { - return .init(ordinary: self) - } - } - - @inlinable - nonisolated - public func checkIsolated() { - precondition(CFEqual(queue.runLoop, CFRunLoopGetCurrent()), "Unexpected isolation context, expected to be executing on \(runLoop)") - } - - @available(macOS 14.0, iOS 17.0, watchOS 10.0, tvOS 17.0, *) - @inlinable - nonisolated - public func enqueue(_ job: consuming ExecutorJob) { - queue.schedule(.init(job)) - - if !inRunLoop { - CFRunLoopWakeUp(queue.runLoop) - } - } - - @inlinable - nonisolated - public func enqueue(_ job: UnownedJob) { - queue.schedule(job) - if !inRunLoop { - CFRunLoopWakeUp(queue.runLoop) - } - } - -} - -// MARK: Concurrency TaskExecutor -@available(macOS 15.0, iOS 18.0, watchOS 11.0, tvOS 18.0, visionOS 2.0, *) -extension RunLoopPriorityExecutor: TaskExecutor { } - - -extension RunLoopPriorityExecutor { - - @usableFromInline - struct Boxed { - @usableFromInline - unowned let value:RunLoopPriorityExecutor? - - @usableFromInline - init(value: RunLoopPriorityExecutor?) { - self.value = value - } - - } - - // we rarely access this state, only when creating and deinitialzing - // so it is reasonable to use global lock rather than thread local - // since thread local keeps stored reference alive and - // user can call this method from thread pool( Concurrency, libdispatch) - @usableFromInline - static let cache: some UnfairStateLock<[ObjectIdentifier: Unmanaged]> = createUncheckedStateLock(uncheckedState: [:]) - - -} - -extension RunLoopPriorityExecutor { - - - /// Transform current Thread as the RunLoop Executor, and run the runLoop - /// - /// - /// Actual behavior depends on the current RunLoop state. - /// - /// 1) called from existing `RunLoopPriorityExecutor` thread. - /// existing Executor is returned - /// This does not runs runloop. RunLoop is deactivated when this, and all previous executor is dead. - /// 2) called from `MainThread` - /// create dummy executor and return. Dummy executor dispatch all the jobs to the `MainActor` - /// 3) called from active runloop Thread. (someone is already controlling the RunLoop) - /// create executor and return. This executor does not controls the runLoop. Existing RunLoop owner has the resposibility to keep runLoop alive, otherwise enqued Job would leak. - /// - /// 4) called from fresh Thread (no one is running runLoop) - /// create optimized executor, call `setupHandle` than controls the RunLoop of current Thread. - /// This function does not return, until executor is dead. So,`setupHandle` is the entrypoint of using the executor. - /// - Parameter setupHandle: called right before runLoop runs, runloop is active until executor is dead. this block is called exactly once. - /// - Important: when using it with existing active runLoop, keep runloop alive until Executor is gracefully deinitialized - @inlinable - public static func getOrCreate(_ block: (consuming RunLoopPriorityExecutor) -> Void) { - if Thread.isMainThread { - let executor = Self() - (consume block)(executor) - return - } - let runLoop = RunLoop.current.getCFRunLoop() - let key = ObjectIdentifier(runLoop) - if let existing = cache.withLock({ - $0[key]?.takeUnretainedValue() - }) { - (consume block)(existing) - return - } - let source:CFRunLoopSource - do { - let executor = Self() - Self.cache.withLock{ - $0[key] = .passUnretained(executor) - } - source = executor.source - (consume block)(executor) - } - while CFRunLoopSourceIsValid(source), RunLoop.current.run(mode: .default, before: .distantFuture) { - - } - } - - -} - - - - - - -/* - - Executor -> contains runLoop and Context - Context has JobQueue - One or More Executor can reference the same Context And RunLoop - if Executor reference the same RunLoop than the Context also must be smae - Context has connection to RunLoop Source and Observer - Context owns the Source and Observer, - Source has no info about the context - Observer has a weak reference to the Context - when Context is destroyed (no executor is alived), it stops the Source, but do not destroy RunLoop Observer - RunLoopObserver it self checks the weak reference of Context and do its clean up - - thread_local -> store - - - - - */ - - diff --git a/Sources/TetraConcurrentQueueShim/concurrentqueue.h b/Sources/TetraConcurrentQueueShim/concurrentqueue.h deleted file mode 100644 index 99caefc..0000000 --- a/Sources/TetraConcurrentQueueShim/concurrentqueue.h +++ /dev/null @@ -1,3747 +0,0 @@ -// Provides a C++11 implementation of a multi-producer, multi-consumer lock-free queue. -// An overview, including benchmark results, is provided here: -// http://moodycamel.com/blog/2014/a-fast-general-purpose-lock-free-queue-for-c++ -// The full design is also described in excruciating detail at: -// http://moodycamel.com/blog/2014/detailed-design-of-a-lock-free-queue - -// Simplified BSD license: -// Copyright (c) 2013-2020, Cameron Desrochers. -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without modification, -// are permitted provided that the following conditions are met: -// -// - Redistributions of source code must retain the above copyright notice, this list of -// conditions and the following disclaimer. -// - Redistributions in binary form must reproduce the above copyright notice, this list of -// conditions and the following disclaimer in the documentation and/or other materials -// provided with the distribution. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF -// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL -// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT -// OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) -// HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR -// TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, -// EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -// Also dual-licensed under the Boost Software License (see LICENSE.md) - -#pragma once - -#if defined(__GNUC__) && !defined(__INTEL_COMPILER) -// Disable -Wconversion warnings (spuriously triggered when Traits::size_t and -// Traits::index_t are set to < 32 bits, causing integer promotion, causing warnings -// upon assigning any computed values) -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wconversion" - -#ifdef MCDBGQ_USE_RELACY -#pragma GCC diagnostic ignored "-Wint-to-pointer-cast" -#endif -#endif - -#if defined(_MSC_VER) && (!defined(_HAS_CXX17) || !_HAS_CXX17) -// VS2019 with /W4 warns about constant conditional expressions but unless /std=c++17 or higher -// does not support `if constexpr`, so we have no choice but to simply disable the warning -#pragma warning(push) -#pragma warning(disable: 4127) // conditional expression is constant -#endif - -#if defined(__APPLE__) -#include "TargetConditionals.h" -#endif - -#ifdef MCDBGQ_USE_RELACY -#include "relacy/relacy_std.hpp" -#include "relacy_shims.h" -// We only use malloc/free anyway, and the delete macro messes up `= delete` method declarations. -// We'll override the default trait malloc ourselves without a macro. -#undef new -#undef delete -#undef malloc -#undef free -#else -#include // Requires C++11. Sorry VS2010. -#include -#endif -#include // for max_align_t -#include -#include -#include -#include -#include -#include -#include // for CHAR_BIT -#include -#include // partly for __WINPTHREADS_VERSION if on MinGW-w64 w/ POSIX threading -#include // used for thread exit synchronization - -// Platform-specific definitions of a numeric thread ID type and an invalid value -namespace moodycamel { namespace details { - template struct thread_id_converter { - typedef thread_id_t thread_id_numeric_size_t; - typedef thread_id_t thread_id_hash_t; - static thread_id_hash_t prehash(thread_id_t const& x) { return x; } - }; -} } -#if defined(MCDBGQ_USE_RELACY) -namespace moodycamel { namespace details { - typedef std::uint32_t thread_id_t; - static const thread_id_t invalid_thread_id = 0xFFFFFFFFU; - static const thread_id_t invalid_thread_id2 = 0xFFFFFFFEU; - static inline thread_id_t thread_id() { return rl::thread_index(); } -} } -#elif defined(_WIN32) || defined(__WINDOWS__) || defined(__WIN32__) -// No sense pulling in windows.h in a header, we'll manually declare the function -// we use and rely on backwards-compatibility for this not to break -extern "C" __declspec(dllimport) unsigned long __stdcall GetCurrentThreadId(void); -namespace moodycamel { namespace details { - static_assert(sizeof(unsigned long) == sizeof(std::uint32_t), "Expected size of unsigned long to be 32 bits on Windows"); - typedef std::uint32_t thread_id_t; - static const thread_id_t invalid_thread_id = 0; // See http://blogs.msdn.com/b/oldnewthing/archive/2004/02/23/78395.aspx - static const thread_id_t invalid_thread_id2 = 0xFFFFFFFFU; // Not technically guaranteed to be invalid, but is never used in practice. Note that all Win32 thread IDs are presently multiples of 4. - static inline thread_id_t thread_id() { return static_cast(::GetCurrentThreadId()); } -} } -#elif defined(__arm__) || defined(_M_ARM) || defined(__aarch64__) || (defined(__APPLE__) && TARGET_OS_IPHONE) || defined(__MVS__) || defined(MOODYCAMEL_NO_THREAD_LOCAL) -namespace moodycamel { namespace details { - static_assert(sizeof(std::thread::id) == 4 || sizeof(std::thread::id) == 8, "std::thread::id is expected to be either 4 or 8 bytes"); - - typedef std::thread::id thread_id_t; - static const thread_id_t invalid_thread_id; // Default ctor creates invalid ID - - // Note we don't define a invalid_thread_id2 since std::thread::id doesn't have one; it's - // only used if MOODYCAMEL_CPP11_THREAD_LOCAL_SUPPORTED is defined anyway, which it won't - // be. - static inline thread_id_t thread_id() { return std::this_thread::get_id(); } - - template struct thread_id_size { }; - template<> struct thread_id_size<4> { typedef std::uint32_t numeric_t; }; - template<> struct thread_id_size<8> { typedef std::uint64_t numeric_t; }; - - template<> struct thread_id_converter { - typedef thread_id_size::numeric_t thread_id_numeric_size_t; -#ifndef __APPLE__ - typedef std::size_t thread_id_hash_t; -#else - typedef thread_id_numeric_size_t thread_id_hash_t; -#endif - - static thread_id_hash_t prehash(thread_id_t const& x) - { -#ifndef __APPLE__ - return std::hash()(x); -#else - return *reinterpret_cast(&x); -#endif - } - }; -} } -#else -// Use a nice trick from this answer: http://stackoverflow.com/a/8438730/21475 -// In order to get a numeric thread ID in a platform-independent way, we use a thread-local -// static variable's address as a thread identifier :-) -#if defined(__GNUC__) || defined(__INTEL_COMPILER) -#define MOODYCAMEL_THREADLOCAL __thread -#elif defined(_MSC_VER) -#define MOODYCAMEL_THREADLOCAL __declspec(thread) -#else -// Assume C++11 compliant compiler -#define MOODYCAMEL_THREADLOCAL thread_local -#endif -namespace moodycamel { namespace details { - typedef std::uintptr_t thread_id_t; - static const thread_id_t invalid_thread_id = 0; // Address can't be nullptr - static const thread_id_t invalid_thread_id2 = 1; // Member accesses off a null pointer are also generally invalid. Plus it's not aligned. - inline thread_id_t thread_id() { static MOODYCAMEL_THREADLOCAL int x; return reinterpret_cast(&x); } -} } -#endif - -// Constexpr if -#ifndef MOODYCAMEL_CONSTEXPR_IF -#if (defined(_MSC_VER) && defined(_HAS_CXX17) && _HAS_CXX17) || __cplusplus > 201402L -#define MOODYCAMEL_CONSTEXPR_IF if constexpr -#define MOODYCAMEL_MAYBE_UNUSED [[maybe_unused]] -#else -#define MOODYCAMEL_CONSTEXPR_IF if -#define MOODYCAMEL_MAYBE_UNUSED -#endif -#endif - -// Exceptions -#ifndef MOODYCAMEL_EXCEPTIONS_ENABLED -#if (defined(_MSC_VER) && defined(_CPPUNWIND)) || (defined(__GNUC__) && defined(__EXCEPTIONS)) || (!defined(_MSC_VER) && !defined(__GNUC__)) -#define MOODYCAMEL_EXCEPTIONS_ENABLED -#endif -#endif -#ifdef MOODYCAMEL_EXCEPTIONS_ENABLED -#define MOODYCAMEL_TRY try -#define MOODYCAMEL_CATCH(...) catch(__VA_ARGS__) -#define MOODYCAMEL_RETHROW throw -#define MOODYCAMEL_THROW(expr) throw (expr) -#else -#define MOODYCAMEL_TRY MOODYCAMEL_CONSTEXPR_IF (true) -#define MOODYCAMEL_CATCH(...) else MOODYCAMEL_CONSTEXPR_IF (false) -#define MOODYCAMEL_RETHROW -#define MOODYCAMEL_THROW(expr) -#endif - -#ifndef MOODYCAMEL_NOEXCEPT -#if !defined(MOODYCAMEL_EXCEPTIONS_ENABLED) -#define MOODYCAMEL_NOEXCEPT -#define MOODYCAMEL_NOEXCEPT_CTOR(type, valueType, expr) true -#define MOODYCAMEL_NOEXCEPT_ASSIGN(type, valueType, expr) true -#elif defined(_MSC_VER) && defined(_NOEXCEPT) && _MSC_VER < 1800 -// VS2012's std::is_nothrow_[move_]constructible is broken and returns true when it shouldn't :-( -// We have to assume *all* non-trivial constructors may throw on VS2012! -#define MOODYCAMEL_NOEXCEPT _NOEXCEPT -#define MOODYCAMEL_NOEXCEPT_CTOR(type, valueType, expr) (std::is_rvalue_reference::value && std::is_move_constructible::value ? std::is_trivially_move_constructible::value : std::is_trivially_copy_constructible::value) -#define MOODYCAMEL_NOEXCEPT_ASSIGN(type, valueType, expr) ((std::is_rvalue_reference::value && std::is_move_assignable::value ? std::is_trivially_move_assignable::value || std::is_nothrow_move_assignable::value : std::is_trivially_copy_assignable::value || std::is_nothrow_copy_assignable::value) && MOODYCAMEL_NOEXCEPT_CTOR(type, valueType, expr)) -#elif defined(_MSC_VER) && defined(_NOEXCEPT) && _MSC_VER < 1900 -#define MOODYCAMEL_NOEXCEPT _NOEXCEPT -#define MOODYCAMEL_NOEXCEPT_CTOR(type, valueType, expr) (std::is_rvalue_reference::value && std::is_move_constructible::value ? std::is_trivially_move_constructible::value || std::is_nothrow_move_constructible::value : std::is_trivially_copy_constructible::value || std::is_nothrow_copy_constructible::value) -#define MOODYCAMEL_NOEXCEPT_ASSIGN(type, valueType, expr) ((std::is_rvalue_reference::value && std::is_move_assignable::value ? std::is_trivially_move_assignable::value || std::is_nothrow_move_assignable::value : std::is_trivially_copy_assignable::value || std::is_nothrow_copy_assignable::value) && MOODYCAMEL_NOEXCEPT_CTOR(type, valueType, expr)) -#else -#define MOODYCAMEL_NOEXCEPT noexcept -#define MOODYCAMEL_NOEXCEPT_CTOR(type, valueType, expr) noexcept(expr) -#define MOODYCAMEL_NOEXCEPT_ASSIGN(type, valueType, expr) noexcept(expr) -#endif -#endif - -#ifndef MOODYCAMEL_CPP11_THREAD_LOCAL_SUPPORTED -#ifdef MCDBGQ_USE_RELACY -#define MOODYCAMEL_CPP11_THREAD_LOCAL_SUPPORTED -#else -// VS2013 doesn't support `thread_local`, and MinGW-w64 w/ POSIX threading has a crippling bug: http://sourceforge.net/p/mingw-w64/bugs/445 -// g++ <=4.7 doesn't support thread_local either. -// Finally, iOS/ARM doesn't have support for it either, and g++/ARM allows it to compile but it's unconfirmed to actually work -#if (!defined(_MSC_VER) || _MSC_VER >= 1900) && (!defined(__MINGW32__) && !defined(__MINGW64__) || !defined(__WINPTHREADS_VERSION)) && (!defined(__GNUC__) || __GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 8)) && (!defined(__APPLE__) || !TARGET_OS_IPHONE) && !defined(__arm__) && !defined(_M_ARM) && !defined(__aarch64__) && !defined(__MVS__) -// Assume `thread_local` is fully supported in all other C++11 compilers/platforms -#define MOODYCAMEL_CPP11_THREAD_LOCAL_SUPPORTED // tentatively enabled for now; years ago several users report having problems with it on -#endif -#endif -#endif - -// VS2012 doesn't support deleted functions. -// In this case, we declare the function normally but don't define it. A link error will be generated if the function is called. -#ifndef MOODYCAMEL_DELETE_FUNCTION -#if defined(_MSC_VER) && _MSC_VER < 1800 -#define MOODYCAMEL_DELETE_FUNCTION -#else -#define MOODYCAMEL_DELETE_FUNCTION = delete -#endif -#endif - -namespace moodycamel { namespace details { -#ifndef MOODYCAMEL_ALIGNAS -// VS2013 doesn't support alignas or alignof, and align() requires a constant literal -#if defined(_MSC_VER) && _MSC_VER <= 1800 -#define MOODYCAMEL_ALIGNAS(alignment) __declspec(align(alignment)) -#define MOODYCAMEL_ALIGNOF(obj) __alignof(obj) -#define MOODYCAMEL_ALIGNED_TYPE_LIKE(T, obj) typename details::Vs2013Aligned::value, T>::type - template struct Vs2013Aligned { }; // default, unsupported alignment - template struct Vs2013Aligned<1, T> { typedef __declspec(align(1)) T type; }; - template struct Vs2013Aligned<2, T> { typedef __declspec(align(2)) T type; }; - template struct Vs2013Aligned<4, T> { typedef __declspec(align(4)) T type; }; - template struct Vs2013Aligned<8, T> { typedef __declspec(align(8)) T type; }; - template struct Vs2013Aligned<16, T> { typedef __declspec(align(16)) T type; }; - template struct Vs2013Aligned<32, T> { typedef __declspec(align(32)) T type; }; - template struct Vs2013Aligned<64, T> { typedef __declspec(align(64)) T type; }; - template struct Vs2013Aligned<128, T> { typedef __declspec(align(128)) T type; }; - template struct Vs2013Aligned<256, T> { typedef __declspec(align(256)) T type; }; -#else - template struct identity { typedef T type; }; -#define MOODYCAMEL_ALIGNAS(alignment) alignas(alignment) -#define MOODYCAMEL_ALIGNOF(obj) alignof(obj) -#define MOODYCAMEL_ALIGNED_TYPE_LIKE(T, obj) alignas(alignof(obj)) typename details::identity::type -#endif -#endif -} } - - -// TSAN can false report races in lock-free code. To enable TSAN to be used from projects that use this one, -// we can apply per-function compile-time suppression. -// See https://clang.llvm.org/docs/ThreadSanitizer.html#has-feature-thread-sanitizer -#define MOODYCAMEL_NO_TSAN -#if defined(__has_feature) - #if __has_feature(thread_sanitizer) - #undef MOODYCAMEL_NO_TSAN - #define MOODYCAMEL_NO_TSAN __attribute__((no_sanitize("thread"))) - #endif // TSAN -#endif // TSAN - -// Compiler-specific likely/unlikely hints -namespace moodycamel { namespace details { -#if defined(__GNUC__) - static inline bool (likely)(bool x) { return __builtin_expect((x), true); } - static inline bool (unlikely)(bool x) { return __builtin_expect((x), false); } -#else - static inline bool (likely)(bool x) { return x; } - static inline bool (unlikely)(bool x) { return x; } -#endif -} } - -#ifdef MOODYCAMEL_QUEUE_INTERNAL_DEBUG -#include "internal/concurrentqueue_internal_debug.h" -#endif - -namespace moodycamel { -namespace details { - template - struct const_numeric_max { - static_assert(std::is_integral::value, "const_numeric_max can only be used with integers"); - static const T value = std::numeric_limits::is_signed - ? (static_cast(1) << (sizeof(T) * CHAR_BIT - 1)) - static_cast(1) - : static_cast(-1); - }; - -#if defined(__GLIBCXX__) - typedef ::max_align_t std_max_align_t; // libstdc++ forgot to add it to std:: for a while -#else - typedef std::max_align_t std_max_align_t; // Others (e.g. MSVC) insist it can *only* be accessed via std:: -#endif - - // Some platforms have incorrectly set max_align_t to a type with <8 bytes alignment even while supporting - // 8-byte aligned scalar values (*cough* 32-bit iOS). Work around this with our own union. See issue #64. - typedef union { - std_max_align_t x; - long long y; - void* z; - } max_align_t; -} - -// Default traits for the ConcurrentQueue. To change some of the -// traits without re-implementing all of them, inherit from this -// struct and shadow the declarations you wish to be different; -// since the traits are used as a template type parameter, the -// shadowed declarations will be used where defined, and the defaults -// otherwise. -struct ConcurrentQueueDefaultTraits -{ - // General-purpose size type. std::size_t is strongly recommended. - typedef std::size_t size_t; - - // The type used for the enqueue and dequeue indices. Must be at least as - // large as size_t. Should be significantly larger than the number of elements - // you expect to hold at once, especially if you have a high turnover rate; - // for example, on 32-bit x86, if you expect to have over a hundred million - // elements or pump several million elements through your queue in a very - // short space of time, using a 32-bit type *may* trigger a race condition. - // A 64-bit int type is recommended in that case, and in practice will - // prevent a race condition no matter the usage of the queue. Note that - // whether the queue is lock-free with a 64-int type depends on the whether - // std::atomic is lock-free, which is platform-specific. - typedef std::size_t index_t; - - // Internally, all elements are enqueued and dequeued from multi-element - // blocks; this is the smallest controllable unit. If you expect few elements - // but many producers, a smaller block size should be favoured. For few producers - // and/or many elements, a larger block size is preferred. A sane default - // is provided. Must be a power of 2. - static const size_t BLOCK_SIZE = 32; - - // For explicit producers (i.e. when using a producer token), the block is - // checked for being empty by iterating through a list of flags, one per element. - // For large block sizes, this is too inefficient, and switching to an atomic - // counter-based approach is faster. The switch is made for block sizes strictly - // larger than this threshold. - static const size_t EXPLICIT_BLOCK_EMPTY_COUNTER_THRESHOLD = 32; - - // How many full blocks can be expected for a single explicit producer? This should - // reflect that number's maximum for optimal performance. Must be a power of 2. - static const size_t EXPLICIT_INITIAL_INDEX_SIZE = 32; - - // How many full blocks can be expected for a single implicit producer? This should - // reflect that number's maximum for optimal performance. Must be a power of 2. - static const size_t IMPLICIT_INITIAL_INDEX_SIZE = 32; - - // The initial size of the hash table mapping thread IDs to implicit producers. - // Note that the hash is resized every time it becomes half full. - // Must be a power of two, and either 0 or at least 1. If 0, implicit production - // (using the enqueue methods without an explicit producer token) is disabled. - static const size_t INITIAL_IMPLICIT_PRODUCER_HASH_SIZE = 32; - - // Controls the number of items that an explicit consumer (i.e. one with a token) - // must consume before it causes all consumers to rotate and move on to the next - // internal queue. - static const std::uint32_t EXPLICIT_CONSUMER_CONSUMPTION_QUOTA_BEFORE_ROTATE = 256; - - // The maximum number of elements (inclusive) that can be enqueued to a sub-queue. - // Enqueue operations that would cause this limit to be surpassed will fail. Note - // that this limit is enforced at the block level (for performance reasons), i.e. - // it's rounded up to the nearest block size. - static const size_t MAX_SUBQUEUE_SIZE = details::const_numeric_max::value; - - // The number of times to spin before sleeping when waiting on a semaphore. - // Recommended values are on the order of 1000-10000 unless the number of - // consumer threads exceeds the number of idle cores (in which case try 0-100). - // Only affects instances of the BlockingConcurrentQueue. - static const int MAX_SEMA_SPINS = 10000; - - // Whether to recycle dynamically-allocated blocks into an internal free list or - // not. If false, only pre-allocated blocks (controlled by the constructor - // arguments) will be recycled, and all others will be `free`d back to the heap. - // Note that blocks consumed by explicit producers are only freed on destruction - // of the queue (not following destruction of the token) regardless of this trait. - static const bool RECYCLE_ALLOCATED_BLOCKS = false; - - -#ifndef MCDBGQ_USE_RELACY - // Memory allocation can be customized if needed. - // malloc should return nullptr on failure, and handle alignment like std::malloc. -#if defined(malloc) || defined(free) - // Gah, this is 2015, stop defining macros that break standard code already! - // Work around malloc/free being special macros: - static inline void* WORKAROUND_malloc(size_t size) { return malloc(size); } - static inline void WORKAROUND_free(void* ptr) { return free(ptr); } - static inline void* (malloc)(size_t size) { return WORKAROUND_malloc(size); } - static inline void (free)(void* ptr) { return WORKAROUND_free(ptr); } -#else - static inline void* malloc(size_t size) { return std::malloc(size); } - static inline void free(void* ptr) { return std::free(ptr); } -#endif -#else - // Debug versions when running under the Relacy race detector (ignore - // these in user code) - static inline void* malloc(size_t size) { return rl::rl_malloc(size, $); } - static inline void free(void* ptr) { return rl::rl_free(ptr, $); } -#endif -}; - - -// When producing or consuming many elements, the most efficient way is to: -// 1) Use one of the bulk-operation methods of the queue with a token -// 2) Failing that, use the bulk-operation methods without a token -// 3) Failing that, create a token and use that with the single-item methods -// 4) Failing that, use the single-parameter methods of the queue -// Having said that, don't create tokens willy-nilly -- ideally there should be -// a maximum of one token per thread (of each kind). -struct ProducerToken; -struct ConsumerToken; - -template class ConcurrentQueue; -template class BlockingConcurrentQueue; -class ConcurrentQueueTests; - - -namespace details -{ - struct ConcurrentQueueProducerTypelessBase - { - ConcurrentQueueProducerTypelessBase* next; - std::atomic inactive; - ProducerToken* token; - - ConcurrentQueueProducerTypelessBase() - : next(nullptr), inactive(false), token(nullptr) - { - } - }; - - template struct _hash_32_or_64 { - static inline std::uint32_t hash(std::uint32_t h) - { - // MurmurHash3 finalizer -- see https://code.google.com/p/smhasher/source/browse/trunk/MurmurHash3.cpp - // Since the thread ID is already unique, all we really want to do is propagate that - // uniqueness evenly across all the bits, so that we can use a subset of the bits while - // reducing collisions significantly - h ^= h >> 16; - h *= 0x85ebca6b; - h ^= h >> 13; - h *= 0xc2b2ae35; - return h ^ (h >> 16); - } - }; - template<> struct _hash_32_or_64<1> { - static inline std::uint64_t hash(std::uint64_t h) - { - h ^= h >> 33; - h *= 0xff51afd7ed558ccd; - h ^= h >> 33; - h *= 0xc4ceb9fe1a85ec53; - return h ^ (h >> 33); - } - }; - template struct hash_32_or_64 : public _hash_32_or_64<(size > 4)> { }; - - static inline size_t hash_thread_id(thread_id_t id) - { - static_assert(sizeof(thread_id_t) <= 8, "Expected a platform where thread IDs are at most 64-bit values"); - return static_cast(hash_32_or_64::thread_id_hash_t)>::hash( - thread_id_converter::prehash(id))); - } - - template - static inline bool circular_less_than(T a, T b) - { - static_assert(std::is_integral::value && !std::numeric_limits::is_signed, "circular_less_than is intended to be used only with unsigned integer types"); - return static_cast(a - b) > static_cast(static_cast(1) << (static_cast(sizeof(T) * CHAR_BIT - 1))); - // Note: extra parens around rhs of operator<< is MSVC bug: https://developercommunity2.visualstudio.com/t/C4554-triggers-when-both-lhs-and-rhs-is/10034931 - // silencing the bug requires #pragma warning(disable: 4554) around the calling code and has no effect when done here. - } - - template - static inline char* align_for(char* ptr) - { - const std::size_t alignment = std::alignment_of::value; - return ptr + (alignment - (reinterpret_cast(ptr) % alignment)) % alignment; - } - - template - static inline T ceil_to_pow_2(T x) - { - static_assert(std::is_integral::value && !std::numeric_limits::is_signed, "ceil_to_pow_2 is intended to be used only with unsigned integer types"); - - // Adapted from http://graphics.stanford.edu/~seander/bithacks.html#RoundUpPowerOf2 - --x; - x |= x >> 1; - x |= x >> 2; - x |= x >> 4; - for (std::size_t i = 1; i < sizeof(T); i <<= 1) { - x |= x >> (i << 3); - } - ++x; - return x; - } - - template - static inline void swap_relaxed(std::atomic& left, std::atomic& right) - { - T temp = std::move(left.load(std::memory_order_relaxed)); - left.store(std::move(right.load(std::memory_order_relaxed)), std::memory_order_relaxed); - right.store(std::move(temp), std::memory_order_relaxed); - } - - template - static inline T const& nomove(T const& x) - { - return x; - } - - template - struct nomove_if - { - template - static inline T const& eval(T const& x) - { - return x; - } - }; - - template<> - struct nomove_if - { - template - static inline auto eval(U&& x) - -> decltype(std::forward(x)) - { - return std::forward(x); - } - }; - - template - static inline auto deref_noexcept(It& it) MOODYCAMEL_NOEXCEPT -> decltype(*it) - { - return *it; - } - -#if defined(__clang__) || !defined(__GNUC__) || __GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 8) - template struct is_trivially_destructible : std::is_trivially_destructible { }; -#else - template struct is_trivially_destructible : std::has_trivial_destructor { }; -#endif - -#ifdef MOODYCAMEL_CPP11_THREAD_LOCAL_SUPPORTED -#ifdef MCDBGQ_USE_RELACY - typedef RelacyThreadExitListener ThreadExitListener; - typedef RelacyThreadExitNotifier ThreadExitNotifier; -#else - class ThreadExitNotifier; - - struct ThreadExitListener - { - typedef void (*callback_t)(void*); - callback_t callback; - void* userData; - - ThreadExitListener* next; // reserved for use by the ThreadExitNotifier - ThreadExitNotifier* chain; // reserved for use by the ThreadExitNotifier - }; - - class ThreadExitNotifier - { - public: - static void subscribe(ThreadExitListener* listener) - { - auto& tlsInst = instance(); - std::lock_guard guard(mutex()); - listener->next = tlsInst.tail; - listener->chain = &tlsInst; - tlsInst.tail = listener; - } - - static void unsubscribe(ThreadExitListener* listener) - { - std::lock_guard guard(mutex()); - if (!listener->chain) { - return; // race with ~ThreadExitNotifier - } - auto& tlsInst = *listener->chain; - listener->chain = nullptr; - ThreadExitListener** prev = &tlsInst.tail; - for (auto ptr = tlsInst.tail; ptr != nullptr; ptr = ptr->next) { - if (ptr == listener) { - *prev = ptr->next; - break; - } - prev = &ptr->next; - } - } - - private: - ThreadExitNotifier() : tail(nullptr) { } - ThreadExitNotifier(ThreadExitNotifier const&) MOODYCAMEL_DELETE_FUNCTION; - ThreadExitNotifier& operator=(ThreadExitNotifier const&) MOODYCAMEL_DELETE_FUNCTION; - - ~ThreadExitNotifier() - { - // This thread is about to exit, let everyone know! - assert(this == &instance() && "If this assert fails, you likely have a buggy compiler! Change the preprocessor conditions such that MOODYCAMEL_CPP11_THREAD_LOCAL_SUPPORTED is no longer defined."); - std::lock_guard guard(mutex()); - for (auto ptr = tail; ptr != nullptr; ptr = ptr->next) { - ptr->chain = nullptr; - ptr->callback(ptr->userData); - } - } - - // Thread-local - static inline ThreadExitNotifier& instance() - { - static thread_local ThreadExitNotifier notifier; - return notifier; - } - - static inline std::mutex& mutex() - { - // Must be static because the ThreadExitNotifier could be destroyed while unsubscribe is called - static std::mutex mutex; - return mutex; - } - - private: - ThreadExitListener* tail; - }; -#endif -#endif - - template struct static_is_lock_free_num { enum { value = 0 }; }; - template<> struct static_is_lock_free_num { enum { value = ATOMIC_CHAR_LOCK_FREE }; }; - template<> struct static_is_lock_free_num { enum { value = ATOMIC_SHORT_LOCK_FREE }; }; - template<> struct static_is_lock_free_num { enum { value = ATOMIC_INT_LOCK_FREE }; }; - template<> struct static_is_lock_free_num { enum { value = ATOMIC_LONG_LOCK_FREE }; }; - template<> struct static_is_lock_free_num { enum { value = ATOMIC_LLONG_LOCK_FREE }; }; - template struct static_is_lock_free : static_is_lock_free_num::type> { }; - template<> struct static_is_lock_free { enum { value = ATOMIC_BOOL_LOCK_FREE }; }; - template struct static_is_lock_free { enum { value = ATOMIC_POINTER_LOCK_FREE }; }; -} - - -struct ProducerToken -{ - template - explicit ProducerToken(ConcurrentQueue& queue); - - template - explicit ProducerToken(BlockingConcurrentQueue& queue); - - ProducerToken(ProducerToken&& other) MOODYCAMEL_NOEXCEPT - : producer(other.producer) - { - other.producer = nullptr; - if (producer != nullptr) { - producer->token = this; - } - } - - inline ProducerToken& operator=(ProducerToken&& other) MOODYCAMEL_NOEXCEPT - { - swap(other); - return *this; - } - - void swap(ProducerToken& other) MOODYCAMEL_NOEXCEPT - { - std::swap(producer, other.producer); - if (producer != nullptr) { - producer->token = this; - } - if (other.producer != nullptr) { - other.producer->token = &other; - } - } - - // A token is always valid unless: - // 1) Memory allocation failed during construction - // 2) It was moved via the move constructor - // (Note: assignment does a swap, leaving both potentially valid) - // 3) The associated queue was destroyed - // Note that if valid() returns true, that only indicates - // that the token is valid for use with a specific queue, - // but not which one; that's up to the user to track. - inline bool valid() const { return producer != nullptr; } - - ~ProducerToken() - { - if (producer != nullptr) { - producer->token = nullptr; - producer->inactive.store(true, std::memory_order_release); - } - } - - // Disable copying and assignment - ProducerToken(ProducerToken const&) MOODYCAMEL_DELETE_FUNCTION; - ProducerToken& operator=(ProducerToken const&) MOODYCAMEL_DELETE_FUNCTION; - -private: - template friend class ConcurrentQueue; - friend class ConcurrentQueueTests; - -protected: - details::ConcurrentQueueProducerTypelessBase* producer; -}; - - -struct ConsumerToken -{ - template - explicit ConsumerToken(ConcurrentQueue& q); - - template - explicit ConsumerToken(BlockingConcurrentQueue& q); - - ConsumerToken(ConsumerToken&& other) MOODYCAMEL_NOEXCEPT - : initialOffset(other.initialOffset), lastKnownGlobalOffset(other.lastKnownGlobalOffset), itemsConsumedFromCurrent(other.itemsConsumedFromCurrent), currentProducer(other.currentProducer), desiredProducer(other.desiredProducer) - { - } - - inline ConsumerToken& operator=(ConsumerToken&& other) MOODYCAMEL_NOEXCEPT - { - swap(other); - return *this; - } - - void swap(ConsumerToken& other) MOODYCAMEL_NOEXCEPT - { - std::swap(initialOffset, other.initialOffset); - std::swap(lastKnownGlobalOffset, other.lastKnownGlobalOffset); - std::swap(itemsConsumedFromCurrent, other.itemsConsumedFromCurrent); - std::swap(currentProducer, other.currentProducer); - std::swap(desiredProducer, other.desiredProducer); - } - - // Disable copying and assignment - ConsumerToken(ConsumerToken const&) MOODYCAMEL_DELETE_FUNCTION; - ConsumerToken& operator=(ConsumerToken const&) MOODYCAMEL_DELETE_FUNCTION; - -private: - template friend class ConcurrentQueue; - friend class ConcurrentQueueTests; - -private: // but shared with ConcurrentQueue - std::uint32_t initialOffset; - std::uint32_t lastKnownGlobalOffset; - std::uint32_t itemsConsumedFromCurrent; - details::ConcurrentQueueProducerTypelessBase* currentProducer; - details::ConcurrentQueueProducerTypelessBase* desiredProducer; -}; - -// Need to forward-declare this swap because it's in a namespace. -// See http://stackoverflow.com/questions/4492062/why-does-a-c-friend-class-need-a-forward-declaration-only-in-other-namespaces -template -inline void swap(typename ConcurrentQueue::ImplicitProducerKVP& a, typename ConcurrentQueue::ImplicitProducerKVP& b) MOODYCAMEL_NOEXCEPT; - - -template -class ConcurrentQueue -{ -public: - typedef ::moodycamel::ProducerToken producer_token_t; - typedef ::moodycamel::ConsumerToken consumer_token_t; - - typedef typename Traits::index_t index_t; - typedef typename Traits::size_t size_t; - - static const size_t BLOCK_SIZE = static_cast(Traits::BLOCK_SIZE); - static const size_t EXPLICIT_BLOCK_EMPTY_COUNTER_THRESHOLD = static_cast(Traits::EXPLICIT_BLOCK_EMPTY_COUNTER_THRESHOLD); - static const size_t EXPLICIT_INITIAL_INDEX_SIZE = static_cast(Traits::EXPLICIT_INITIAL_INDEX_SIZE); - static const size_t IMPLICIT_INITIAL_INDEX_SIZE = static_cast(Traits::IMPLICIT_INITIAL_INDEX_SIZE); - static const size_t INITIAL_IMPLICIT_PRODUCER_HASH_SIZE = static_cast(Traits::INITIAL_IMPLICIT_PRODUCER_HASH_SIZE); - static const std::uint32_t EXPLICIT_CONSUMER_CONSUMPTION_QUOTA_BEFORE_ROTATE = static_cast(Traits::EXPLICIT_CONSUMER_CONSUMPTION_QUOTA_BEFORE_ROTATE); -#ifdef _MSC_VER -#pragma warning(push) -#pragma warning(disable: 4307) // + integral constant overflow (that's what the ternary expression is for!) -#pragma warning(disable: 4309) // static_cast: Truncation of constant value -#endif - static const size_t MAX_SUBQUEUE_SIZE = (details::const_numeric_max::value - static_cast(Traits::MAX_SUBQUEUE_SIZE) < BLOCK_SIZE) ? details::const_numeric_max::value : ((static_cast(Traits::MAX_SUBQUEUE_SIZE) + (BLOCK_SIZE - 1)) / BLOCK_SIZE * BLOCK_SIZE); -#ifdef _MSC_VER -#pragma warning(pop) -#endif - - static_assert(!std::numeric_limits::is_signed && std::is_integral::value, "Traits::size_t must be an unsigned integral type"); - static_assert(!std::numeric_limits::is_signed && std::is_integral::value, "Traits::index_t must be an unsigned integral type"); - static_assert(sizeof(index_t) >= sizeof(size_t), "Traits::index_t must be at least as wide as Traits::size_t"); - static_assert((BLOCK_SIZE > 1) && !(BLOCK_SIZE & (BLOCK_SIZE - 1)), "Traits::BLOCK_SIZE must be a power of 2 (and at least 2)"); - static_assert((EXPLICIT_BLOCK_EMPTY_COUNTER_THRESHOLD > 1) && !(EXPLICIT_BLOCK_EMPTY_COUNTER_THRESHOLD & (EXPLICIT_BLOCK_EMPTY_COUNTER_THRESHOLD - 1)), "Traits::EXPLICIT_BLOCK_EMPTY_COUNTER_THRESHOLD must be a power of 2 (and greater than 1)"); - static_assert((EXPLICIT_INITIAL_INDEX_SIZE > 1) && !(EXPLICIT_INITIAL_INDEX_SIZE & (EXPLICIT_INITIAL_INDEX_SIZE - 1)), "Traits::EXPLICIT_INITIAL_INDEX_SIZE must be a power of 2 (and greater than 1)"); - static_assert((IMPLICIT_INITIAL_INDEX_SIZE > 1) && !(IMPLICIT_INITIAL_INDEX_SIZE & (IMPLICIT_INITIAL_INDEX_SIZE - 1)), "Traits::IMPLICIT_INITIAL_INDEX_SIZE must be a power of 2 (and greater than 1)"); - static_assert((INITIAL_IMPLICIT_PRODUCER_HASH_SIZE == 0) || !(INITIAL_IMPLICIT_PRODUCER_HASH_SIZE & (INITIAL_IMPLICIT_PRODUCER_HASH_SIZE - 1)), "Traits::INITIAL_IMPLICIT_PRODUCER_HASH_SIZE must be a power of 2"); - static_assert(INITIAL_IMPLICIT_PRODUCER_HASH_SIZE == 0 || INITIAL_IMPLICIT_PRODUCER_HASH_SIZE >= 1, "Traits::INITIAL_IMPLICIT_PRODUCER_HASH_SIZE must be at least 1 (or 0 to disable implicit enqueueing)"); - -public: - // Creates a queue with at least `capacity` element slots; note that the - // actual number of elements that can be inserted without additional memory - // allocation depends on the number of producers and the block size (e.g. if - // the block size is equal to `capacity`, only a single block will be allocated - // up-front, which means only a single producer will be able to enqueue elements - // without an extra allocation -- blocks aren't shared between producers). - // This method is not thread safe -- it is up to the user to ensure that the - // queue is fully constructed before it starts being used by other threads (this - // includes making the memory effects of construction visible, possibly with a - // memory barrier). - explicit ConcurrentQueue(size_t capacity = 32 * BLOCK_SIZE) - : producerListTail(nullptr), - producerCount(0), - initialBlockPoolIndex(0), - nextExplicitConsumerId(0), - globalExplicitConsumerOffset(0) - { - implicitProducerHashResizeInProgress.clear(std::memory_order_relaxed); - populate_initial_implicit_producer_hash(); - populate_initial_block_list(capacity / BLOCK_SIZE + ((capacity & (BLOCK_SIZE - 1)) == 0 ? 0 : 1)); - -#ifdef MOODYCAMEL_QUEUE_INTERNAL_DEBUG - // Track all the producers using a fully-resolved typed list for - // each kind; this makes it possible to debug them starting from - // the root queue object (otherwise wacky casts are needed that - // don't compile in the debugger's expression evaluator). - explicitProducers.store(nullptr, std::memory_order_relaxed); - implicitProducers.store(nullptr, std::memory_order_relaxed); -#endif - } - - // Computes the correct amount of pre-allocated blocks for you based - // on the minimum number of elements you want available at any given - // time, and the maximum concurrent number of each type of producer. - ConcurrentQueue(size_t minCapacity, size_t maxExplicitProducers, size_t maxImplicitProducers) - : producerListTail(nullptr), - producerCount(0), - initialBlockPoolIndex(0), - nextExplicitConsumerId(0), - globalExplicitConsumerOffset(0) - { - implicitProducerHashResizeInProgress.clear(std::memory_order_relaxed); - populate_initial_implicit_producer_hash(); - size_t blocks = (((minCapacity + BLOCK_SIZE - 1) / BLOCK_SIZE) - 1) * (maxExplicitProducers + 1) + 2 * (maxExplicitProducers + maxImplicitProducers); - populate_initial_block_list(blocks); - -#ifdef MOODYCAMEL_QUEUE_INTERNAL_DEBUG - explicitProducers.store(nullptr, std::memory_order_relaxed); - implicitProducers.store(nullptr, std::memory_order_relaxed); -#endif - } - - // Note: The queue should not be accessed concurrently while it's - // being deleted. It's up to the user to synchronize this. - // This method is not thread safe. - ~ConcurrentQueue() - { - // Destroy producers - auto ptr = producerListTail.load(std::memory_order_relaxed); - while (ptr != nullptr) { - auto next = ptr->next_prod(); - if (ptr->token != nullptr) { - ptr->token->producer = nullptr; - } - destroy(ptr); - ptr = next; - } - - // Destroy implicit producer hash tables - MOODYCAMEL_CONSTEXPR_IF (INITIAL_IMPLICIT_PRODUCER_HASH_SIZE != 0) { - auto hash = implicitProducerHash.load(std::memory_order_relaxed); - while (hash != nullptr) { - auto prev = hash->prev; - if (prev != nullptr) { // The last hash is part of this object and was not allocated dynamically - for (size_t i = 0; i != hash->capacity; ++i) { - hash->entries[i].~ImplicitProducerKVP(); - } - hash->~ImplicitProducerHash(); - (Traits::free)(hash); - } - hash = prev; - } - } - - // Destroy global free list - auto block = freeList.head_unsafe(); - while (block != nullptr) { - auto next = block->freeListNext.load(std::memory_order_relaxed); - if (block->dynamicallyAllocated) { - destroy(block); - } - block = next; - } - - // Destroy initial free list - destroy_array(initialBlockPool, initialBlockPoolSize); - } - - // Disable copying and copy assignment - ConcurrentQueue(ConcurrentQueue const&) MOODYCAMEL_DELETE_FUNCTION; - ConcurrentQueue& operator=(ConcurrentQueue const&) MOODYCAMEL_DELETE_FUNCTION; - - // Moving is supported, but note that it is *not* a thread-safe operation. - // Nobody can use the queue while it's being moved, and the memory effects - // of that move must be propagated to other threads before they can use it. - // Note: When a queue is moved, its tokens are still valid but can only be - // used with the destination queue (i.e. semantically they are moved along - // with the queue itself). - ConcurrentQueue(ConcurrentQueue&& other) MOODYCAMEL_NOEXCEPT - : producerListTail(other.producerListTail.load(std::memory_order_relaxed)), - producerCount(other.producerCount.load(std::memory_order_relaxed)), - initialBlockPoolIndex(other.initialBlockPoolIndex.load(std::memory_order_relaxed)), - initialBlockPool(other.initialBlockPool), - initialBlockPoolSize(other.initialBlockPoolSize), - freeList(std::move(other.freeList)), - nextExplicitConsumerId(other.nextExplicitConsumerId.load(std::memory_order_relaxed)), - globalExplicitConsumerOffset(other.globalExplicitConsumerOffset.load(std::memory_order_relaxed)) - { - // Move the other one into this, and leave the other one as an empty queue - implicitProducerHashResizeInProgress.clear(std::memory_order_relaxed); - populate_initial_implicit_producer_hash(); - swap_implicit_producer_hashes(other); - - other.producerListTail.store(nullptr, std::memory_order_relaxed); - other.producerCount.store(0, std::memory_order_relaxed); - other.nextExplicitConsumerId.store(0, std::memory_order_relaxed); - other.globalExplicitConsumerOffset.store(0, std::memory_order_relaxed); - -#ifdef MOODYCAMEL_QUEUE_INTERNAL_DEBUG - explicitProducers.store(other.explicitProducers.load(std::memory_order_relaxed), std::memory_order_relaxed); - other.explicitProducers.store(nullptr, std::memory_order_relaxed); - implicitProducers.store(other.implicitProducers.load(std::memory_order_relaxed), std::memory_order_relaxed); - other.implicitProducers.store(nullptr, std::memory_order_relaxed); -#endif - - other.initialBlockPoolIndex.store(0, std::memory_order_relaxed); - other.initialBlockPoolSize = 0; - other.initialBlockPool = nullptr; - - reown_producers(); - } - - inline ConcurrentQueue& operator=(ConcurrentQueue&& other) MOODYCAMEL_NOEXCEPT - { - return swap_internal(other); - } - - // Swaps this queue's state with the other's. Not thread-safe. - // Swapping two queues does not invalidate their tokens, however - // the tokens that were created for one queue must be used with - // only the swapped queue (i.e. the tokens are tied to the - // queue's movable state, not the object itself). - inline void swap(ConcurrentQueue& other) MOODYCAMEL_NOEXCEPT - { - swap_internal(other); - } - -private: - ConcurrentQueue& swap_internal(ConcurrentQueue& other) - { - if (this == &other) { - return *this; - } - - details::swap_relaxed(producerListTail, other.producerListTail); - details::swap_relaxed(producerCount, other.producerCount); - details::swap_relaxed(initialBlockPoolIndex, other.initialBlockPoolIndex); - std::swap(initialBlockPool, other.initialBlockPool); - std::swap(initialBlockPoolSize, other.initialBlockPoolSize); - freeList.swap(other.freeList); - details::swap_relaxed(nextExplicitConsumerId, other.nextExplicitConsumerId); - details::swap_relaxed(globalExplicitConsumerOffset, other.globalExplicitConsumerOffset); - - swap_implicit_producer_hashes(other); - - reown_producers(); - other.reown_producers(); - -#ifdef MOODYCAMEL_QUEUE_INTERNAL_DEBUG - details::swap_relaxed(explicitProducers, other.explicitProducers); - details::swap_relaxed(implicitProducers, other.implicitProducers); -#endif - - return *this; - } - -public: - // Enqueues a single item (by copying it). - // Allocates memory if required. Only fails if memory allocation fails (or implicit - // production is disabled because Traits::INITIAL_IMPLICIT_PRODUCER_HASH_SIZE is 0, - // or Traits::MAX_SUBQUEUE_SIZE has been defined and would be surpassed). - // Thread-safe. - inline bool enqueue(T const& item) - { - MOODYCAMEL_CONSTEXPR_IF (INITIAL_IMPLICIT_PRODUCER_HASH_SIZE == 0) return false; - else return inner_enqueue(item); - } - - // Enqueues a single item (by moving it, if possible). - // Allocates memory if required. Only fails if memory allocation fails (or implicit - // production is disabled because Traits::INITIAL_IMPLICIT_PRODUCER_HASH_SIZE is 0, - // or Traits::MAX_SUBQUEUE_SIZE has been defined and would be surpassed). - // Thread-safe. - inline bool enqueue(T&& item) - { - MOODYCAMEL_CONSTEXPR_IF (INITIAL_IMPLICIT_PRODUCER_HASH_SIZE == 0) return false; - else return inner_enqueue(std::move(item)); - } - - // Enqueues a single item (by copying it) using an explicit producer token. - // Allocates memory if required. Only fails if memory allocation fails (or - // Traits::MAX_SUBQUEUE_SIZE has been defined and would be surpassed). - // Thread-safe. - inline bool enqueue(producer_token_t const& token, T const& item) - { - return inner_enqueue(token, item); - } - - // Enqueues a single item (by moving it, if possible) using an explicit producer token. - // Allocates memory if required. Only fails if memory allocation fails (or - // Traits::MAX_SUBQUEUE_SIZE has been defined and would be surpassed). - // Thread-safe. - inline bool enqueue(producer_token_t const& token, T&& item) - { - return inner_enqueue(token, std::move(item)); - } - - // Enqueues several items. - // Allocates memory if required. Only fails if memory allocation fails (or - // implicit production is disabled because Traits::INITIAL_IMPLICIT_PRODUCER_HASH_SIZE - // is 0, or Traits::MAX_SUBQUEUE_SIZE has been defined and would be surpassed). - // Note: Use std::make_move_iterator if the elements should be moved instead of copied. - // Thread-safe. - template - bool enqueue_bulk(It itemFirst, size_t count) - { - MOODYCAMEL_CONSTEXPR_IF (INITIAL_IMPLICIT_PRODUCER_HASH_SIZE == 0) return false; - else return inner_enqueue_bulk(itemFirst, count); - } - - // Enqueues several items using an explicit producer token. - // Allocates memory if required. Only fails if memory allocation fails - // (or Traits::MAX_SUBQUEUE_SIZE has been defined and would be surpassed). - // Note: Use std::make_move_iterator if the elements should be moved - // instead of copied. - // Thread-safe. - template - bool enqueue_bulk(producer_token_t const& token, It itemFirst, size_t count) - { - return inner_enqueue_bulk(token, itemFirst, count); - } - - // Enqueues a single item (by copying it). - // Does not allocate memory. Fails if not enough room to enqueue (or implicit - // production is disabled because Traits::INITIAL_IMPLICIT_PRODUCER_HASH_SIZE - // is 0). - // Thread-safe. - inline bool try_enqueue(T const& item) - { - MOODYCAMEL_CONSTEXPR_IF (INITIAL_IMPLICIT_PRODUCER_HASH_SIZE == 0) return false; - else return inner_enqueue(item); - } - - // Enqueues a single item (by moving it, if possible). - // Does not allocate memory (except for one-time implicit producer). - // Fails if not enough room to enqueue (or implicit production is - // disabled because Traits::INITIAL_IMPLICIT_PRODUCER_HASH_SIZE is 0). - // Thread-safe. - inline bool try_enqueue(T&& item) - { - MOODYCAMEL_CONSTEXPR_IF (INITIAL_IMPLICIT_PRODUCER_HASH_SIZE == 0) return false; - else return inner_enqueue(std::move(item)); - } - - // Enqueues a single item (by copying it) using an explicit producer token. - // Does not allocate memory. Fails if not enough room to enqueue. - // Thread-safe. - inline bool try_enqueue(producer_token_t const& token, T const& item) - { - return inner_enqueue(token, item); - } - - // Enqueues a single item (by moving it, if possible) using an explicit producer token. - // Does not allocate memory. Fails if not enough room to enqueue. - // Thread-safe. - inline bool try_enqueue(producer_token_t const& token, T&& item) - { - return inner_enqueue(token, std::move(item)); - } - - // Enqueues several items. - // Does not allocate memory (except for one-time implicit producer). - // Fails if not enough room to enqueue (or implicit production is - // disabled because Traits::INITIAL_IMPLICIT_PRODUCER_HASH_SIZE is 0). - // Note: Use std::make_move_iterator if the elements should be moved - // instead of copied. - // Thread-safe. - template - bool try_enqueue_bulk(It itemFirst, size_t count) - { - MOODYCAMEL_CONSTEXPR_IF (INITIAL_IMPLICIT_PRODUCER_HASH_SIZE == 0) return false; - else return inner_enqueue_bulk(itemFirst, count); - } - - // Enqueues several items using an explicit producer token. - // Does not allocate memory. Fails if not enough room to enqueue. - // Note: Use std::make_move_iterator if the elements should be moved - // instead of copied. - // Thread-safe. - template - bool try_enqueue_bulk(producer_token_t const& token, It itemFirst, size_t count) - { - return inner_enqueue_bulk(token, itemFirst, count); - } - - - - // Attempts to dequeue from the queue. - // Returns false if all producer streams appeared empty at the time they - // were checked (so, the queue is likely but not guaranteed to be empty). - // Never allocates. Thread-safe. - template - bool try_dequeue(U& item) - { - // Instead of simply trying each producer in turn (which could cause needless contention on the first - // producer), we score them heuristically. - size_t nonEmptyCount = 0; - ProducerBase* best = nullptr; - size_t bestSize = 0; - for (auto ptr = producerListTail.load(std::memory_order_acquire); nonEmptyCount < 3 && ptr != nullptr; ptr = ptr->next_prod()) { - auto size = ptr->size_approx(); - if (size > 0) { - if (size > bestSize) { - bestSize = size; - best = ptr; - } - ++nonEmptyCount; - } - } - - // If there was at least one non-empty queue but it appears empty at the time - // we try to dequeue from it, we need to make sure every queue's been tried - if (nonEmptyCount > 0) { - if ((details::likely)(best->dequeue(item))) { - return true; - } - for (auto ptr = producerListTail.load(std::memory_order_acquire); ptr != nullptr; ptr = ptr->next_prod()) { - if (ptr != best && ptr->dequeue(item)) { - return true; - } - } - } - return false; - } - - // Attempts to dequeue from the queue. - // Returns false if all producer streams appeared empty at the time they - // were checked (so, the queue is likely but not guaranteed to be empty). - // This differs from the try_dequeue(item) method in that this one does - // not attempt to reduce contention by interleaving the order that producer - // streams are dequeued from. So, using this method can reduce overall throughput - // under contention, but will give more predictable results in single-threaded - // consumer scenarios. This is mostly only useful for internal unit tests. - // Never allocates. Thread-safe. - template - bool try_dequeue_non_interleaved(U& item) - { - for (auto ptr = producerListTail.load(std::memory_order_acquire); ptr != nullptr; ptr = ptr->next_prod()) { - if (ptr->dequeue(item)) { - return true; - } - } - return false; - } - - // Attempts to dequeue from the queue using an explicit consumer token. - // Returns false if all producer streams appeared empty at the time they - // were checked (so, the queue is likely but not guaranteed to be empty). - // Never allocates. Thread-safe. - template - bool try_dequeue(consumer_token_t& token, U& item) - { - // The idea is roughly as follows: - // Every 256 items from one producer, make everyone rotate (increase the global offset) -> this means the highest efficiency consumer dictates the rotation speed of everyone else, more or less - // If you see that the global offset has changed, you must reset your consumption counter and move to your designated place - // If there's no items where you're supposed to be, keep moving until you find a producer with some items - // If the global offset has not changed but you've run out of items to consume, move over from your current position until you find an producer with something in it - - if (token.desiredProducer == nullptr || token.lastKnownGlobalOffset != globalExplicitConsumerOffset.load(std::memory_order_relaxed)) { - if (!update_current_producer_after_rotation(token)) { - return false; - } - } - - // If there was at least one non-empty queue but it appears empty at the time - // we try to dequeue from it, we need to make sure every queue's been tried - if (static_cast(token.currentProducer)->dequeue(item)) { - if (++token.itemsConsumedFromCurrent == EXPLICIT_CONSUMER_CONSUMPTION_QUOTA_BEFORE_ROTATE) { - globalExplicitConsumerOffset.fetch_add(1, std::memory_order_relaxed); - } - return true; - } - - auto tail = producerListTail.load(std::memory_order_acquire); - auto ptr = static_cast(token.currentProducer)->next_prod(); - if (ptr == nullptr) { - ptr = tail; - } - while (ptr != static_cast(token.currentProducer)) { - if (ptr->dequeue(item)) { - token.currentProducer = ptr; - token.itemsConsumedFromCurrent = 1; - return true; - } - ptr = ptr->next_prod(); - if (ptr == nullptr) { - ptr = tail; - } - } - return false; - } - - // Attempts to dequeue several elements from the queue. - // Returns the number of items actually dequeued. - // Returns 0 if all producer streams appeared empty at the time they - // were checked (so, the queue is likely but not guaranteed to be empty). - // Never allocates. Thread-safe. - template - size_t try_dequeue_bulk(It itemFirst, size_t max) - { - size_t count = 0; - for (auto ptr = producerListTail.load(std::memory_order_acquire); ptr != nullptr; ptr = ptr->next_prod()) { - count += ptr->dequeue_bulk(itemFirst, max - count); - if (count == max) { - break; - } - } - return count; - } - - // Attempts to dequeue several elements from the queue using an explicit consumer token. - // Returns the number of items actually dequeued. - // Returns 0 if all producer streams appeared empty at the time they - // were checked (so, the queue is likely but not guaranteed to be empty). - // Never allocates. Thread-safe. - template - size_t try_dequeue_bulk(consumer_token_t& token, It itemFirst, size_t max) - { - if (token.desiredProducer == nullptr || token.lastKnownGlobalOffset != globalExplicitConsumerOffset.load(std::memory_order_relaxed)) { - if (!update_current_producer_after_rotation(token)) { - return 0; - } - } - - size_t count = static_cast(token.currentProducer)->dequeue_bulk(itemFirst, max); - if (count == max) { - if ((token.itemsConsumedFromCurrent += static_cast(max)) >= EXPLICIT_CONSUMER_CONSUMPTION_QUOTA_BEFORE_ROTATE) { - globalExplicitConsumerOffset.fetch_add(1, std::memory_order_relaxed); - } - return max; - } - token.itemsConsumedFromCurrent += static_cast(count); - max -= count; - - auto tail = producerListTail.load(std::memory_order_acquire); - auto ptr = static_cast(token.currentProducer)->next_prod(); - if (ptr == nullptr) { - ptr = tail; - } - while (ptr != static_cast(token.currentProducer)) { - auto dequeued = ptr->dequeue_bulk(itemFirst, max); - count += dequeued; - if (dequeued != 0) { - token.currentProducer = ptr; - token.itemsConsumedFromCurrent = static_cast(dequeued); - } - if (dequeued == max) { - break; - } - max -= dequeued; - ptr = ptr->next_prod(); - if (ptr == nullptr) { - ptr = tail; - } - } - return count; - } - - - - // Attempts to dequeue from a specific producer's inner queue. - // If you happen to know which producer you want to dequeue from, this - // is significantly faster than using the general-case try_dequeue methods. - // Returns false if the producer's queue appeared empty at the time it - // was checked (so, the queue is likely but not guaranteed to be empty). - // Never allocates. Thread-safe. - template - inline bool try_dequeue_from_producer(producer_token_t const& producer, U& item) - { - return static_cast(producer.producer)->dequeue(item); - } - - // Attempts to dequeue several elements from a specific producer's inner queue. - // Returns the number of items actually dequeued. - // If you happen to know which producer you want to dequeue from, this - // is significantly faster than using the general-case try_dequeue methods. - // Returns 0 if the producer's queue appeared empty at the time it - // was checked (so, the queue is likely but not guaranteed to be empty). - // Never allocates. Thread-safe. - template - inline size_t try_dequeue_bulk_from_producer(producer_token_t const& producer, It itemFirst, size_t max) - { - return static_cast(producer.producer)->dequeue_bulk(itemFirst, max); - } - - - // Returns an estimate of the total number of elements currently in the queue. This - // estimate is only accurate if the queue has completely stabilized before it is called - // (i.e. all enqueue and dequeue operations have completed and their memory effects are - // visible on the calling thread, and no further operations start while this method is - // being called). - // Thread-safe. - size_t size_approx() const - { - size_t size = 0; - for (auto ptr = producerListTail.load(std::memory_order_acquire); ptr != nullptr; ptr = ptr->next_prod()) { - size += ptr->size_approx(); - } - return size; - } - - - // Returns true if the underlying atomic variables used by - // the queue are lock-free (they should be on most platforms). - // Thread-safe. - static constexpr bool is_lock_free() - { - return - details::static_is_lock_free::value == 2 && - details::static_is_lock_free::value == 2 && - details::static_is_lock_free::value == 2 && - details::static_is_lock_free::value == 2 && - details::static_is_lock_free::value == 2 && - details::static_is_lock_free::thread_id_numeric_size_t>::value == 2; - } - - -private: - friend struct ProducerToken; - friend struct ConsumerToken; - struct ExplicitProducer; - friend struct ExplicitProducer; - struct ImplicitProducer; - friend struct ImplicitProducer; - friend class ConcurrentQueueTests; - - enum AllocationMode { CanAlloc, CannotAlloc }; - - - /////////////////////////////// - // Queue methods - /////////////////////////////// - - template - inline bool inner_enqueue(producer_token_t const& token, U&& element) - { - return static_cast(token.producer)->ConcurrentQueue::ExplicitProducer::template enqueue(std::forward(element)); - } - - template - inline bool inner_enqueue(U&& element) - { - auto producer = get_or_add_implicit_producer(); - return producer == nullptr ? false : producer->ConcurrentQueue::ImplicitProducer::template enqueue(std::forward(element)); - } - - template - inline bool inner_enqueue_bulk(producer_token_t const& token, It itemFirst, size_t count) - { - return static_cast(token.producer)->ConcurrentQueue::ExplicitProducer::template enqueue_bulk(itemFirst, count); - } - - template - inline bool inner_enqueue_bulk(It itemFirst, size_t count) - { - auto producer = get_or_add_implicit_producer(); - return producer == nullptr ? false : producer->ConcurrentQueue::ImplicitProducer::template enqueue_bulk(itemFirst, count); - } - - inline bool update_current_producer_after_rotation(consumer_token_t& token) - { - // Ah, there's been a rotation, figure out where we should be! - auto tail = producerListTail.load(std::memory_order_acquire); - if (token.desiredProducer == nullptr && tail == nullptr) { - return false; - } - auto prodCount = producerCount.load(std::memory_order_relaxed); - auto globalOffset = globalExplicitConsumerOffset.load(std::memory_order_relaxed); - if ((details::unlikely)(token.desiredProducer == nullptr)) { - // Aha, first time we're dequeueing anything. - // Figure out our local position - // Note: offset is from start, not end, but we're traversing from end -- subtract from count first - std::uint32_t offset = prodCount - 1 - (token.initialOffset % prodCount); - token.desiredProducer = tail; - for (std::uint32_t i = 0; i != offset; ++i) { - token.desiredProducer = static_cast(token.desiredProducer)->next_prod(); - if (token.desiredProducer == nullptr) { - token.desiredProducer = tail; - } - } - } - - std::uint32_t delta = globalOffset - token.lastKnownGlobalOffset; - if (delta >= prodCount) { - delta = delta % prodCount; - } - for (std::uint32_t i = 0; i != delta; ++i) { - token.desiredProducer = static_cast(token.desiredProducer)->next_prod(); - if (token.desiredProducer == nullptr) { - token.desiredProducer = tail; - } - } - - token.lastKnownGlobalOffset = globalOffset; - token.currentProducer = token.desiredProducer; - token.itemsConsumedFromCurrent = 0; - return true; - } - - - /////////////////////////// - // Free list - /////////////////////////// - - template - struct FreeListNode - { - FreeListNode() : freeListRefs(0), freeListNext(nullptr) { } - - std::atomic freeListRefs; - std::atomic freeListNext; - }; - - // A simple CAS-based lock-free free list. Not the fastest thing in the world under heavy contention, but - // simple and correct (assuming nodes are never freed until after the free list is destroyed), and fairly - // speedy under low contention. - template // N must inherit FreeListNode or have the same fields (and initialization of them) - struct FreeList - { - FreeList() : freeListHead(nullptr) { } - FreeList(FreeList&& other) : freeListHead(other.freeListHead.load(std::memory_order_relaxed)) { other.freeListHead.store(nullptr, std::memory_order_relaxed); } - void swap(FreeList& other) { details::swap_relaxed(freeListHead, other.freeListHead); } - - FreeList(FreeList const&) MOODYCAMEL_DELETE_FUNCTION; - FreeList& operator=(FreeList const&) MOODYCAMEL_DELETE_FUNCTION; - - inline void add(N* node) - { -#ifdef MCDBGQ_NOLOCKFREE_FREELIST - debug::DebugLock lock(mutex); -#endif - // We know that the should-be-on-freelist bit is 0 at this point, so it's safe to - // set it using a fetch_add - if (node->freeListRefs.fetch_add(SHOULD_BE_ON_FREELIST, std::memory_order_acq_rel) == 0) { - // Oh look! We were the last ones referencing this node, and we know - // we want to add it to the free list, so let's do it! - add_knowing_refcount_is_zero(node); - } - } - - inline N* try_get() - { -#ifdef MCDBGQ_NOLOCKFREE_FREELIST - debug::DebugLock lock(mutex); -#endif - auto head = freeListHead.load(std::memory_order_acquire); - while (head != nullptr) { - auto prevHead = head; - auto refs = head->freeListRefs.load(std::memory_order_relaxed); - if ((refs & REFS_MASK) == 0 || !head->freeListRefs.compare_exchange_strong(refs, refs + 1, std::memory_order_acquire, std::memory_order_relaxed)) { - head = freeListHead.load(std::memory_order_acquire); - continue; - } - - // Good, reference count has been incremented (it wasn't at zero), which means we can read the - // next and not worry about it changing between now and the time we do the CAS - auto next = head->freeListNext.load(std::memory_order_relaxed); - if (freeListHead.compare_exchange_strong(head, next, std::memory_order_acquire, std::memory_order_relaxed)) { - // Yay, got the node. This means it was on the list, which means shouldBeOnFreeList must be false no - // matter the refcount (because nobody else knows it's been taken off yet, it can't have been put back on). - assert((head->freeListRefs.load(std::memory_order_relaxed) & SHOULD_BE_ON_FREELIST) == 0); - - // Decrease refcount twice, once for our ref, and once for the list's ref - head->freeListRefs.fetch_sub(2, std::memory_order_release); - return head; - } - - // OK, the head must have changed on us, but we still need to decrease the refcount we increased. - // Note that we don't need to release any memory effects, but we do need to ensure that the reference - // count decrement happens-after the CAS on the head. - refs = prevHead->freeListRefs.fetch_sub(1, std::memory_order_acq_rel); - if (refs == SHOULD_BE_ON_FREELIST + 1) { - add_knowing_refcount_is_zero(prevHead); - } - } - - return nullptr; - } - - // Useful for traversing the list when there's no contention (e.g. to destroy remaining nodes) - N* head_unsafe() const { return freeListHead.load(std::memory_order_relaxed); } - - private: - inline void add_knowing_refcount_is_zero(N* node) - { - // Since the refcount is zero, and nobody can increase it once it's zero (except us, and we run - // only one copy of this method per node at a time, i.e. the single thread case), then we know - // we can safely change the next pointer of the node; however, once the refcount is back above - // zero, then other threads could increase it (happens under heavy contention, when the refcount - // goes to zero in between a load and a refcount increment of a node in try_get, then back up to - // something non-zero, then the refcount increment is done by the other thread) -- so, if the CAS - // to add the node to the actual list fails, decrease the refcount and leave the add operation to - // the next thread who puts the refcount back at zero (which could be us, hence the loop). - auto head = freeListHead.load(std::memory_order_relaxed); - while (true) { - node->freeListNext.store(head, std::memory_order_relaxed); - node->freeListRefs.store(1, std::memory_order_release); - if (!freeListHead.compare_exchange_strong(head, node, std::memory_order_release, std::memory_order_relaxed)) { - // Hmm, the add failed, but we can only try again when the refcount goes back to zero - if (node->freeListRefs.fetch_add(SHOULD_BE_ON_FREELIST - 1, std::memory_order_release) == 1) { - continue; - } - } - return; - } - } - - private: - // Implemented like a stack, but where node order doesn't matter (nodes are inserted out of order under contention) - std::atomic freeListHead; - - static const std::uint32_t REFS_MASK = 0x7FFFFFFF; - static const std::uint32_t SHOULD_BE_ON_FREELIST = 0x80000000; - -#ifdef MCDBGQ_NOLOCKFREE_FREELIST - debug::DebugMutex mutex; -#endif - }; - - - /////////////////////////// - // Block - /////////////////////////// - - enum InnerQueueContext { implicit_context = 0, explicit_context = 1 }; - - struct Block - { - Block() - : next(nullptr), elementsCompletelyDequeued(0), freeListRefs(0), freeListNext(nullptr), dynamicallyAllocated(true) - { -#ifdef MCDBGQ_TRACKMEM - owner = nullptr; -#endif - } - - template - inline bool is_empty() const - { - MOODYCAMEL_CONSTEXPR_IF (context == explicit_context && BLOCK_SIZE <= EXPLICIT_BLOCK_EMPTY_COUNTER_THRESHOLD) { - // Check flags - for (size_t i = 0; i < BLOCK_SIZE; ++i) { - if (!emptyFlags[i].load(std::memory_order_relaxed)) { - return false; - } - } - - // Aha, empty; make sure we have all other memory effects that happened before the empty flags were set - std::atomic_thread_fence(std::memory_order_acquire); - return true; - } - else { - // Check counter - if (elementsCompletelyDequeued.load(std::memory_order_relaxed) == BLOCK_SIZE) { - std::atomic_thread_fence(std::memory_order_acquire); - return true; - } - assert(elementsCompletelyDequeued.load(std::memory_order_relaxed) <= BLOCK_SIZE); - return false; - } - } - - // Returns true if the block is now empty (does not apply in explicit context) - template - inline bool set_empty(MOODYCAMEL_MAYBE_UNUSED index_t i) - { - MOODYCAMEL_CONSTEXPR_IF (context == explicit_context && BLOCK_SIZE <= EXPLICIT_BLOCK_EMPTY_COUNTER_THRESHOLD) { - // Set flag - assert(!emptyFlags[BLOCK_SIZE - 1 - static_cast(i & static_cast(BLOCK_SIZE - 1))].load(std::memory_order_relaxed)); - emptyFlags[BLOCK_SIZE - 1 - static_cast(i & static_cast(BLOCK_SIZE - 1))].store(true, std::memory_order_release); - return false; - } - else { - // Increment counter - auto prevVal = elementsCompletelyDequeued.fetch_add(1, std::memory_order_release); - assert(prevVal < BLOCK_SIZE); - return prevVal == BLOCK_SIZE - 1; - } - } - - // Sets multiple contiguous item statuses to 'empty' (assumes no wrapping and count > 0). - // Returns true if the block is now empty (does not apply in explicit context). - template - inline bool set_many_empty(MOODYCAMEL_MAYBE_UNUSED index_t i, size_t count) - { - MOODYCAMEL_CONSTEXPR_IF (context == explicit_context && BLOCK_SIZE <= EXPLICIT_BLOCK_EMPTY_COUNTER_THRESHOLD) { - // Set flags - std::atomic_thread_fence(std::memory_order_release); - i = BLOCK_SIZE - 1 - static_cast(i & static_cast(BLOCK_SIZE - 1)) - count + 1; - for (size_t j = 0; j != count; ++j) { - assert(!emptyFlags[i + j].load(std::memory_order_relaxed)); - emptyFlags[i + j].store(true, std::memory_order_relaxed); - } - return false; - } - else { - // Increment counter - auto prevVal = elementsCompletelyDequeued.fetch_add(count, std::memory_order_release); - assert(prevVal + count <= BLOCK_SIZE); - return prevVal + count == BLOCK_SIZE; - } - } - - template - inline void set_all_empty() - { - MOODYCAMEL_CONSTEXPR_IF (context == explicit_context && BLOCK_SIZE <= EXPLICIT_BLOCK_EMPTY_COUNTER_THRESHOLD) { - // Set all flags - for (size_t i = 0; i != BLOCK_SIZE; ++i) { - emptyFlags[i].store(true, std::memory_order_relaxed); - } - } - else { - // Reset counter - elementsCompletelyDequeued.store(BLOCK_SIZE, std::memory_order_relaxed); - } - } - - template - inline void reset_empty() - { - MOODYCAMEL_CONSTEXPR_IF (context == explicit_context && BLOCK_SIZE <= EXPLICIT_BLOCK_EMPTY_COUNTER_THRESHOLD) { - // Reset flags - for (size_t i = 0; i != BLOCK_SIZE; ++i) { - emptyFlags[i].store(false, std::memory_order_relaxed); - } - } - else { - // Reset counter - elementsCompletelyDequeued.store(0, std::memory_order_relaxed); - } - } - - inline T* operator[](index_t idx) MOODYCAMEL_NOEXCEPT { return static_cast(static_cast(elements)) + static_cast(idx & static_cast(BLOCK_SIZE - 1)); } - inline T const* operator[](index_t idx) const MOODYCAMEL_NOEXCEPT { return static_cast(static_cast(elements)) + static_cast(idx & static_cast(BLOCK_SIZE - 1)); } - - private: - static_assert(std::alignment_of::value <= sizeof(T), "The queue does not support types with an alignment greater than their size at this time"); - MOODYCAMEL_ALIGNED_TYPE_LIKE(char[sizeof(T) * BLOCK_SIZE], T) elements; - public: - Block* next; - std::atomic elementsCompletelyDequeued; - std::atomic emptyFlags[BLOCK_SIZE <= EXPLICIT_BLOCK_EMPTY_COUNTER_THRESHOLD ? BLOCK_SIZE : 1]; - public: - std::atomic freeListRefs; - std::atomic freeListNext; - bool dynamicallyAllocated; // Perhaps a better name for this would be 'isNotPartOfInitialBlockPool' - -#ifdef MCDBGQ_TRACKMEM - void* owner; -#endif - }; - static_assert(std::alignment_of::value >= std::alignment_of::value, "Internal error: Blocks must be at least as aligned as the type they are wrapping"); - - -#ifdef MCDBGQ_TRACKMEM -public: - struct MemStats; -private: -#endif - - /////////////////////////// - // Producer base - /////////////////////////// - - struct ProducerBase : public details::ConcurrentQueueProducerTypelessBase - { - ProducerBase(ConcurrentQueue* parent_, bool isExplicit_) : - tailIndex(0), - headIndex(0), - dequeueOptimisticCount(0), - dequeueOvercommit(0), - tailBlock(nullptr), - isExplicit(isExplicit_), - parent(parent_) - { - } - - virtual ~ProducerBase() { } - - template - inline bool dequeue(U& element) - { - if (isExplicit) { - return static_cast(this)->dequeue(element); - } - else { - return static_cast(this)->dequeue(element); - } - } - - template - inline size_t dequeue_bulk(It& itemFirst, size_t max) - { - if (isExplicit) { - return static_cast(this)->dequeue_bulk(itemFirst, max); - } - else { - return static_cast(this)->dequeue_bulk(itemFirst, max); - } - } - - inline ProducerBase* next_prod() const { return static_cast(next); } - - inline size_t size_approx() const - { - auto tail = tailIndex.load(std::memory_order_relaxed); - auto head = headIndex.load(std::memory_order_relaxed); - return details::circular_less_than(head, tail) ? static_cast(tail - head) : 0; - } - - inline index_t getTail() const { return tailIndex.load(std::memory_order_relaxed); } - protected: - std::atomic tailIndex; // Where to enqueue to next - std::atomic headIndex; // Where to dequeue from next - - std::atomic dequeueOptimisticCount; - std::atomic dequeueOvercommit; - - Block* tailBlock; - - public: - bool isExplicit; - ConcurrentQueue* parent; - - protected: -#ifdef MCDBGQ_TRACKMEM - friend struct MemStats; -#endif - }; - - - /////////////////////////// - // Explicit queue - /////////////////////////// - - struct ExplicitProducer : public ProducerBase - { - explicit ExplicitProducer(ConcurrentQueue* parent_) : - ProducerBase(parent_, true), - blockIndex(nullptr), - pr_blockIndexSlotsUsed(0), - pr_blockIndexSize(EXPLICIT_INITIAL_INDEX_SIZE >> 1), - pr_blockIndexFront(0), - pr_blockIndexEntries(nullptr), - pr_blockIndexRaw(nullptr) - { - size_t poolBasedIndexSize = details::ceil_to_pow_2(parent_->initialBlockPoolSize) >> 1; - if (poolBasedIndexSize > pr_blockIndexSize) { - pr_blockIndexSize = poolBasedIndexSize; - } - - new_block_index(0); // This creates an index with double the number of current entries, i.e. EXPLICIT_INITIAL_INDEX_SIZE - } - - ~ExplicitProducer() - { - // Destruct any elements not yet dequeued. - // Since we're in the destructor, we can assume all elements - // are either completely dequeued or completely not (no halfways). - if (this->tailBlock != nullptr) { // Note this means there must be a block index too - // First find the block that's partially dequeued, if any - Block* halfDequeuedBlock = nullptr; - if ((this->headIndex.load(std::memory_order_relaxed) & static_cast(BLOCK_SIZE - 1)) != 0) { - // The head's not on a block boundary, meaning a block somewhere is partially dequeued - // (or the head block is the tail block and was fully dequeued, but the head/tail are still not on a boundary) - size_t i = (pr_blockIndexFront - pr_blockIndexSlotsUsed) & (pr_blockIndexSize - 1); - while (details::circular_less_than(pr_blockIndexEntries[i].base + BLOCK_SIZE, this->headIndex.load(std::memory_order_relaxed))) { - i = (i + 1) & (pr_blockIndexSize - 1); - } - assert(details::circular_less_than(pr_blockIndexEntries[i].base, this->headIndex.load(std::memory_order_relaxed))); - halfDequeuedBlock = pr_blockIndexEntries[i].block; - } - - // Start at the head block (note the first line in the loop gives us the head from the tail on the first iteration) - auto block = this->tailBlock; - do { - block = block->next; - if (block->ConcurrentQueue::Block::template is_empty()) { - continue; - } - - size_t i = 0; // Offset into block - if (block == halfDequeuedBlock) { - i = static_cast(this->headIndex.load(std::memory_order_relaxed) & static_cast(BLOCK_SIZE - 1)); - } - - // Walk through all the items in the block; if this is the tail block, we need to stop when we reach the tail index - auto lastValidIndex = (this->tailIndex.load(std::memory_order_relaxed) & static_cast(BLOCK_SIZE - 1)) == 0 ? BLOCK_SIZE : static_cast(this->tailIndex.load(std::memory_order_relaxed) & static_cast(BLOCK_SIZE - 1)); - while (i != BLOCK_SIZE && (block != this->tailBlock || i != lastValidIndex)) { - (*block)[i++]->~T(); - } - } while (block != this->tailBlock); - } - - // Destroy all blocks that we own - if (this->tailBlock != nullptr) { - auto block = this->tailBlock; - do { - auto nextBlock = block->next; - this->parent->add_block_to_free_list(block); - block = nextBlock; - } while (block != this->tailBlock); - } - - // Destroy the block indices - auto header = static_cast(pr_blockIndexRaw); - while (header != nullptr) { - auto prev = static_cast(header->prev); - header->~BlockIndexHeader(); - (Traits::free)(header); - header = prev; - } - } - - template - inline bool enqueue(U&& element) - { - index_t currentTailIndex = this->tailIndex.load(std::memory_order_relaxed); - index_t newTailIndex = 1 + currentTailIndex; - if ((currentTailIndex & static_cast(BLOCK_SIZE - 1)) == 0) { - // We reached the end of a block, start a new one - auto startBlock = this->tailBlock; - auto originalBlockIndexSlotsUsed = pr_blockIndexSlotsUsed; - if (this->tailBlock != nullptr && this->tailBlock->next->ConcurrentQueue::Block::template is_empty()) { - // We can re-use the block ahead of us, it's empty! - this->tailBlock = this->tailBlock->next; - this->tailBlock->ConcurrentQueue::Block::template reset_empty(); - - // We'll put the block on the block index (guaranteed to be room since we're conceptually removing the - // last block from it first -- except instead of removing then adding, we can just overwrite). - // Note that there must be a valid block index here, since even if allocation failed in the ctor, - // it would have been re-attempted when adding the first block to the queue; since there is such - // a block, a block index must have been successfully allocated. - } - else { - // Whatever head value we see here is >= the last value we saw here (relatively), - // and <= its current value. Since we have the most recent tail, the head must be - // <= to it. - auto head = this->headIndex.load(std::memory_order_relaxed); - assert(!details::circular_less_than(currentTailIndex, head)); - if (!details::circular_less_than(head, currentTailIndex + BLOCK_SIZE) - || (MAX_SUBQUEUE_SIZE != details::const_numeric_max::value && (MAX_SUBQUEUE_SIZE == 0 || MAX_SUBQUEUE_SIZE - BLOCK_SIZE < currentTailIndex - head))) { - // We can't enqueue in another block because there's not enough leeway -- the - // tail could surpass the head by the time the block fills up! (Or we'll exceed - // the size limit, if the second part of the condition was true.) - return false; - } - // We're going to need a new block; check that the block index has room - if (pr_blockIndexRaw == nullptr || pr_blockIndexSlotsUsed == pr_blockIndexSize) { - // Hmm, the circular block index is already full -- we'll need - // to allocate a new index. Note pr_blockIndexRaw can only be nullptr if - // the initial allocation failed in the constructor. - - MOODYCAMEL_CONSTEXPR_IF (allocMode == CannotAlloc) { - return false; - } - else if (!new_block_index(pr_blockIndexSlotsUsed)) { - return false; - } - } - - // Insert a new block in the circular linked list - auto newBlock = this->parent->ConcurrentQueue::template requisition_block(); - if (newBlock == nullptr) { - return false; - } -#ifdef MCDBGQ_TRACKMEM - newBlock->owner = this; -#endif - newBlock->ConcurrentQueue::Block::template reset_empty(); - if (this->tailBlock == nullptr) { - newBlock->next = newBlock; - } - else { - newBlock->next = this->tailBlock->next; - this->tailBlock->next = newBlock; - } - this->tailBlock = newBlock; - ++pr_blockIndexSlotsUsed; - } - - MOODYCAMEL_CONSTEXPR_IF (!MOODYCAMEL_NOEXCEPT_CTOR(T, U, new (static_cast(nullptr)) T(std::forward(element)))) { - // The constructor may throw. We want the element not to appear in the queue in - // that case (without corrupting the queue): - MOODYCAMEL_TRY { - new ((*this->tailBlock)[currentTailIndex]) T(std::forward(element)); - } - MOODYCAMEL_CATCH (...) { - // Revert change to the current block, but leave the new block available - // for next time - pr_blockIndexSlotsUsed = originalBlockIndexSlotsUsed; - this->tailBlock = startBlock == nullptr ? this->tailBlock : startBlock; - MOODYCAMEL_RETHROW; - } - } - else { - (void)startBlock; - (void)originalBlockIndexSlotsUsed; - } - - // Add block to block index - auto& entry = blockIndex.load(std::memory_order_relaxed)->entries[pr_blockIndexFront]; - entry.base = currentTailIndex; - entry.block = this->tailBlock; - blockIndex.load(std::memory_order_relaxed)->front.store(pr_blockIndexFront, std::memory_order_release); - pr_blockIndexFront = (pr_blockIndexFront + 1) & (pr_blockIndexSize - 1); - - MOODYCAMEL_CONSTEXPR_IF (!MOODYCAMEL_NOEXCEPT_CTOR(T, U, new (static_cast(nullptr)) T(std::forward(element)))) { - this->tailIndex.store(newTailIndex, std::memory_order_release); - return true; - } - } - - // Enqueue - new ((*this->tailBlock)[currentTailIndex]) T(std::forward(element)); - - this->tailIndex.store(newTailIndex, std::memory_order_release); - return true; - } - - template - bool dequeue(U& element) - { - auto tail = this->tailIndex.load(std::memory_order_relaxed); - auto overcommit = this->dequeueOvercommit.load(std::memory_order_relaxed); - if (details::circular_less_than(this->dequeueOptimisticCount.load(std::memory_order_relaxed) - overcommit, tail)) { - // Might be something to dequeue, let's give it a try - - // Note that this if is purely for performance purposes in the common case when the queue is - // empty and the values are eventually consistent -- we may enter here spuriously. - - // Note that whatever the values of overcommit and tail are, they are not going to change (unless we - // change them) and must be the same value at this point (inside the if) as when the if condition was - // evaluated. - - // We insert an acquire fence here to synchronize-with the release upon incrementing dequeueOvercommit below. - // This ensures that whatever the value we got loaded into overcommit, the load of dequeueOptisticCount in - // the fetch_add below will result in a value at least as recent as that (and therefore at least as large). - // Note that I believe a compiler (signal) fence here would be sufficient due to the nature of fetch_add (all - // read-modify-write operations are guaranteed to work on the latest value in the modification order), but - // unfortunately that can't be shown to be correct using only the C++11 standard. - // See http://stackoverflow.com/questions/18223161/what-are-the-c11-memory-ordering-guarantees-in-this-corner-case - std::atomic_thread_fence(std::memory_order_acquire); - - // Increment optimistic counter, then check if it went over the boundary - auto myDequeueCount = this->dequeueOptimisticCount.fetch_add(1, std::memory_order_relaxed); - - // Note that since dequeueOvercommit must be <= dequeueOptimisticCount (because dequeueOvercommit is only ever - // incremented after dequeueOptimisticCount -- this is enforced in the `else` block below), and since we now - // have a version of dequeueOptimisticCount that is at least as recent as overcommit (due to the release upon - // incrementing dequeueOvercommit and the acquire above that synchronizes with it), overcommit <= myDequeueCount. - // However, we can't assert this since both dequeueOptimisticCount and dequeueOvercommit may (independently) - // overflow; in such a case, though, the logic still holds since the difference between the two is maintained. - - // Note that we reload tail here in case it changed; it will be the same value as before or greater, since - // this load is sequenced after (happens after) the earlier load above. This is supported by read-read - // coherency (as defined in the standard), explained here: http://en.cppreference.com/w/cpp/atomic/memory_order - tail = this->tailIndex.load(std::memory_order_acquire); - if ((details::likely)(details::circular_less_than(myDequeueCount - overcommit, tail))) { - // Guaranteed to be at least one element to dequeue! - - // Get the index. Note that since there's guaranteed to be at least one element, this - // will never exceed tail. We need to do an acquire-release fence here since it's possible - // that whatever condition got us to this point was for an earlier enqueued element (that - // we already see the memory effects for), but that by the time we increment somebody else - // has incremented it, and we need to see the memory effects for *that* element, which is - // in such a case is necessarily visible on the thread that incremented it in the first - // place with the more current condition (they must have acquired a tail that is at least - // as recent). - auto index = this->headIndex.fetch_add(1, std::memory_order_acq_rel); - - - // Determine which block the element is in - - auto localBlockIndex = blockIndex.load(std::memory_order_acquire); - auto localBlockIndexHead = localBlockIndex->front.load(std::memory_order_acquire); - - // We need to be careful here about subtracting and dividing because of index wrap-around. - // When an index wraps, we need to preserve the sign of the offset when dividing it by the - // block size (in order to get a correct signed block count offset in all cases): - auto headBase = localBlockIndex->entries[localBlockIndexHead].base; - auto blockBaseIndex = index & ~static_cast(BLOCK_SIZE - 1); - auto offset = static_cast(static_cast::type>(blockBaseIndex - headBase) / static_cast::type>(BLOCK_SIZE)); - auto block = localBlockIndex->entries[(localBlockIndexHead + offset) & (localBlockIndex->size - 1)].block; - - // Dequeue - auto& el = *((*block)[index]); - if (!MOODYCAMEL_NOEXCEPT_ASSIGN(T, T&&, element = std::move(el))) { - // Make sure the element is still fully dequeued and destroyed even if the assignment - // throws - struct Guard { - Block* block; - index_t index; - - ~Guard() - { - (*block)[index]->~T(); - block->ConcurrentQueue::Block::template set_empty(index); - } - } guard = { block, index }; - - element = std::move(el); // NOLINT - } - else { - element = std::move(el); // NOLINT - el.~T(); // NOLINT - block->ConcurrentQueue::Block::template set_empty(index); - } - - return true; - } - else { - // Wasn't anything to dequeue after all; make the effective dequeue count eventually consistent - this->dequeueOvercommit.fetch_add(1, std::memory_order_release); // Release so that the fetch_add on dequeueOptimisticCount is guaranteed to happen before this write - } - } - - return false; - } - - template - bool MOODYCAMEL_NO_TSAN enqueue_bulk(It itemFirst, size_t count) - { - // First, we need to make sure we have enough room to enqueue all of the elements; - // this means pre-allocating blocks and putting them in the block index (but only if - // all the allocations succeeded). - index_t startTailIndex = this->tailIndex.load(std::memory_order_relaxed); - auto startBlock = this->tailBlock; - auto originalBlockIndexFront = pr_blockIndexFront; - auto originalBlockIndexSlotsUsed = pr_blockIndexSlotsUsed; - - Block* firstAllocatedBlock = nullptr; - - // Figure out how many blocks we'll need to allocate, and do so - size_t blockBaseDiff = ((startTailIndex + count - 1) & ~static_cast(BLOCK_SIZE - 1)) - ((startTailIndex - 1) & ~static_cast(BLOCK_SIZE - 1)); - index_t currentTailIndex = (startTailIndex - 1) & ~static_cast(BLOCK_SIZE - 1); - if (blockBaseDiff > 0) { - // Allocate as many blocks as possible from ahead - while (blockBaseDiff > 0 && this->tailBlock != nullptr && this->tailBlock->next != firstAllocatedBlock && this->tailBlock->next->ConcurrentQueue::Block::template is_empty()) { - blockBaseDiff -= static_cast(BLOCK_SIZE); - currentTailIndex += static_cast(BLOCK_SIZE); - - this->tailBlock = this->tailBlock->next; - firstAllocatedBlock = firstAllocatedBlock == nullptr ? this->tailBlock : firstAllocatedBlock; - - auto& entry = blockIndex.load(std::memory_order_relaxed)->entries[pr_blockIndexFront]; - entry.base = currentTailIndex; - entry.block = this->tailBlock; - pr_blockIndexFront = (pr_blockIndexFront + 1) & (pr_blockIndexSize - 1); - } - - // Now allocate as many blocks as necessary from the block pool - while (blockBaseDiff > 0) { - blockBaseDiff -= static_cast(BLOCK_SIZE); - currentTailIndex += static_cast(BLOCK_SIZE); - - auto head = this->headIndex.load(std::memory_order_relaxed); - assert(!details::circular_less_than(currentTailIndex, head)); - bool full = !details::circular_less_than(head, currentTailIndex + BLOCK_SIZE) || (MAX_SUBQUEUE_SIZE != details::const_numeric_max::value && (MAX_SUBQUEUE_SIZE == 0 || MAX_SUBQUEUE_SIZE - BLOCK_SIZE < currentTailIndex - head)); - if (pr_blockIndexRaw == nullptr || pr_blockIndexSlotsUsed == pr_blockIndexSize || full) { - MOODYCAMEL_CONSTEXPR_IF (allocMode == CannotAlloc) { - // Failed to allocate, undo changes (but keep injected blocks) - pr_blockIndexFront = originalBlockIndexFront; - pr_blockIndexSlotsUsed = originalBlockIndexSlotsUsed; - this->tailBlock = startBlock == nullptr ? firstAllocatedBlock : startBlock; - return false; - } - else if (full || !new_block_index(originalBlockIndexSlotsUsed)) { - // Failed to allocate, undo changes (but keep injected blocks) - pr_blockIndexFront = originalBlockIndexFront; - pr_blockIndexSlotsUsed = originalBlockIndexSlotsUsed; - this->tailBlock = startBlock == nullptr ? firstAllocatedBlock : startBlock; - return false; - } - - // pr_blockIndexFront is updated inside new_block_index, so we need to - // update our fallback value too (since we keep the new index even if we - // later fail) - originalBlockIndexFront = originalBlockIndexSlotsUsed; - } - - // Insert a new block in the circular linked list - auto newBlock = this->parent->ConcurrentQueue::template requisition_block(); - if (newBlock == nullptr) { - pr_blockIndexFront = originalBlockIndexFront; - pr_blockIndexSlotsUsed = originalBlockIndexSlotsUsed; - this->tailBlock = startBlock == nullptr ? firstAllocatedBlock : startBlock; - return false; - } - -#ifdef MCDBGQ_TRACKMEM - newBlock->owner = this; -#endif - newBlock->ConcurrentQueue::Block::template set_all_empty(); - if (this->tailBlock == nullptr) { - newBlock->next = newBlock; - } - else { - newBlock->next = this->tailBlock->next; - this->tailBlock->next = newBlock; - } - this->tailBlock = newBlock; - firstAllocatedBlock = firstAllocatedBlock == nullptr ? this->tailBlock : firstAllocatedBlock; - - ++pr_blockIndexSlotsUsed; - - auto& entry = blockIndex.load(std::memory_order_relaxed)->entries[pr_blockIndexFront]; - entry.base = currentTailIndex; - entry.block = this->tailBlock; - pr_blockIndexFront = (pr_blockIndexFront + 1) & (pr_blockIndexSize - 1); - } - - // Excellent, all allocations succeeded. Reset each block's emptiness before we fill them up, and - // publish the new block index front - auto block = firstAllocatedBlock; - while (true) { - block->ConcurrentQueue::Block::template reset_empty(); - if (block == this->tailBlock) { - break; - } - block = block->next; - } - - MOODYCAMEL_CONSTEXPR_IF (MOODYCAMEL_NOEXCEPT_CTOR(T, decltype(*itemFirst), new (static_cast(nullptr)) T(details::deref_noexcept(itemFirst)))) { - blockIndex.load(std::memory_order_relaxed)->front.store((pr_blockIndexFront - 1) & (pr_blockIndexSize - 1), std::memory_order_release); - } - } - - // Enqueue, one block at a time - index_t newTailIndex = startTailIndex + static_cast(count); - currentTailIndex = startTailIndex; - auto endBlock = this->tailBlock; - this->tailBlock = startBlock; - assert((startTailIndex & static_cast(BLOCK_SIZE - 1)) != 0 || firstAllocatedBlock != nullptr || count == 0); - if ((startTailIndex & static_cast(BLOCK_SIZE - 1)) == 0 && firstAllocatedBlock != nullptr) { - this->tailBlock = firstAllocatedBlock; - } - while (true) { - index_t stopIndex = (currentTailIndex & ~static_cast(BLOCK_SIZE - 1)) + static_cast(BLOCK_SIZE); - if (details::circular_less_than(newTailIndex, stopIndex)) { - stopIndex = newTailIndex; - } - MOODYCAMEL_CONSTEXPR_IF (MOODYCAMEL_NOEXCEPT_CTOR(T, decltype(*itemFirst), new (static_cast(nullptr)) T(details::deref_noexcept(itemFirst)))) { - while (currentTailIndex != stopIndex) { - new ((*this->tailBlock)[currentTailIndex++]) T(*itemFirst++); - } - } - else { - MOODYCAMEL_TRY { - while (currentTailIndex != stopIndex) { - // Must use copy constructor even if move constructor is available - // because we may have to revert if there's an exception. - // Sorry about the horrible templated next line, but it was the only way - // to disable moving *at compile time*, which is important because a type - // may only define a (noexcept) move constructor, and so calls to the - // cctor will not compile, even if they are in an if branch that will never - // be executed - new ((*this->tailBlock)[currentTailIndex]) T(details::nomove_if(nullptr)) T(details::deref_noexcept(itemFirst)))>::eval(*itemFirst)); - ++currentTailIndex; - ++itemFirst; - } - } - MOODYCAMEL_CATCH (...) { - // Oh dear, an exception's been thrown -- destroy the elements that - // were enqueued so far and revert the entire bulk operation (we'll keep - // any allocated blocks in our linked list for later, though). - auto constructedStopIndex = currentTailIndex; - auto lastBlockEnqueued = this->tailBlock; - - pr_blockIndexFront = originalBlockIndexFront; - pr_blockIndexSlotsUsed = originalBlockIndexSlotsUsed; - this->tailBlock = startBlock == nullptr ? firstAllocatedBlock : startBlock; - - if (!details::is_trivially_destructible::value) { - auto block = startBlock; - if ((startTailIndex & static_cast(BLOCK_SIZE - 1)) == 0) { - block = firstAllocatedBlock; - } - currentTailIndex = startTailIndex; - while (true) { - stopIndex = (currentTailIndex & ~static_cast(BLOCK_SIZE - 1)) + static_cast(BLOCK_SIZE); - if (details::circular_less_than(constructedStopIndex, stopIndex)) { - stopIndex = constructedStopIndex; - } - while (currentTailIndex != stopIndex) { - (*block)[currentTailIndex++]->~T(); - } - if (block == lastBlockEnqueued) { - break; - } - block = block->next; - } - } - MOODYCAMEL_RETHROW; - } - } - - if (this->tailBlock == endBlock) { - assert(currentTailIndex == newTailIndex); - break; - } - this->tailBlock = this->tailBlock->next; - } - - MOODYCAMEL_CONSTEXPR_IF (!MOODYCAMEL_NOEXCEPT_CTOR(T, decltype(*itemFirst), new (static_cast(nullptr)) T(details::deref_noexcept(itemFirst)))) { - if (firstAllocatedBlock != nullptr) - blockIndex.load(std::memory_order_relaxed)->front.store((pr_blockIndexFront - 1) & (pr_blockIndexSize - 1), std::memory_order_release); - } - - this->tailIndex.store(newTailIndex, std::memory_order_release); - return true; - } - - template - size_t dequeue_bulk(It& itemFirst, size_t max) - { - auto tail = this->tailIndex.load(std::memory_order_relaxed); - auto overcommit = this->dequeueOvercommit.load(std::memory_order_relaxed); - auto desiredCount = static_cast(tail - (this->dequeueOptimisticCount.load(std::memory_order_relaxed) - overcommit)); - if (details::circular_less_than(0, desiredCount)) { - desiredCount = desiredCount < max ? desiredCount : max; - std::atomic_thread_fence(std::memory_order_acquire); - - auto myDequeueCount = this->dequeueOptimisticCount.fetch_add(desiredCount, std::memory_order_relaxed); - - tail = this->tailIndex.load(std::memory_order_acquire); - auto actualCount = static_cast(tail - (myDequeueCount - overcommit)); - if (details::circular_less_than(0, actualCount)) { - actualCount = desiredCount < actualCount ? desiredCount : actualCount; - if (actualCount < desiredCount) { - this->dequeueOvercommit.fetch_add(desiredCount - actualCount, std::memory_order_release); - } - - // Get the first index. Note that since there's guaranteed to be at least actualCount elements, this - // will never exceed tail. - auto firstIndex = this->headIndex.fetch_add(actualCount, std::memory_order_acq_rel); - - // Determine which block the first element is in - auto localBlockIndex = blockIndex.load(std::memory_order_acquire); - auto localBlockIndexHead = localBlockIndex->front.load(std::memory_order_acquire); - - auto headBase = localBlockIndex->entries[localBlockIndexHead].base; - auto firstBlockBaseIndex = firstIndex & ~static_cast(BLOCK_SIZE - 1); - auto offset = static_cast(static_cast::type>(firstBlockBaseIndex - headBase) / static_cast::type>(BLOCK_SIZE)); - auto indexIndex = (localBlockIndexHead + offset) & (localBlockIndex->size - 1); - - // Iterate the blocks and dequeue - auto index = firstIndex; - do { - auto firstIndexInBlock = index; - index_t endIndex = (index & ~static_cast(BLOCK_SIZE - 1)) + static_cast(BLOCK_SIZE); - endIndex = details::circular_less_than(firstIndex + static_cast(actualCount), endIndex) ? firstIndex + static_cast(actualCount) : endIndex; - auto block = localBlockIndex->entries[indexIndex].block; - if (MOODYCAMEL_NOEXCEPT_ASSIGN(T, T&&, details::deref_noexcept(itemFirst) = std::move((*(*block)[index])))) { - while (index != endIndex) { - auto& el = *((*block)[index]); - *itemFirst++ = std::move(el); - el.~T(); - ++index; - } - } - else { - MOODYCAMEL_TRY { - while (index != endIndex) { - auto& el = *((*block)[index]); - *itemFirst = std::move(el); - ++itemFirst; - el.~T(); - ++index; - } - } - MOODYCAMEL_CATCH (...) { - // It's too late to revert the dequeue, but we can make sure that all - // the dequeued objects are properly destroyed and the block index - // (and empty count) are properly updated before we propagate the exception - do { - block = localBlockIndex->entries[indexIndex].block; - while (index != endIndex) { - (*block)[index++]->~T(); - } - block->ConcurrentQueue::Block::template set_many_empty(firstIndexInBlock, static_cast(endIndex - firstIndexInBlock)); - indexIndex = (indexIndex + 1) & (localBlockIndex->size - 1); - - firstIndexInBlock = index; - endIndex = (index & ~static_cast(BLOCK_SIZE - 1)) + static_cast(BLOCK_SIZE); - endIndex = details::circular_less_than(firstIndex + static_cast(actualCount), endIndex) ? firstIndex + static_cast(actualCount) : endIndex; - } while (index != firstIndex + actualCount); - - MOODYCAMEL_RETHROW; - } - } - block->ConcurrentQueue::Block::template set_many_empty(firstIndexInBlock, static_cast(endIndex - firstIndexInBlock)); - indexIndex = (indexIndex + 1) & (localBlockIndex->size - 1); - } while (index != firstIndex + actualCount); - - return actualCount; - } - else { - // Wasn't anything to dequeue after all; make the effective dequeue count eventually consistent - this->dequeueOvercommit.fetch_add(desiredCount, std::memory_order_release); - } - } - - return 0; - } - - private: - struct BlockIndexEntry - { - index_t base; - Block* block; - }; - - struct BlockIndexHeader - { - size_t size; - std::atomic front; // Current slot (not next, like pr_blockIndexFront) - BlockIndexEntry* entries; - void* prev; - }; - - - bool new_block_index(size_t numberOfFilledSlotsToExpose) - { - auto prevBlockSizeMask = pr_blockIndexSize - 1; - - // Create the new block - pr_blockIndexSize <<= 1; - auto newRawPtr = static_cast((Traits::malloc)(sizeof(BlockIndexHeader) + std::alignment_of::value - 1 + sizeof(BlockIndexEntry) * pr_blockIndexSize)); - if (newRawPtr == nullptr) { - pr_blockIndexSize >>= 1; // Reset to allow graceful retry - return false; - } - - auto newBlockIndexEntries = reinterpret_cast(details::align_for(newRawPtr + sizeof(BlockIndexHeader))); - - // Copy in all the old indices, if any - size_t j = 0; - if (pr_blockIndexSlotsUsed != 0) { - auto i = (pr_blockIndexFront - pr_blockIndexSlotsUsed) & prevBlockSizeMask; - do { - newBlockIndexEntries[j++] = pr_blockIndexEntries[i]; - i = (i + 1) & prevBlockSizeMask; - } while (i != pr_blockIndexFront); - } - - // Update everything - auto header = new (newRawPtr) BlockIndexHeader; - header->size = pr_blockIndexSize; - header->front.store(numberOfFilledSlotsToExpose - 1, std::memory_order_relaxed); - header->entries = newBlockIndexEntries; - header->prev = pr_blockIndexRaw; // we link the new block to the old one so we can free it later - - pr_blockIndexFront = j; - pr_blockIndexEntries = newBlockIndexEntries; - pr_blockIndexRaw = newRawPtr; - blockIndex.store(header, std::memory_order_release); - - return true; - } - - private: - std::atomic blockIndex; - - // To be used by producer only -- consumer must use the ones in referenced by blockIndex - size_t pr_blockIndexSlotsUsed; - size_t pr_blockIndexSize; - size_t pr_blockIndexFront; // Next slot (not current) - BlockIndexEntry* pr_blockIndexEntries; - void* pr_blockIndexRaw; - -#ifdef MOODYCAMEL_QUEUE_INTERNAL_DEBUG - public: - ExplicitProducer* nextExplicitProducer; - private: -#endif - -#ifdef MCDBGQ_TRACKMEM - friend struct MemStats; -#endif - }; - - - ////////////////////////////////// - // Implicit queue - ////////////////////////////////// - - struct ImplicitProducer : public ProducerBase - { - ImplicitProducer(ConcurrentQueue* parent_) : - ProducerBase(parent_, false), - nextBlockIndexCapacity(IMPLICIT_INITIAL_INDEX_SIZE), - blockIndex(nullptr) - { - new_block_index(); - } - - ~ImplicitProducer() - { - // Note that since we're in the destructor we can assume that all enqueue/dequeue operations - // completed already; this means that all undequeued elements are placed contiguously across - // contiguous blocks, and that only the first and last remaining blocks can be only partially - // empty (all other remaining blocks must be completely full). - -#ifdef MOODYCAMEL_CPP11_THREAD_LOCAL_SUPPORTED - // Unregister ourselves for thread termination notification - if (!this->inactive.load(std::memory_order_relaxed)) { - details::ThreadExitNotifier::unsubscribe(&threadExitListener); - } -#endif - - // Destroy all remaining elements! - auto tail = this->tailIndex.load(std::memory_order_relaxed); - auto index = this->headIndex.load(std::memory_order_relaxed); - Block* block = nullptr; - assert(index == tail || details::circular_less_than(index, tail)); - bool forceFreeLastBlock = index != tail; // If we enter the loop, then the last (tail) block will not be freed - while (index != tail) { - if ((index & static_cast(BLOCK_SIZE - 1)) == 0 || block == nullptr) { - if (block != nullptr) { - // Free the old block - this->parent->add_block_to_free_list(block); - } - - block = get_block_index_entry_for_index(index)->value.load(std::memory_order_relaxed); - } - - ((*block)[index])->~T(); - ++index; - } - // Even if the queue is empty, there's still one block that's not on the free list - // (unless the head index reached the end of it, in which case the tail will be poised - // to create a new block). - if (this->tailBlock != nullptr && (forceFreeLastBlock || (tail & static_cast(BLOCK_SIZE - 1)) != 0)) { - this->parent->add_block_to_free_list(this->tailBlock); - } - - // Destroy block index - auto localBlockIndex = blockIndex.load(std::memory_order_relaxed); - if (localBlockIndex != nullptr) { - for (size_t i = 0; i != localBlockIndex->capacity; ++i) { - localBlockIndex->index[i]->~BlockIndexEntry(); - } - do { - auto prev = localBlockIndex->prev; - localBlockIndex->~BlockIndexHeader(); - (Traits::free)(localBlockIndex); - localBlockIndex = prev; - } while (localBlockIndex != nullptr); - } - } - - template - inline bool enqueue(U&& element) - { - index_t currentTailIndex = this->tailIndex.load(std::memory_order_relaxed); - index_t newTailIndex = 1 + currentTailIndex; - if ((currentTailIndex & static_cast(BLOCK_SIZE - 1)) == 0) { - // We reached the end of a block, start a new one - auto head = this->headIndex.load(std::memory_order_relaxed); - assert(!details::circular_less_than(currentTailIndex, head)); - if (!details::circular_less_than(head, currentTailIndex + BLOCK_SIZE) || (MAX_SUBQUEUE_SIZE != details::const_numeric_max::value && (MAX_SUBQUEUE_SIZE == 0 || MAX_SUBQUEUE_SIZE - BLOCK_SIZE < currentTailIndex - head))) { - return false; - } -#ifdef MCDBGQ_NOLOCKFREE_IMPLICITPRODBLOCKINDEX - debug::DebugLock lock(mutex); -#endif - // Find out where we'll be inserting this block in the block index - BlockIndexEntry* idxEntry; - if (!insert_block_index_entry(idxEntry, currentTailIndex)) { - return false; - } - - // Get ahold of a new block - auto newBlock = this->parent->ConcurrentQueue::template requisition_block(); - if (newBlock == nullptr) { - rewind_block_index_tail(); - idxEntry->value.store(nullptr, std::memory_order_relaxed); - return false; - } -#ifdef MCDBGQ_TRACKMEM - newBlock->owner = this; -#endif - newBlock->ConcurrentQueue::Block::template reset_empty(); - - MOODYCAMEL_CONSTEXPR_IF (!MOODYCAMEL_NOEXCEPT_CTOR(T, U, new (static_cast(nullptr)) T(std::forward(element)))) { - // May throw, try to insert now before we publish the fact that we have this new block - MOODYCAMEL_TRY { - new ((*newBlock)[currentTailIndex]) T(std::forward(element)); - } - MOODYCAMEL_CATCH (...) { - rewind_block_index_tail(); - idxEntry->value.store(nullptr, std::memory_order_relaxed); - this->parent->add_block_to_free_list(newBlock); - MOODYCAMEL_RETHROW; - } - } - - // Insert the new block into the index - idxEntry->value.store(newBlock, std::memory_order_relaxed); - - this->tailBlock = newBlock; - - MOODYCAMEL_CONSTEXPR_IF (!MOODYCAMEL_NOEXCEPT_CTOR(T, U, new (static_cast(nullptr)) T(std::forward(element)))) { - this->tailIndex.store(newTailIndex, std::memory_order_release); - return true; - } - } - - // Enqueue - new ((*this->tailBlock)[currentTailIndex]) T(std::forward(element)); - - this->tailIndex.store(newTailIndex, std::memory_order_release); - return true; - } - - template - bool dequeue(U& element) - { - // See ExplicitProducer::dequeue for rationale and explanation - index_t tail = this->tailIndex.load(std::memory_order_relaxed); - index_t overcommit = this->dequeueOvercommit.load(std::memory_order_relaxed); - if (details::circular_less_than(this->dequeueOptimisticCount.load(std::memory_order_relaxed) - overcommit, tail)) { - std::atomic_thread_fence(std::memory_order_acquire); - - index_t myDequeueCount = this->dequeueOptimisticCount.fetch_add(1, std::memory_order_relaxed); - tail = this->tailIndex.load(std::memory_order_acquire); - if ((details::likely)(details::circular_less_than(myDequeueCount - overcommit, tail))) { - index_t index = this->headIndex.fetch_add(1, std::memory_order_acq_rel); - - // Determine which block the element is in - auto entry = get_block_index_entry_for_index(index); - - // Dequeue - auto block = entry->value.load(std::memory_order_relaxed); - auto& el = *((*block)[index]); - - if (!MOODYCAMEL_NOEXCEPT_ASSIGN(T, T&&, element = std::move(el))) { -#ifdef MCDBGQ_NOLOCKFREE_IMPLICITPRODBLOCKINDEX - // Note: Acquiring the mutex with every dequeue instead of only when a block - // is released is very sub-optimal, but it is, after all, purely debug code. - debug::DebugLock lock(producer->mutex); -#endif - struct Guard { - Block* block; - index_t index; - BlockIndexEntry* entry; - ConcurrentQueue* parent; - - ~Guard() - { - (*block)[index]->~T(); - if (block->ConcurrentQueue::Block::template set_empty(index)) { - entry->value.store(nullptr, std::memory_order_relaxed); - parent->add_block_to_free_list(block); - } - } - } guard = { block, index, entry, this->parent }; - - element = std::move(el); // NOLINT - } - else { - element = std::move(el); // NOLINT - el.~T(); // NOLINT - - if (block->ConcurrentQueue::Block::template set_empty(index)) { - { -#ifdef MCDBGQ_NOLOCKFREE_IMPLICITPRODBLOCKINDEX - debug::DebugLock lock(mutex); -#endif - // Add the block back into the global free pool (and remove from block index) - entry->value.store(nullptr, std::memory_order_relaxed); - } - this->parent->add_block_to_free_list(block); // releases the above store - } - } - - return true; - } - else { - this->dequeueOvercommit.fetch_add(1, std::memory_order_release); - } - } - - return false; - } - -#ifdef _MSC_VER -#pragma warning(push) -#pragma warning(disable: 4706) // assignment within conditional expression -#endif - template - bool enqueue_bulk(It itemFirst, size_t count) - { - // First, we need to make sure we have enough room to enqueue all of the elements; - // this means pre-allocating blocks and putting them in the block index (but only if - // all the allocations succeeded). - - // Note that the tailBlock we start off with may not be owned by us any more; - // this happens if it was filled up exactly to the top (setting tailIndex to - // the first index of the next block which is not yet allocated), then dequeued - // completely (putting it on the free list) before we enqueue again. - - index_t startTailIndex = this->tailIndex.load(std::memory_order_relaxed); - auto startBlock = this->tailBlock; - Block* firstAllocatedBlock = nullptr; - auto endBlock = this->tailBlock; - - // Figure out how many blocks we'll need to allocate, and do so - size_t blockBaseDiff = ((startTailIndex + count - 1) & ~static_cast(BLOCK_SIZE - 1)) - ((startTailIndex - 1) & ~static_cast(BLOCK_SIZE - 1)); - index_t currentTailIndex = (startTailIndex - 1) & ~static_cast(BLOCK_SIZE - 1); - if (blockBaseDiff > 0) { -#ifdef MCDBGQ_NOLOCKFREE_IMPLICITPRODBLOCKINDEX - debug::DebugLock lock(mutex); -#endif - do { - blockBaseDiff -= static_cast(BLOCK_SIZE); - currentTailIndex += static_cast(BLOCK_SIZE); - - // Find out where we'll be inserting this block in the block index - BlockIndexEntry* idxEntry = nullptr; // initialization here unnecessary but compiler can't always tell - Block* newBlock; - bool indexInserted = false; - auto head = this->headIndex.load(std::memory_order_relaxed); - assert(!details::circular_less_than(currentTailIndex, head)); - bool full = !details::circular_less_than(head, currentTailIndex + BLOCK_SIZE) || (MAX_SUBQUEUE_SIZE != details::const_numeric_max::value && (MAX_SUBQUEUE_SIZE == 0 || MAX_SUBQUEUE_SIZE - BLOCK_SIZE < currentTailIndex - head)); - - if (full || !(indexInserted = insert_block_index_entry(idxEntry, currentTailIndex)) || (newBlock = this->parent->ConcurrentQueue::template requisition_block()) == nullptr) { - // Index allocation or block allocation failed; revert any other allocations - // and index insertions done so far for this operation - if (indexInserted) { - rewind_block_index_tail(); - idxEntry->value.store(nullptr, std::memory_order_relaxed); - } - currentTailIndex = (startTailIndex - 1) & ~static_cast(BLOCK_SIZE - 1); - for (auto block = firstAllocatedBlock; block != nullptr; block = block->next) { - currentTailIndex += static_cast(BLOCK_SIZE); - idxEntry = get_block_index_entry_for_index(currentTailIndex); - idxEntry->value.store(nullptr, std::memory_order_relaxed); - rewind_block_index_tail(); - } - this->parent->add_blocks_to_free_list(firstAllocatedBlock); - this->tailBlock = startBlock; - - return false; - } - -#ifdef MCDBGQ_TRACKMEM - newBlock->owner = this; -#endif - newBlock->ConcurrentQueue::Block::template reset_empty(); - newBlock->next = nullptr; - - // Insert the new block into the index - idxEntry->value.store(newBlock, std::memory_order_relaxed); - - // Store the chain of blocks so that we can undo if later allocations fail, - // and so that we can find the blocks when we do the actual enqueueing - if ((startTailIndex & static_cast(BLOCK_SIZE - 1)) != 0 || firstAllocatedBlock != nullptr) { - assert(this->tailBlock != nullptr); - this->tailBlock->next = newBlock; - } - this->tailBlock = newBlock; - endBlock = newBlock; - firstAllocatedBlock = firstAllocatedBlock == nullptr ? newBlock : firstAllocatedBlock; - } while (blockBaseDiff > 0); - } - - // Enqueue, one block at a time - index_t newTailIndex = startTailIndex + static_cast(count); - currentTailIndex = startTailIndex; - this->tailBlock = startBlock; - assert((startTailIndex & static_cast(BLOCK_SIZE - 1)) != 0 || firstAllocatedBlock != nullptr || count == 0); - if ((startTailIndex & static_cast(BLOCK_SIZE - 1)) == 0 && firstAllocatedBlock != nullptr) { - this->tailBlock = firstAllocatedBlock; - } - while (true) { - index_t stopIndex = (currentTailIndex & ~static_cast(BLOCK_SIZE - 1)) + static_cast(BLOCK_SIZE); - if (details::circular_less_than(newTailIndex, stopIndex)) { - stopIndex = newTailIndex; - } - MOODYCAMEL_CONSTEXPR_IF (MOODYCAMEL_NOEXCEPT_CTOR(T, decltype(*itemFirst), new (static_cast(nullptr)) T(details::deref_noexcept(itemFirst)))) { - while (currentTailIndex != stopIndex) { - new ((*this->tailBlock)[currentTailIndex++]) T(*itemFirst++); - } - } - else { - MOODYCAMEL_TRY { - while (currentTailIndex != stopIndex) { - new ((*this->tailBlock)[currentTailIndex]) T(details::nomove_if(nullptr)) T(details::deref_noexcept(itemFirst)))>::eval(*itemFirst)); - ++currentTailIndex; - ++itemFirst; - } - } - MOODYCAMEL_CATCH (...) { - auto constructedStopIndex = currentTailIndex; - auto lastBlockEnqueued = this->tailBlock; - - if (!details::is_trivially_destructible::value) { - auto block = startBlock; - if ((startTailIndex & static_cast(BLOCK_SIZE - 1)) == 0) { - block = firstAllocatedBlock; - } - currentTailIndex = startTailIndex; - while (true) { - stopIndex = (currentTailIndex & ~static_cast(BLOCK_SIZE - 1)) + static_cast(BLOCK_SIZE); - if (details::circular_less_than(constructedStopIndex, stopIndex)) { - stopIndex = constructedStopIndex; - } - while (currentTailIndex != stopIndex) { - (*block)[currentTailIndex++]->~T(); - } - if (block == lastBlockEnqueued) { - break; - } - block = block->next; - } - } - - currentTailIndex = (startTailIndex - 1) & ~static_cast(BLOCK_SIZE - 1); - for (auto block = firstAllocatedBlock; block != nullptr; block = block->next) { - currentTailIndex += static_cast(BLOCK_SIZE); - auto idxEntry = get_block_index_entry_for_index(currentTailIndex); - idxEntry->value.store(nullptr, std::memory_order_relaxed); - rewind_block_index_tail(); - } - this->parent->add_blocks_to_free_list(firstAllocatedBlock); - this->tailBlock = startBlock; - MOODYCAMEL_RETHROW; - } - } - - if (this->tailBlock == endBlock) { - assert(currentTailIndex == newTailIndex); - break; - } - this->tailBlock = this->tailBlock->next; - } - this->tailIndex.store(newTailIndex, std::memory_order_release); - return true; - } -#ifdef _MSC_VER -#pragma warning(pop) -#endif - - template - size_t dequeue_bulk(It& itemFirst, size_t max) - { - auto tail = this->tailIndex.load(std::memory_order_relaxed); - auto overcommit = this->dequeueOvercommit.load(std::memory_order_relaxed); - auto desiredCount = static_cast(tail - (this->dequeueOptimisticCount.load(std::memory_order_relaxed) - overcommit)); - if (details::circular_less_than(0, desiredCount)) { - desiredCount = desiredCount < max ? desiredCount : max; - std::atomic_thread_fence(std::memory_order_acquire); - - auto myDequeueCount = this->dequeueOptimisticCount.fetch_add(desiredCount, std::memory_order_relaxed); - - tail = this->tailIndex.load(std::memory_order_acquire); - auto actualCount = static_cast(tail - (myDequeueCount - overcommit)); - if (details::circular_less_than(0, actualCount)) { - actualCount = desiredCount < actualCount ? desiredCount : actualCount; - if (actualCount < desiredCount) { - this->dequeueOvercommit.fetch_add(desiredCount - actualCount, std::memory_order_release); - } - - // Get the first index. Note that since there's guaranteed to be at least actualCount elements, this - // will never exceed tail. - auto firstIndex = this->headIndex.fetch_add(actualCount, std::memory_order_acq_rel); - - // Iterate the blocks and dequeue - auto index = firstIndex; - BlockIndexHeader* localBlockIndex; - auto indexIndex = get_block_index_index_for_index(index, localBlockIndex); - do { - auto blockStartIndex = index; - index_t endIndex = (index & ~static_cast(BLOCK_SIZE - 1)) + static_cast(BLOCK_SIZE); - endIndex = details::circular_less_than(firstIndex + static_cast(actualCount), endIndex) ? firstIndex + static_cast(actualCount) : endIndex; - - auto entry = localBlockIndex->index[indexIndex]; - auto block = entry->value.load(std::memory_order_relaxed); - if (MOODYCAMEL_NOEXCEPT_ASSIGN(T, T&&, details::deref_noexcept(itemFirst) = std::move((*(*block)[index])))) { - while (index != endIndex) { - auto& el = *((*block)[index]); - *itemFirst++ = std::move(el); - el.~T(); - ++index; - } - } - else { - MOODYCAMEL_TRY { - while (index != endIndex) { - auto& el = *((*block)[index]); - *itemFirst = std::move(el); - ++itemFirst; - el.~T(); - ++index; - } - } - MOODYCAMEL_CATCH (...) { - do { - entry = localBlockIndex->index[indexIndex]; - block = entry->value.load(std::memory_order_relaxed); - while (index != endIndex) { - (*block)[index++]->~T(); - } - - if (block->ConcurrentQueue::Block::template set_many_empty(blockStartIndex, static_cast(endIndex - blockStartIndex))) { -#ifdef MCDBGQ_NOLOCKFREE_IMPLICITPRODBLOCKINDEX - debug::DebugLock lock(mutex); -#endif - entry->value.store(nullptr, std::memory_order_relaxed); - this->parent->add_block_to_free_list(block); - } - indexIndex = (indexIndex + 1) & (localBlockIndex->capacity - 1); - - blockStartIndex = index; - endIndex = (index & ~static_cast(BLOCK_SIZE - 1)) + static_cast(BLOCK_SIZE); - endIndex = details::circular_less_than(firstIndex + static_cast(actualCount), endIndex) ? firstIndex + static_cast(actualCount) : endIndex; - } while (index != firstIndex + actualCount); - - MOODYCAMEL_RETHROW; - } - } - if (block->ConcurrentQueue::Block::template set_many_empty(blockStartIndex, static_cast(endIndex - blockStartIndex))) { - { -#ifdef MCDBGQ_NOLOCKFREE_IMPLICITPRODBLOCKINDEX - debug::DebugLock lock(mutex); -#endif - // Note that the set_many_empty above did a release, meaning that anybody who acquires the block - // we're about to free can use it safely since our writes (and reads!) will have happened-before then. - entry->value.store(nullptr, std::memory_order_relaxed); - } - this->parent->add_block_to_free_list(block); // releases the above store - } - indexIndex = (indexIndex + 1) & (localBlockIndex->capacity - 1); - } while (index != firstIndex + actualCount); - - return actualCount; - } - else { - this->dequeueOvercommit.fetch_add(desiredCount, std::memory_order_release); - } - } - - return 0; - } - - private: - // The block size must be > 1, so any number with the low bit set is an invalid block base index - static const index_t INVALID_BLOCK_BASE = 1; - - struct BlockIndexEntry - { - std::atomic key; - std::atomic value; - }; - - struct BlockIndexHeader - { - size_t capacity; - std::atomic tail; - BlockIndexEntry* entries; - BlockIndexEntry** index; - BlockIndexHeader* prev; - }; - - template - inline bool insert_block_index_entry(BlockIndexEntry*& idxEntry, index_t blockStartIndex) - { - auto localBlockIndex = blockIndex.load(std::memory_order_relaxed); // We're the only writer thread, relaxed is OK - if (localBlockIndex == nullptr) { - return false; // this can happen if new_block_index failed in the constructor - } - size_t newTail = (localBlockIndex->tail.load(std::memory_order_relaxed) + 1) & (localBlockIndex->capacity - 1); - idxEntry = localBlockIndex->index[newTail]; - if (idxEntry->key.load(std::memory_order_relaxed) == INVALID_BLOCK_BASE || - idxEntry->value.load(std::memory_order_relaxed) == nullptr) { - - idxEntry->key.store(blockStartIndex, std::memory_order_relaxed); - localBlockIndex->tail.store(newTail, std::memory_order_release); - return true; - } - - // No room in the old block index, try to allocate another one! - MOODYCAMEL_CONSTEXPR_IF (allocMode == CannotAlloc) { - return false; - } - else if (!new_block_index()) { - return false; - } - else { - localBlockIndex = blockIndex.load(std::memory_order_relaxed); - newTail = (localBlockIndex->tail.load(std::memory_order_relaxed) + 1) & (localBlockIndex->capacity - 1); - idxEntry = localBlockIndex->index[newTail]; - assert(idxEntry->key.load(std::memory_order_relaxed) == INVALID_BLOCK_BASE); - idxEntry->key.store(blockStartIndex, std::memory_order_relaxed); - localBlockIndex->tail.store(newTail, std::memory_order_release); - return true; - } - } - - inline void rewind_block_index_tail() - { - auto localBlockIndex = blockIndex.load(std::memory_order_relaxed); - localBlockIndex->tail.store((localBlockIndex->tail.load(std::memory_order_relaxed) - 1) & (localBlockIndex->capacity - 1), std::memory_order_relaxed); - } - - inline BlockIndexEntry* get_block_index_entry_for_index(index_t index) const - { - BlockIndexHeader* localBlockIndex; - auto idx = get_block_index_index_for_index(index, localBlockIndex); - return localBlockIndex->index[idx]; - } - - inline size_t get_block_index_index_for_index(index_t index, BlockIndexHeader*& localBlockIndex) const - { -#ifdef MCDBGQ_NOLOCKFREE_IMPLICITPRODBLOCKINDEX - debug::DebugLock lock(mutex); -#endif - index &= ~static_cast(BLOCK_SIZE - 1); - localBlockIndex = blockIndex.load(std::memory_order_acquire); - auto tail = localBlockIndex->tail.load(std::memory_order_acquire); - auto tailBase = localBlockIndex->index[tail]->key.load(std::memory_order_relaxed); - assert(tailBase != INVALID_BLOCK_BASE); - // Note: Must use division instead of shift because the index may wrap around, causing a negative - // offset, whose negativity we want to preserve - auto offset = static_cast(static_cast::type>(index - tailBase) / static_cast::type>(BLOCK_SIZE)); - size_t idx = (tail + offset) & (localBlockIndex->capacity - 1); - assert(localBlockIndex->index[idx]->key.load(std::memory_order_relaxed) == index && localBlockIndex->index[idx]->value.load(std::memory_order_relaxed) != nullptr); - return idx; - } - - bool new_block_index() - { - auto prev = blockIndex.load(std::memory_order_relaxed); - size_t prevCapacity = prev == nullptr ? 0 : prev->capacity; - auto entryCount = prev == nullptr ? nextBlockIndexCapacity : prevCapacity; - auto raw = static_cast((Traits::malloc)( - sizeof(BlockIndexHeader) + - std::alignment_of::value - 1 + sizeof(BlockIndexEntry) * entryCount + - std::alignment_of::value - 1 + sizeof(BlockIndexEntry*) * nextBlockIndexCapacity)); - if (raw == nullptr) { - return false; - } - - auto header = new (raw) BlockIndexHeader; - auto entries = reinterpret_cast(details::align_for(raw + sizeof(BlockIndexHeader))); - auto index = reinterpret_cast(details::align_for(reinterpret_cast(entries) + sizeof(BlockIndexEntry) * entryCount)); - if (prev != nullptr) { - auto prevTail = prev->tail.load(std::memory_order_relaxed); - auto prevPos = prevTail; - size_t i = 0; - do { - prevPos = (prevPos + 1) & (prev->capacity - 1); - index[i++] = prev->index[prevPos]; - } while (prevPos != prevTail); - assert(i == prevCapacity); - } - for (size_t i = 0; i != entryCount; ++i) { - new (entries + i) BlockIndexEntry; - entries[i].key.store(INVALID_BLOCK_BASE, std::memory_order_relaxed); - index[prevCapacity + i] = entries + i; - } - header->prev = prev; - header->entries = entries; - header->index = index; - header->capacity = nextBlockIndexCapacity; - header->tail.store((prevCapacity - 1) & (nextBlockIndexCapacity - 1), std::memory_order_relaxed); - - blockIndex.store(header, std::memory_order_release); - - nextBlockIndexCapacity <<= 1; - - return true; - } - - private: - size_t nextBlockIndexCapacity; - std::atomic blockIndex; - -#ifdef MOODYCAMEL_CPP11_THREAD_LOCAL_SUPPORTED - public: - details::ThreadExitListener threadExitListener; - private: -#endif - -#ifdef MOODYCAMEL_QUEUE_INTERNAL_DEBUG - public: - ImplicitProducer* nextImplicitProducer; - private: -#endif - -#ifdef MCDBGQ_NOLOCKFREE_IMPLICITPRODBLOCKINDEX - mutable debug::DebugMutex mutex; -#endif -#ifdef MCDBGQ_TRACKMEM - friend struct MemStats; -#endif - }; - - - ////////////////////////////////// - // Block pool manipulation - ////////////////////////////////// - - void populate_initial_block_list(size_t blockCount) - { - initialBlockPoolSize = blockCount; - if (initialBlockPoolSize == 0) { - initialBlockPool = nullptr; - return; - } - - initialBlockPool = create_array(blockCount); - if (initialBlockPool == nullptr) { - initialBlockPoolSize = 0; - } - for (size_t i = 0; i < initialBlockPoolSize; ++i) { - initialBlockPool[i].dynamicallyAllocated = false; - } - } - - inline Block* try_get_block_from_initial_pool() - { - if (initialBlockPoolIndex.load(std::memory_order_relaxed) >= initialBlockPoolSize) { - return nullptr; - } - - auto index = initialBlockPoolIndex.fetch_add(1, std::memory_order_relaxed); - - return index < initialBlockPoolSize ? (initialBlockPool + index) : nullptr; - } - - inline void add_block_to_free_list(Block* block) - { -#ifdef MCDBGQ_TRACKMEM - block->owner = nullptr; -#endif - if (!Traits::RECYCLE_ALLOCATED_BLOCKS && block->dynamicallyAllocated) { - destroy(block); - } - else { - freeList.add(block); - } - } - - inline void add_blocks_to_free_list(Block* block) - { - while (block != nullptr) { - auto next = block->next; - add_block_to_free_list(block); - block = next; - } - } - - inline Block* try_get_block_from_free_list() - { - return freeList.try_get(); - } - - // Gets a free block from one of the memory pools, or allocates a new one (if applicable) - template - Block* requisition_block() - { - auto block = try_get_block_from_initial_pool(); - if (block != nullptr) { - return block; - } - - block = try_get_block_from_free_list(); - if (block != nullptr) { - return block; - } - - MOODYCAMEL_CONSTEXPR_IF (canAlloc == CanAlloc) { - return create(); - } - else { - return nullptr; - } - } - - -#ifdef MCDBGQ_TRACKMEM - public: - struct MemStats { - size_t allocatedBlocks; - size_t usedBlocks; - size_t freeBlocks; - size_t ownedBlocksExplicit; - size_t ownedBlocksImplicit; - size_t implicitProducers; - size_t explicitProducers; - size_t elementsEnqueued; - size_t blockClassBytes; - size_t queueClassBytes; - size_t implicitBlockIndexBytes; - size_t explicitBlockIndexBytes; - - friend class ConcurrentQueue; - - private: - static MemStats getFor(ConcurrentQueue* q) - { - MemStats stats = { 0 }; - - stats.elementsEnqueued = q->size_approx(); - - auto block = q->freeList.head_unsafe(); - while (block != nullptr) { - ++stats.allocatedBlocks; - ++stats.freeBlocks; - block = block->freeListNext.load(std::memory_order_relaxed); - } - - for (auto ptr = q->producerListTail.load(std::memory_order_acquire); ptr != nullptr; ptr = ptr->next_prod()) { - bool implicit = dynamic_cast(ptr) != nullptr; - stats.implicitProducers += implicit ? 1 : 0; - stats.explicitProducers += implicit ? 0 : 1; - - if (implicit) { - auto prod = static_cast(ptr); - stats.queueClassBytes += sizeof(ImplicitProducer); - auto head = prod->headIndex.load(std::memory_order_relaxed); - auto tail = prod->tailIndex.load(std::memory_order_relaxed); - auto hash = prod->blockIndex.load(std::memory_order_relaxed); - if (hash != nullptr) { - for (size_t i = 0; i != hash->capacity; ++i) { - if (hash->index[i]->key.load(std::memory_order_relaxed) != ImplicitProducer::INVALID_BLOCK_BASE && hash->index[i]->value.load(std::memory_order_relaxed) != nullptr) { - ++stats.allocatedBlocks; - ++stats.ownedBlocksImplicit; - } - } - stats.implicitBlockIndexBytes += hash->capacity * sizeof(typename ImplicitProducer::BlockIndexEntry); - for (; hash != nullptr; hash = hash->prev) { - stats.implicitBlockIndexBytes += sizeof(typename ImplicitProducer::BlockIndexHeader) + hash->capacity * sizeof(typename ImplicitProducer::BlockIndexEntry*); - } - } - for (; details::circular_less_than(head, tail); head += BLOCK_SIZE) { - //auto block = prod->get_block_index_entry_for_index(head); - ++stats.usedBlocks; - } - } - else { - auto prod = static_cast(ptr); - stats.queueClassBytes += sizeof(ExplicitProducer); - auto tailBlock = prod->tailBlock; - bool wasNonEmpty = false; - if (tailBlock != nullptr) { - auto block = tailBlock; - do { - ++stats.allocatedBlocks; - if (!block->ConcurrentQueue::Block::template is_empty() || wasNonEmpty) { - ++stats.usedBlocks; - wasNonEmpty = wasNonEmpty || block != tailBlock; - } - ++stats.ownedBlocksExplicit; - block = block->next; - } while (block != tailBlock); - } - auto index = prod->blockIndex.load(std::memory_order_relaxed); - while (index != nullptr) { - stats.explicitBlockIndexBytes += sizeof(typename ExplicitProducer::BlockIndexHeader) + index->size * sizeof(typename ExplicitProducer::BlockIndexEntry); - index = static_cast(index->prev); - } - } - } - - auto freeOnInitialPool = q->initialBlockPoolIndex.load(std::memory_order_relaxed) >= q->initialBlockPoolSize ? 0 : q->initialBlockPoolSize - q->initialBlockPoolIndex.load(std::memory_order_relaxed); - stats.allocatedBlocks += freeOnInitialPool; - stats.freeBlocks += freeOnInitialPool; - - stats.blockClassBytes = sizeof(Block) * stats.allocatedBlocks; - stats.queueClassBytes += sizeof(ConcurrentQueue); - - return stats; - } - }; - - // For debugging only. Not thread-safe. - MemStats getMemStats() - { - return MemStats::getFor(this); - } - private: - friend struct MemStats; -#endif - - - ////////////////////////////////// - // Producer list manipulation - ////////////////////////////////// - - ProducerBase* recycle_or_create_producer(bool isExplicit) - { -#ifdef MCDBGQ_NOLOCKFREE_IMPLICITPRODHASH - debug::DebugLock lock(implicitProdMutex); -#endif - // Try to re-use one first - for (auto ptr = producerListTail.load(std::memory_order_acquire); ptr != nullptr; ptr = ptr->next_prod()) { - if (ptr->inactive.load(std::memory_order_relaxed) && ptr->isExplicit == isExplicit) { - bool expected = true; - if (ptr->inactive.compare_exchange_strong(expected, /* desired */ false, std::memory_order_acquire, std::memory_order_relaxed)) { - // We caught one! It's been marked as activated, the caller can have it - return ptr; - } - } - } - - return add_producer(isExplicit ? static_cast(create(this)) : create(this)); - } - - ProducerBase* add_producer(ProducerBase* producer) - { - // Handle failed memory allocation - if (producer == nullptr) { - return nullptr; - } - - producerCount.fetch_add(1, std::memory_order_relaxed); - - // Add it to the lock-free list - auto prevTail = producerListTail.load(std::memory_order_relaxed); - do { - producer->next = prevTail; - } while (!producerListTail.compare_exchange_weak(prevTail, producer, std::memory_order_release, std::memory_order_relaxed)); - -#ifdef MOODYCAMEL_QUEUE_INTERNAL_DEBUG - if (producer->isExplicit) { - auto prevTailExplicit = explicitProducers.load(std::memory_order_relaxed); - do { - static_cast(producer)->nextExplicitProducer = prevTailExplicit; - } while (!explicitProducers.compare_exchange_weak(prevTailExplicit, static_cast(producer), std::memory_order_release, std::memory_order_relaxed)); - } - else { - auto prevTailImplicit = implicitProducers.load(std::memory_order_relaxed); - do { - static_cast(producer)->nextImplicitProducer = prevTailImplicit; - } while (!implicitProducers.compare_exchange_weak(prevTailImplicit, static_cast(producer), std::memory_order_release, std::memory_order_relaxed)); - } -#endif - - return producer; - } - - void reown_producers() - { - // After another instance is moved-into/swapped-with this one, all the - // producers we stole still think their parents are the other queue. - // So fix them up! - for (auto ptr = producerListTail.load(std::memory_order_relaxed); ptr != nullptr; ptr = ptr->next_prod()) { - ptr->parent = this; - } - } - - - ////////////////////////////////// - // Implicit producer hash - ////////////////////////////////// - - struct ImplicitProducerKVP - { - std::atomic key; - ImplicitProducer* value; // No need for atomicity since it's only read by the thread that sets it in the first place - - ImplicitProducerKVP() : value(nullptr) { } - - ImplicitProducerKVP(ImplicitProducerKVP&& other) MOODYCAMEL_NOEXCEPT - { - key.store(other.key.load(std::memory_order_relaxed), std::memory_order_relaxed); - value = other.value; - } - - inline ImplicitProducerKVP& operator=(ImplicitProducerKVP&& other) MOODYCAMEL_NOEXCEPT - { - swap(other); - return *this; - } - - inline void swap(ImplicitProducerKVP& other) MOODYCAMEL_NOEXCEPT - { - if (this != &other) { - details::swap_relaxed(key, other.key); - std::swap(value, other.value); - } - } - }; - - template - friend void moodycamel::swap(typename ConcurrentQueue::ImplicitProducerKVP&, typename ConcurrentQueue::ImplicitProducerKVP&) MOODYCAMEL_NOEXCEPT; - - struct ImplicitProducerHash - { - size_t capacity; - ImplicitProducerKVP* entries; - ImplicitProducerHash* prev; - }; - - inline void populate_initial_implicit_producer_hash() - { - MOODYCAMEL_CONSTEXPR_IF (INITIAL_IMPLICIT_PRODUCER_HASH_SIZE == 0) { - return; - } - else { - implicitProducerHashCount.store(0, std::memory_order_relaxed); - auto hash = &initialImplicitProducerHash; - hash->capacity = INITIAL_IMPLICIT_PRODUCER_HASH_SIZE; - hash->entries = &initialImplicitProducerHashEntries[0]; - for (size_t i = 0; i != INITIAL_IMPLICIT_PRODUCER_HASH_SIZE; ++i) { - initialImplicitProducerHashEntries[i].key.store(details::invalid_thread_id, std::memory_order_relaxed); - } - hash->prev = nullptr; - implicitProducerHash.store(hash, std::memory_order_relaxed); - } - } - - void swap_implicit_producer_hashes(ConcurrentQueue& other) - { - MOODYCAMEL_CONSTEXPR_IF (INITIAL_IMPLICIT_PRODUCER_HASH_SIZE == 0) { - return; - } - else { - // Swap (assumes our implicit producer hash is initialized) - initialImplicitProducerHashEntries.swap(other.initialImplicitProducerHashEntries); - initialImplicitProducerHash.entries = &initialImplicitProducerHashEntries[0]; - other.initialImplicitProducerHash.entries = &other.initialImplicitProducerHashEntries[0]; - - details::swap_relaxed(implicitProducerHashCount, other.implicitProducerHashCount); - - details::swap_relaxed(implicitProducerHash, other.implicitProducerHash); - if (implicitProducerHash.load(std::memory_order_relaxed) == &other.initialImplicitProducerHash) { - implicitProducerHash.store(&initialImplicitProducerHash, std::memory_order_relaxed); - } - else { - ImplicitProducerHash* hash; - for (hash = implicitProducerHash.load(std::memory_order_relaxed); hash->prev != &other.initialImplicitProducerHash; hash = hash->prev) { - continue; - } - hash->prev = &initialImplicitProducerHash; - } - if (other.implicitProducerHash.load(std::memory_order_relaxed) == &initialImplicitProducerHash) { - other.implicitProducerHash.store(&other.initialImplicitProducerHash, std::memory_order_relaxed); - } - else { - ImplicitProducerHash* hash; - for (hash = other.implicitProducerHash.load(std::memory_order_relaxed); hash->prev != &initialImplicitProducerHash; hash = hash->prev) { - continue; - } - hash->prev = &other.initialImplicitProducerHash; - } - } - } - - // Only fails (returns nullptr) if memory allocation fails - ImplicitProducer* get_or_add_implicit_producer() - { - // Note that since the data is essentially thread-local (key is thread ID), - // there's a reduced need for fences (memory ordering is already consistent - // for any individual thread), except for the current table itself. - - // Start by looking for the thread ID in the current and all previous hash tables. - // If it's not found, it must not be in there yet, since this same thread would - // have added it previously to one of the tables that we traversed. - - // Code and algorithm adapted from http://preshing.com/20130605/the-worlds-simplest-lock-free-hash-table - -#ifdef MCDBGQ_NOLOCKFREE_IMPLICITPRODHASH - debug::DebugLock lock(implicitProdMutex); -#endif - - auto id = details::thread_id(); - auto hashedId = details::hash_thread_id(id); - - auto mainHash = implicitProducerHash.load(std::memory_order_acquire); - assert(mainHash != nullptr); // silence clang-tidy and MSVC warnings (hash cannot be null) - for (auto hash = mainHash; hash != nullptr; hash = hash->prev) { - // Look for the id in this hash - auto index = hashedId; - while (true) { // Not an infinite loop because at least one slot is free in the hash table - index &= hash->capacity - 1u; - - auto probedKey = hash->entries[index].key.load(std::memory_order_relaxed); - if (probedKey == id) { - // Found it! If we had to search several hashes deep, though, we should lazily add it - // to the current main hash table to avoid the extended search next time. - // Note there's guaranteed to be room in the current hash table since every subsequent - // table implicitly reserves space for all previous tables (there's only one - // implicitProducerHashCount). - auto value = hash->entries[index].value; - if (hash != mainHash) { - index = hashedId; - while (true) { - index &= mainHash->capacity - 1u; - auto empty = details::invalid_thread_id; -#ifdef MOODYCAMEL_CPP11_THREAD_LOCAL_SUPPORTED - auto reusable = details::invalid_thread_id2; - if (mainHash->entries[index].key.compare_exchange_strong(empty, id, std::memory_order_seq_cst, std::memory_order_relaxed) || - mainHash->entries[index].key.compare_exchange_strong(reusable, id, std::memory_order_seq_cst, std::memory_order_relaxed)) { -#else - if (mainHash->entries[index].key.compare_exchange_strong(empty, id, std::memory_order_seq_cst, std::memory_order_relaxed)) { -#endif - mainHash->entries[index].value = value; - break; - } - ++index; - } - } - - return value; - } - if (probedKey == details::invalid_thread_id) { - break; // Not in this hash table - } - ++index; - } - } - - // Insert! - auto newCount = 1 + implicitProducerHashCount.fetch_add(1, std::memory_order_relaxed); - while (true) { - // NOLINTNEXTLINE(clang-analyzer-core.NullDereference) - if (newCount >= (mainHash->capacity >> 1) && !implicitProducerHashResizeInProgress.test_and_set(std::memory_order_acquire)) { - // We've acquired the resize lock, try to allocate a bigger hash table. - // Note the acquire fence synchronizes with the release fence at the end of this block, and hence when - // we reload implicitProducerHash it must be the most recent version (it only gets changed within this - // locked block). - mainHash = implicitProducerHash.load(std::memory_order_acquire); - if (newCount >= (mainHash->capacity >> 1)) { - size_t newCapacity = mainHash->capacity << 1; - while (newCount >= (newCapacity >> 1)) { - newCapacity <<= 1; - } - auto raw = static_cast((Traits::malloc)(sizeof(ImplicitProducerHash) + std::alignment_of::value - 1 + sizeof(ImplicitProducerKVP) * newCapacity)); - if (raw == nullptr) { - // Allocation failed - implicitProducerHashCount.fetch_sub(1, std::memory_order_relaxed); - implicitProducerHashResizeInProgress.clear(std::memory_order_relaxed); - return nullptr; - } - - auto newHash = new (raw) ImplicitProducerHash; - newHash->capacity = static_cast(newCapacity); - newHash->entries = reinterpret_cast(details::align_for(raw + sizeof(ImplicitProducerHash))); - for (size_t i = 0; i != newCapacity; ++i) { - new (newHash->entries + i) ImplicitProducerKVP; - newHash->entries[i].key.store(details::invalid_thread_id, std::memory_order_relaxed); - } - newHash->prev = mainHash; - implicitProducerHash.store(newHash, std::memory_order_release); - implicitProducerHashResizeInProgress.clear(std::memory_order_release); - mainHash = newHash; - } - else { - implicitProducerHashResizeInProgress.clear(std::memory_order_release); - } - } - - // If it's < three-quarters full, add to the old one anyway so that we don't have to wait for the next table - // to finish being allocated by another thread (and if we just finished allocating above, the condition will - // always be true) - if (newCount < (mainHash->capacity >> 1) + (mainHash->capacity >> 2)) { - auto producer = static_cast(recycle_or_create_producer(false)); - if (producer == nullptr) { - implicitProducerHashCount.fetch_sub(1, std::memory_order_relaxed); - return nullptr; - } - -#ifdef MOODYCAMEL_CPP11_THREAD_LOCAL_SUPPORTED - producer->threadExitListener.callback = &ConcurrentQueue::implicit_producer_thread_exited_callback; - producer->threadExitListener.userData = producer; - details::ThreadExitNotifier::subscribe(&producer->threadExitListener); -#endif - - auto index = hashedId; - while (true) { - index &= mainHash->capacity - 1u; - auto empty = details::invalid_thread_id; -#ifdef MOODYCAMEL_CPP11_THREAD_LOCAL_SUPPORTED - auto reusable = details::invalid_thread_id2; - if (mainHash->entries[index].key.compare_exchange_strong(reusable, id, std::memory_order_seq_cst, std::memory_order_relaxed)) { - implicitProducerHashCount.fetch_sub(1, std::memory_order_relaxed); // already counted as a used slot - mainHash->entries[index].value = producer; - break; - } -#endif - if (mainHash->entries[index].key.compare_exchange_strong(empty, id, std::memory_order_seq_cst, std::memory_order_relaxed)) { - mainHash->entries[index].value = producer; - break; - } - ++index; - } - return producer; - } - - // Hmm, the old hash is quite full and somebody else is busy allocating a new one. - // We need to wait for the allocating thread to finish (if it succeeds, we add, if not, - // we try to allocate ourselves). - mainHash = implicitProducerHash.load(std::memory_order_acquire); - } - } - -#ifdef MOODYCAMEL_CPP11_THREAD_LOCAL_SUPPORTED - void implicit_producer_thread_exited(ImplicitProducer* producer) - { - // Remove from hash -#ifdef MCDBGQ_NOLOCKFREE_IMPLICITPRODHASH - debug::DebugLock lock(implicitProdMutex); -#endif - auto hash = implicitProducerHash.load(std::memory_order_acquire); - assert(hash != nullptr); // The thread exit listener is only registered if we were added to a hash in the first place - auto id = details::thread_id(); - auto hashedId = details::hash_thread_id(id); - details::thread_id_t probedKey; - - // We need to traverse all the hashes just in case other threads aren't on the current one yet and are - // trying to add an entry thinking there's a free slot (because they reused a producer) - for (; hash != nullptr; hash = hash->prev) { - auto index = hashedId; - do { - index &= hash->capacity - 1u; - probedKey = id; - if (hash->entries[index].key.compare_exchange_strong(probedKey, details::invalid_thread_id2, std::memory_order_seq_cst, std::memory_order_relaxed)) { - break; - } - ++index; - } while (probedKey != details::invalid_thread_id); // Can happen if the hash has changed but we weren't put back in it yet, or if we weren't added to this hash in the first place - } - - // Mark the queue as being recyclable - producer->inactive.store(true, std::memory_order_release); - } - - static void implicit_producer_thread_exited_callback(void* userData) - { - auto producer = static_cast(userData); - auto queue = producer->parent; - queue->implicit_producer_thread_exited(producer); - } -#endif - - ////////////////////////////////// - // Utility functions - ////////////////////////////////// - - template - static inline void* aligned_malloc(size_t size) - { - MOODYCAMEL_CONSTEXPR_IF (std::alignment_of::value <= std::alignment_of::value) - return (Traits::malloc)(size); - else { - size_t alignment = std::alignment_of::value; - void* raw = (Traits::malloc)(size + alignment - 1 + sizeof(void*)); - if (!raw) - return nullptr; - char* ptr = details::align_for(reinterpret_cast(raw) + sizeof(void*)); - *(reinterpret_cast(ptr) - 1) = raw; - return ptr; - } - } - - template - static inline void aligned_free(void* ptr) - { - MOODYCAMEL_CONSTEXPR_IF (std::alignment_of::value <= std::alignment_of::value) - return (Traits::free)(ptr); - else - (Traits::free)(ptr ? *(reinterpret_cast(ptr) - 1) : nullptr); - } - - template - static inline U* create_array(size_t count) - { - assert(count > 0); - U* p = static_cast(aligned_malloc(sizeof(U) * count)); - if (p == nullptr) - return nullptr; - - for (size_t i = 0; i != count; ++i) - new (p + i) U(); - return p; - } - - template - static inline void destroy_array(U* p, size_t count) - { - if (p != nullptr) { - assert(count > 0); - for (size_t i = count; i != 0; ) - (p + --i)->~U(); - } - aligned_free(p); - } - - template - static inline U* create() - { - void* p = aligned_malloc(sizeof(U)); - return p != nullptr ? new (p) U : nullptr; - } - - template - static inline U* create(A1&& a1) - { - void* p = aligned_malloc(sizeof(U)); - return p != nullptr ? new (p) U(std::forward(a1)) : nullptr; - } - - template - static inline void destroy(U* p) - { - if (p != nullptr) - p->~U(); - aligned_free(p); - } - -private: - std::atomic producerListTail; - std::atomic producerCount; - - std::atomic initialBlockPoolIndex; - Block* initialBlockPool; - size_t initialBlockPoolSize; - -#ifndef MCDBGQ_USEDEBUGFREELIST - FreeList freeList; -#else - debug::DebugFreeList freeList; -#endif - - std::atomic implicitProducerHash; - std::atomic implicitProducerHashCount; // Number of slots logically used - ImplicitProducerHash initialImplicitProducerHash; - std::array initialImplicitProducerHashEntries; - std::atomic_flag implicitProducerHashResizeInProgress; - - std::atomic nextExplicitConsumerId; - std::atomic globalExplicitConsumerOffset; - -#ifdef MCDBGQ_NOLOCKFREE_IMPLICITPRODHASH - debug::DebugMutex implicitProdMutex; -#endif - -#ifdef MOODYCAMEL_QUEUE_INTERNAL_DEBUG - std::atomic explicitProducers; - std::atomic implicitProducers; -#endif -}; - - -template -ProducerToken::ProducerToken(ConcurrentQueue& queue) - : producer(queue.recycle_or_create_producer(true)) -{ - if (producer != nullptr) { - producer->token = this; - } -} - -template -ProducerToken::ProducerToken(BlockingConcurrentQueue& queue) - : producer(reinterpret_cast*>(&queue)->recycle_or_create_producer(true)) -{ - if (producer != nullptr) { - producer->token = this; - } -} - -template -ConsumerToken::ConsumerToken(ConcurrentQueue& queue) - : itemsConsumedFromCurrent(0), currentProducer(nullptr), desiredProducer(nullptr) -{ - initialOffset = queue.nextExplicitConsumerId.fetch_add(1, std::memory_order_release); - lastKnownGlobalOffset = static_cast(-1); -} - -template -ConsumerToken::ConsumerToken(BlockingConcurrentQueue& queue) - : itemsConsumedFromCurrent(0), currentProducer(nullptr), desiredProducer(nullptr) -{ - initialOffset = reinterpret_cast*>(&queue)->nextExplicitConsumerId.fetch_add(1, std::memory_order_release); - lastKnownGlobalOffset = static_cast(-1); -} - -template -inline void swap(ConcurrentQueue& a, ConcurrentQueue& b) MOODYCAMEL_NOEXCEPT -{ - a.swap(b); -} - -inline void swap(ProducerToken& a, ProducerToken& b) MOODYCAMEL_NOEXCEPT -{ - a.swap(b); -} - -inline void swap(ConsumerToken& a, ConsumerToken& b) MOODYCAMEL_NOEXCEPT -{ - a.swap(b); -} - -template -inline void swap(typename ConcurrentQueue::ImplicitProducerKVP& a, typename ConcurrentQueue::ImplicitProducerKVP& b) MOODYCAMEL_NOEXCEPT -{ - a.swap(b); -} - -} - -#if defined(_MSC_VER) && (!defined(_HAS_CXX17) || !_HAS_CXX17) -#pragma warning(pop) -#endif - -#if defined(__GNUC__) && !defined(__INTEL_COMPILER) -#pragma GCC diagnostic pop -#endif diff --git a/Sources/TetraConcurrentQueueShim/include/TetraConcurrentQueueShim.h b/Sources/TetraConcurrentQueueShim/include/TetraConcurrentQueueShim.h deleted file mode 100644 index 6078821..0000000 --- a/Sources/TetraConcurrentQueueShim/include/TetraConcurrentQueueShim.h +++ /dev/null @@ -1,13 +0,0 @@ -// -// Header.h -// Tetra -// -// Created by 박병관 on 1/20/25. -// - -#ifndef TetraConcurrentQueue_h -#define TetraConcurrentQueue_h - -#include "sim.h" - -#endif /* Header_h */ diff --git a/Sources/TetraConcurrentQueueShim/include/sim.h b/Sources/TetraConcurrentQueueShim/include/sim.h deleted file mode 100644 index b393865..0000000 --- a/Sources/TetraConcurrentQueueShim/include/sim.h +++ /dev/null @@ -1,65 +0,0 @@ -// -// Header.h -// Tetra -// -// Created by 박병관 on 1/20/25. -// - -#ifndef ConcurrentQueue_shim_h -#define ConcurrentQueue_shim_h -//#include -//#include "concurrentqueue.hpp" -#include -#include - - -CF_ASSUME_NONNULL_BEGIN - -typedef struct { - void(*perform)(CFTypeRef state, CFArrayRef job); - void(* _Nullable schedule)(CFTypeRef state, CFRunLoopRef rl, CFRunLoopMode mode); - void(* _Nullable cancel)(CFTypeRef state, CFRunLoopRef rl, CFRunLoopMode mode); -} TetraContextData; - -//class -//SWIFT_NONCOPYABLE -//SWIFT_SHARED_REFERENCE(retainSharedObject, releaseSharedObject) -//MyBookQueue { -//public: -// static MyBookQueue* create(); -// -// bool enqueue(CFTypeRef ref); -// CF_RETURNS_NOT_RETAINED _Nullable CFTypeRef try_dequeue(); -//private: -// MyBookQueue(); -// CFDataRef data; -//// class MyActualType; -//// std::unique_ptr impl; -//}; - -CF_EXTERN_C_BEGIN - -bool enqueue_ref_concurrent_queue(void* queue, CFTypeRef ref); - -CF_RETURNS_RETAINED CFRunLoopSourceRef create_tetra_runLoop_executor( - CFTypeRef initialState, - const TetraContextData *tetraContext -); - -CF_RETURNS_RETAINED CFDictionaryRef copy_tetra_runLoop_registry(CFRunLoopSourceRef source); - -bool tetra_enqueue_and_signal(CFRunLoopSourceRef source, CFTypeRef ref); - -CF_RETURNS_NOT_RETAINED -CFTypeRef tetra_get_stateInfo(CFRunLoopSourceRef source); - -CF_EXTERN_C_END - -//void retainSharedObject(MyBookQueue* ref); -//void releaseSharedObject(MyBookQueue* _Nullable ref); - -CF_ASSUME_NONNULL_END - - - -#endif /* ConcurrentQueue_shim_h */ diff --git a/Sources/TetraConcurrentQueueShim/sim.cpp b/Sources/TetraConcurrentQueueShim/sim.cpp deleted file mode 100644 index f4bf0d0..0000000 --- a/Sources/TetraConcurrentQueueShim/sim.cpp +++ /dev/null @@ -1,254 +0,0 @@ -// -// sim.m -// Tetra -// -// Created by 박병관 on 1/20/25. -// - -#define MOODYCAMEL_NO_THREAD_LOCAL -#include "concurrentqueue.h" -#include "sim.h" -#include -#if __APPLE__ -#include -#endif - -struct CFQueueTrait: moodycamel::ConcurrentQueueDefaultTraits { - CF_INLINE void* malloc(size_t size) { - return CFAllocatorAllocate(kCFAllocatorDefault, size, 0); - } - - CF_INLINE void free(void *ptr) { - return CFAllocatorDeallocate(kCFAllocatorDefault, ptr); - } -}; - - -//#undef __APPLE__ -typedef std::shared_ptr CFCppRef; -typedef moodycamel::ConcurrentQueue MyConcurrentQueue; - -typedef struct { - MyConcurrentQueue queue; - moodycamel::ConsumerToken token; - TetraContextData context; - CFTypeRef state; - CFMutableDictionaryRef runLoopRegistry; -#if __APPLE__ - os_unfair_lock_s lock; -#else - std::mutex* lock; -#endif - -} RunLoopContextInfo; - -CFRunLoopSourceRef create_tetra_runLoop_executor( - CFTypeRef initialState, - const TetraContextData *tetraContext -) { - auto queue = MyConcurrentQueue(); - auto token = moodycamel::ConsumerToken(queue); - - RunLoopContextInfo stackInfo = RunLoopContextInfo{ - std::move(queue), - std::move(token), - *tetraContext, - initialState, - CFDictionaryCreateMutable(kCFAllocatorDefault, 0, &kCFTypeDictionaryKeyCallBacks, &kCFTypeDictionaryValueCallBacks), -#if __APPLE__ - OS_UNFAIR_LOCK_INIT, -#else - new (CFQueueTrait::malloc(sizeof(std::mutex))) std::mutex(), -#endif - }; - CFRunLoopSourceContext soureContext = { - 0, - (void*)&stackInfo, - [](const void * stackRawInfo) -> const void * { - void * buffer = CFAllocatorAllocate(kCFAllocatorDefault, sizeof(RunLoopContextInfo), 0); - RunLoopContextInfo* stackInfo = reinterpret_cast(const_cast(stackRawInfo)); - - auto myInfo = new (buffer) RunLoopContextInfo(std::move(*stackInfo)); - myInfo->state = CFRetain(stackInfo->state); - return myInfo; - }, - [](const void * heapRawInfo) { - auto info = reinterpret_cast(const_cast(heapRawInfo)); -#if !__APPLE__ - info->lock->~mutex(); - CFQueueTrait::free(info->lock); -#endif - auto stack = std::move(*info); - CFAllocatorDeallocate(kCFAllocatorDefault, info); - CFRelease(stack.runLoopRegistry); - CFRelease(stack.state); - }, - nullptr, - nullptr, - nullptr, - [](void * info, CFRunLoopRef runLoop, CFRunLoopMode mode) { - //schedule - auto &sourceInfo = *reinterpret_cast(info); - CFRunLoopWakeUp(runLoop); - { - CFMutableDictionaryRef registry = sourceInfo.runLoopRegistry; -#if __APPLE__ - os_unfair_lock_lock(&sourceInfo.lock); -#else - std::lock_guard lock(*sourceInfo.lock); -#endif - CFMutableSetRef set = (CFMutableSetRef)CFDictionaryGetValue(registry, runLoop); - if (!set) { - set = CFSetCreateMutable(kCFAllocatorDefault, 0, &kCFTypeSetCallBacks); - CFDictionarySetValue(registry, runLoop, set); - CFRelease(set); - } -#if __APPLE__ - os_unfair_lock_unlock(&sourceInfo.lock); -#endif - CFSetAddValue(set, mode); - } - if (sourceInfo.context.schedule) { - sourceInfo.context.schedule(sourceInfo.state, runLoop, mode); - } - }, - [](void * info, CFRunLoopRef runLoop, CFRunLoopMode mode) { - // cancel - auto &sourceInfo = *reinterpret_cast(info); - CFRunLoopWakeUp(runLoop); - { - CFMutableDictionaryRef registry = sourceInfo.runLoopRegistry; -#if __APPLE__ - os_unfair_lock_lock(&sourceInfo.lock); -#else - std::lock_guard lock(*sourceInfo.lock); -#endif - CFMutableSetRef set = (CFMutableSetRef)CFDictionaryGetValue(registry, runLoop); - - CFSetRemoveValue(set, mode); - - if (CFSetGetCount(set) == 0) { - CFDictionaryRemoveValue(registry, runLoop); - } -#if __APPLE__ - os_unfair_lock_unlock(&sourceInfo.lock); -#endif - } - if (sourceInfo.context.schedule) { - sourceInfo.context.schedule(sourceInfo.state, runLoop, mode); - } - - }, - [](void *info) { - auto &sourceInfo = *reinterpret_cast(info); - - - CFMutableArrayRef dequeue = CFArrayCreateMutable(kCFAllocatorDefault, 0, &kCFTypeArrayCallBacks); - constexpr size_t buffer_size = 10; - CFCppRef result[buffer_size]; - size_t size = 0; - while ((size = sourceInfo.queue.try_dequeue_bulk(sourceInfo.token, result, buffer_size)) > 0) { - for (int i = 0; i < size; i++) { - CFCppRef ref = std::move(result[i]); - CFArrayAppendValue(dequeue, ref.get()); - } - } - sourceInfo.context.perform(sourceInfo.state, dequeue); - CFRelease(dequeue); - } - }; - - CFRunLoopSourceRef source = CFRunLoopSourceCreate(kCFAllocatorDefault, 0, &soureContext); - return source; -} - -CFDictionaryRef copy_tetra_runLoop_registry(CFRunLoopSourceRef source) { - RunLoopContextInfo* info; - { - CFRunLoopSourceContext context = {}; - CFRunLoopSourceGetContext(source, &context); - info = reinterpret_cast(context.info); - } - RunLoopContextInfo& variable = *info; - CFDictionaryRef registry; - { -#if __APPLE__ - os_unfair_lock_lock(&variable.lock); -#else - std::lock_guard lock(*variable.lock); -#endif - registry = CFDictionaryCreateCopy(kCFAllocatorDefault, variable.runLoopRegistry); -#if __APPLE__ - os_unfair_lock_unlock(&variable.lock); -#endif - } - return registry; -} - -CF_INLINE CFDictionaryRef try_copy_tetra_runLoop_registry(CFRunLoopSourceRef source) { - RunLoopContextInfo* info; - { - CFRunLoopSourceContext context = {}; - CFRunLoopSourceGetContext(source, &context); - info = reinterpret_cast(context.info); - } - RunLoopContextInfo& variable = *info; - CFDictionaryRef registry; - { -#if __APPLE__ - if (os_unfair_lock_trylock(&variable.lock) == false) { - return nullptr; - } -#else - std::unique_lock lock(*variable.lock, std::try_to_lock); - if(!lock.owns_lock()){ - return nullptr; - } -#endif - registry = CFDictionaryCreateCopy(kCFAllocatorDefault, variable.runLoopRegistry); -#if __APPLE__ - os_unfair_lock_unlock(&variable.lock); -#endif - } - return registry; -} - - -bool tetra_enqueue_and_signal(CFRunLoopSourceRef source, CFTypeRef ref) { - RunLoopContextInfo* info; - { - CFRunLoopSourceContext context = {}; - CFRunLoopSourceGetContext(source, &context); - info = reinterpret_cast(context.info); - } - RunLoopContextInfo& variable = *info; - auto ptr = CFCppRef(CFRetain(ref), CFRelease); - const bool success = variable.queue.enqueue(std::move(ptr)); - if (!success) { - return false; - } - CFRunLoopSourceSignal(source); - CFDictionaryRef registry = try_copy_tetra_runLoop_registry(source); - // somebody is already waking up the runloop - if (registry == nullptr) { - - return true; - } - CFDictionaryApplyFunction(registry, [](CFTypeRef key, CFTypeRef value, void * info) { - if (CFRunLoopIsWaiting((CFRunLoopRef) key)) { - CFRunLoopWakeUp((CFRunLoopRef) key); - } - }, nullptr); - CFRelease(registry); - return true; -} - -CFTypeRef tetra_get_stateInfo(CFRunLoopSourceRef source) { - RunLoopContextInfo* info; - { - CFRunLoopSourceContext context = {}; - CFRunLoopSourceGetContext(source, &context); - info = reinterpret_cast(context.info); - } - return info->state; -} diff --git a/Sources/TetraRunLoopConcurrency/MPMCBoundedQueue.swift b/Sources/TetraRunLoopConcurrency/MPMCBoundedQueue.swift new file mode 100644 index 0000000..561d5c4 --- /dev/null +++ b/Sources/TetraRunLoopConcurrency/MPMCBoundedQueue.swift @@ -0,0 +1,150 @@ +// +// MPMCBoundedQueue.swift +// Tetra +// +// Created by 박병관 on 2/14/26. +// +import Atomics +import CriticalSection + +internal struct MPMCBoundedQueue: ~Copyable, @unchecked Sendable { + @usableFromInline + internal struct BufferNode: ~Copyable { + @usableFromInline + internal var data: Element? + + @usableFromInline + internal let sequence: AtomicStore = .init(0) + + @inlinable + init(data: consuming Element) { + self.data = consume data + } + + @inlinable + init() { + self.data = nil + } + } + + @usableFromInline + internal let mask: Int + + @usableFromInline + internal let buffer: ManagedBuffer + + @usableFromInline + internal let head: AtomicStore = .init(0) + + @usableFromInline + internal let tail: AtomicStore = .init(0) + + public var count: Int { + let headIndex = head.load(ordering: .relaxed) + let tailIndex = tail.load(ordering: .relaxed) + return tailIndex < headIndex ? (buffer.header - headIndex + tailIndex) : (tailIndex - headIndex) + } + + public var wasFull: Bool { + buffer.header - count == 1 + } + + public init(size: Int) { + let size = size.nextPowerOf2() + self.mask = size - 1 + self.buffer = .create(minimumCapacity: size, makingHeaderWith: { _ in + size + }) + buffer.withUnsafeMutablePointerToElements { + let pointer = UnsafeMutableBufferPointer(start: $0, count: size) + for i in 0.. sending Element? { + var result:Element? = consume value + return buffer.withUnsafeMutablePointers { + + let pointer = UnsafeMutableBufferPointer(start: $1, count: $0.pointee) + var node: UnsafeMutablePointer! + var pos = tail.load(ordering: .relaxed) + + while true { + node = pointer.baseAddress?.advanced(by: pos & mask) + let seq = node.pointee.sequence.load(ordering: .acquiring) + let difference = seq - pos + + if difference == 0 { + if tail.weakCompareExchange(expected: pos, + desired: pos + 1, + successOrdering: .relaxed, + failureOrdering: .relaxed).exchanged { + break + } + } else if difference < 0 { + let a = consume result + result = nil + return a + } else { + pos = tail.load(ordering: .relaxed) + } + } + swap(&node.pointee.data, &result) + + node.pointee.sequence.store(pos + 1, ordering: .releasing) + return nil + } + + } + + @inlinable + public func dequeue() -> sending Element? { + return buffer.withUnsafeMutablePointers { + let pointer = UnsafeMutableBufferPointer(start: $1, count: $0.pointee) + var node: UnsafeMutablePointer! + var pos = head.load(ordering: .relaxed) + + while true { + node = pointer.baseAddress?.advanced(by: pos & mask) + let seq = node.pointee.sequence.load(ordering: .acquiring) + let difference = seq - (pos + 1) + + if difference == 0 { + if head.weakCompareExchange(expected: pos, desired: pos + 1, successOrdering: .relaxed, failureOrdering: .relaxed).exchanged { + break + } + } else if difference < 0 { + return nil + } else { + pos = head.load(ordering: .relaxed) + } + } + var result: Element? = nil + swap(&result, &node.pointee.data) +// let result = node.pointee.data.move() + node.pointee.sequence.store(pos + mask + 1, ordering: .releasing) + return result + } + + } + + @inline(__always) + public func dequeueAll(_ closure: (consuming sending Element) -> Void) { + while let element = dequeue() { + closure(element) + } + } +} diff --git a/Sources/TetraRunLoopConcurrency/SlicedJobQueue.swift b/Sources/TetraRunLoopConcurrency/SlicedJobQueue.swift new file mode 100644 index 0000000..45ace73 --- /dev/null +++ b/Sources/TetraRunLoopConcurrency/SlicedJobQueue.swift @@ -0,0 +1,730 @@ +// +// SlicedJobQueue.swift +// Tetra +// +// Created by 박병관 on 2/14/26. +// +import Atomics +import Darwin +import CriticalSection +import Dispatch +import Foundation +import HeapModule +import os +import Builtin + +internal struct SlicedJobQueue: ~Copyable, Sendable { + + nonisolated(unsafe) + let jobs:FiveArray<__MPSCQueue> + nonisolated(unsafe) + let boost:FiveArray> +// nonisolated(unsafe) + let delayedJobs: some UnfairStateLock>> = createCheckedStateLock(checkedState: ContiguousArray>.init(repeating: .init(), count: 3)) +// let lock = NSRecursiveLock() + + let cache1:__MPSCQueue.NodeCache +// let cache2:__MPSCQueue.NodeCache + + init(cacheSize:Int = 2048) { + let cache1 = __MPSCQueue.NodeCache(size: cacheSize) +// let cache2 = __MPSCQueue.NodeCache(size: cacheSize / 2) + self.cache1 = cache1 +// self.cache2 = cache2 + boost = .init(initializingWith: { + while !$0.isFull { + $0.append(.init(nil)) + } + }) + jobs = .init(initializingWith: { + while !$0.isFull { + $0.append(.init(cache: cache1)) + } + }) + } + + internal func runBatch( + executor:UnownedSerialExecutor, + taskRef: Builtin.Executor? = nil + ) { + + var currentJobs = ContiguousArray>.init(repeating: [], count: 5) + let qos:DispatchQoS + do { + var _qos = QOS_CLASS_UNSPECIFIED + var priority = Int32(0) + pthread_get_qos_class_np(pthread_self(), &_qos, &priority) + qos = .init(qosClass: .init(rawValue: _qos)!, relativePriority: .init(priority)) + } + var currentQos = qos.qosClass.rawValue + defer { + pthread_set_qos_class_self_np(qos.qosClass.rawValue, .init(qos.relativePriority)) + } + repeat { + + for i in 0..<5 { + while let t = self.jobs[i].dequeue() { + currentJobs[i].append(t) + } + var buffer = ContiguousArray() + buffer.reserveCapacity(currentJobs.capacity) + swap(&buffer, ¤tJobs[i]) + + do { + let qos = switch i { + case 0: + QOS_CLASS_USER_INTERACTIVE + case 1: + QOS_CLASS_USER_INITIATED + case 2: + QOS_CLASS_DEFAULT + case 3: + QOS_CLASS_UTILITY + case 4: + fallthrough + default: + QOS_CLASS_BACKGROUND + } + if qos != currentQos, !buffer.isEmpty { + pthread_set_qos_class_self_np(qos, 0) + currentQos = qos + } + } + do { + if let override = boost[i].exchange(nil, ordering: .relaxed) { + pthread_override_qos_class_end_np(.init(override)) + } + } + if #available(macOS 15.0, iOS 18.0, watchOS 11.0, tvOS 18.0, visionOS 2.0, *), let t = taskRef { + let taskExecutor = UnownedTaskExecutor(t) + for j in buffer { + j.runSynchronously(isolatedTo: executor, taskExecutor: taskExecutor) + } + } else { + for j in buffer { + j.runSynchronously(on: executor) + } + } + } + let times = [ + Dispatch.__dispatch_time(1 << 63,0) & ~(1 << 63), + Dispatch.__dispatch_time(0,0), + 0 &- Dispatch.__dispatch_walltime(nil,0) + ] + self.delayedJobs.withLockUnchecked { + for i in 0..<3 { + while let jobBox = $0[i].min, jobBox.timestamp.target <= times[i] { + $0[i].removeMin() + let index = if #available(macOS 14.0, iOS 17.0, watchOS 10.0, tvOS 17.0, *) { + TaskPriority(jobBox.job.priority)?.jobQueueIndex ?? 4 + } else { + 2 + } + currentJobs[index].append(jobBox.job) + } + } + } + } while !currentJobs.allSatisfy(\.isEmpty) + } + + nonisolated func enqueue(_ job:UnownedJob, _ thread:pthread_t) { + if #available(macOS 14.0, iOS 17.0, watchOS 10.0, tvOS 17.0, *){ + let priority = TaskPriority(job.priority) + let index = priority?.jobQueueIndex ?? 4 + jobs[index].enqueue(job) + if boost[index].load(ordering: .acquiring) != nil { + let qos = switch index { + case 0: + QOS_CLASS_USER_INTERACTIVE + case 1: + QOS_CLASS_USER_INITIATED + case 2: + QOS_CLASS_DEFAULT + case 3: + QOS_CLASS_UTILITY + case 4: + fallthrough + default: + QOS_CLASS_BACKGROUND + } + + let override = pthread_override_qos_class_start_np(thread, qos, 0) + let (exchanged, _) = boost[index].compareExchange(expected: nil, desired: .init(override), ordering: .releasing) + if !exchanged { + pthread_override_qos_class_end_np(override) + } + } + } else { + jobs[2].enqueue(job) + } + } + + @available(macOS 14.0, iOS 17.0, watchOS 10.0, tvOS 17.0, *) + nonisolated + internal func enqueue( + _ job:UnownedJob, + after delay: Swift.Duration, + tolerance: Swift.Duration? = nil, + index:ClockIndex + ) -> Bool { + let (delaySec, delayAtto) = delay.components + let dispatch_now:UInt64 + + switch index { + case .continuous: + let mask = 1 << 63 as dispatch_time_t + dispatch_now = mask + case .suspending: + dispatch_now = 0 + case .walltime: + dispatch_now = .init(DISPATCH_WALLTIME_NOW) + } + let dispatch_target = Dispatch.__dispatch_time( + dispatch_now, + delaySec * Int64(Dispatch.NSEC_PER_SEC) + Int64(delayAtto / 1_000_000_000) + ) + let dispatch_deadline:dispatch_time_t + if let tolerance { + let (tol_sec, tol_atto) = tolerance.components + dispatch_deadline = Dispatch.__dispatch_time( + dispatch_target, + Int64(Dispatch.NSEC_PER_SEC) * tol_sec + Int64(tol_atto / 1_000_000_000) + ) + } else { + dispatch_deadline = dispatch_target + } + let timestamp:Timestamp + switch index { + case .continuous: + timestamp = .init(target: dispatch_target & ~dispatch_now, leeway: (dispatch_deadline & ~dispatch_now) - (dispatch_target & ~dispatch_now)) + break + case .suspending: + timestamp = .init(target: dispatch_target, leeway: dispatch_deadline - dispatch_target) + break + case .walltime: + timestamp = .init(target: 0 &- dispatch_target, leeway: (0 &- dispatch_deadline) - (0 &- dispatch_target)) + break + } + let needsWakeup = delayedJobs.withLock { + let oldStamp = $0[index.rawValue].min?.timestamp + $0[index.rawValue].insert( + .init(job: job, timestamp: timestamp) + ) + let newStamp = $0[index.rawValue].min?.timestamp + return oldStamp != newStamp + } + return needsWakeup + } + + internal enum ClockIndex:Int, Sendable, BitwiseCopyable { + case continuous + case suspending + case walltime + } + + + deinit { +// let span = boost.span + for i in 0..<5 { + if let t = boost[i].exchange(nil, ordering: .relaxed) { + pthread_override_qos_class_end_np(.init(t)) + } + } + } + public struct __MPSCQueue: ~Copyable, @unchecked Sendable { + + typealias ForwardItem = ForwardItemT + + + @usableFromInline + internal struct BufferNode: ~Copyable { + @usableFromInline + internal var data: Element? + + @usableFromInline + internal let next: AtomicStore> = .init(nil) + + @inlinable + internal init(data: consuming Element?) { + self.data = data + } + } + + + @usableFromInline + internal struct Header:Copyable { + @usableFromInline + internal var capacity:Int + } + + @usableFromInline + internal let head: AtomicStore + + @usableFromInline + internal let tail: AtomicStore + + + @usableFromInline + internal let _cache:NodeCache + + @usableFromInline + final class NodeCache: Sendable { + + @usableFromInline + let pool:MPMCBoundedQueue + + deinit { + while let node:UnsafeMutablePointer = pool.dequeue()?.load() { + node.deinitialize(count: 1) + node.deallocate() + } + } + @usableFromInline + init(size:Int = 1024) { + pool = .init(size: size) + } + + + @preconcurrency + @usableFromInline + nonisolated + func consume(_ node: consuming sending UnsafeMutablePointer) { + if let dropped = pool.enqueue(.init(rawValue: node))?.load(BufferNode.self) { + dropped.deinitialize(count: 1) + dropped.deallocate() + } + } + @usableFromInline + func dequeue() -> UnsafeMutablePointer? { + let t = pool.dequeue()?.load(BufferNode.self) + t?.pointee.next.store(nil, ordering: .relaxed) + return t + } + } + + @inlinable + public init(cacheSize: Int = 1024) { + + let node = UnsafeMutablePointer.allocate(capacity: 1) + node.initialize(to: BufferNode(data: nil)) + self.head = .init(.init(rawValue: node)) + self.tail = .init(.init(rawValue: node)) + self._cache = .init(size: cacheSize) + } + + @inlinable + internal init(cache:NodeCache) { + + let node = UnsafeMutablePointer.allocate(capacity: 1) + node.initialize(to: BufferNode(data: nil)) + self.head = .init(.init(rawValue: node)) + self.tail = .init(.init(rawValue: node)) + self._cache = cache + } + + deinit { + while let _ = dequeue() {} + let last = tail.load(ordering: .relaxed).load(BufferNode.self) + last.deinitialize(count: 1) + last.deallocate() +// tail.load(ordering: .relaxed).deallocate() + } + + + + @inlinable + public func enqueue(_ value: consuming sending Element) { + let bufferNode = allocateNode() + bufferNode.pointee.data = consume value + let previous = tail.exchange(.init(rawValue: bufferNode), ordering: .acquiringAndReleasing).load(BufferNode.self) + previous.pointee.next.store(.init(rawValue: bufferNode), ordering: .releasing) + } + + @inlinable + public borrowing func dequeue() -> sending Element? { + let currentHead = head.load(ordering: .relaxed).load(BufferNode.self) + guard let next = currentHead.pointee.next.load(ordering: .acquiring)?.load(BufferNode.self) else { + return nil + } + var result:Element? = nil + swap(&result, &next.pointee.data) +// let result = next.pointee.data.take() + + head.store(.init(rawValue: next), ordering: .releasing) + do { + let t = Int(bitPattern: currentHead) + _cache.consume(.init(bitPattern: t)!) + } + return result + } + + @inline(__always) + public func withFirst(_ body: (borrowing Element?) throws(Failure) -> T) throws(Failure) -> T { + let currentHead = head.load(ordering: .relaxed).load(BufferNode.self) + guard let next = currentHead.pointee.next.load(ordering: .acquiring)?.load(BufferNode.self) else { + return try body(nil) + } + let first = next.pointee.data.take() + let result = try body(first) + next.pointee.data = first + return result + } + + @inline(__always) + public func dequeueAll(_ closure: (consuming sending Element) -> Void) { + while let element = dequeue() { + closure(element) + } + } + + @inlinable + internal func allocateNode() -> UnsafeMutablePointer { + if let node = _cache.dequeue() { + return node + } + let node: UnsafeMutablePointer = .allocate(capacity: 1) + node.initialize(to: BufferNode(data: nil)) + return node + } + } + + +} +struct TimestampJob: Comparable { + + static func < (lhs: Self, rhs: Self) -> Bool { + lhs.timestamp.deadline < rhs.timestamp.deadline + } + + static func > (lhs: Self, rhs: Self) -> Bool { + lhs.timestamp.deadline > rhs.timestamp.deadline + } + + static func == (lhs: Self, rhs: Self) -> Bool { + lhs.timestamp == rhs.timestamp + } + + let timestamp:Timestamp + let job:UnownedJob + + init(job:consuming UnownedJob, timestamp:Timestamp) { + self.job = job + self.timestamp = timestamp + } +} + +struct Timestamp:BitwiseCopyable, Hashable, Sendable, Copyable { + /// The earliest time at which a job should run. + /// + /// Jobs will never be run earlier than this. + var target: UInt64 + + /// The maximum (ideal) tolerable delay. + /// + /// We make no guarantee that we won't run over this, but it is taken + /// into consideration when scheduling jobs. + var leeway: UInt64 + + /// The latest time at which a job should (ideally) run. + /// + /// We may run the job after this point, but we will not run other jobs + /// with later deadlines before this job. + var deadline: UInt64 { + get { + if UInt64.max - target < leeway { + return UInt64.max + } + return target + leeway + } + set { + if newValue < leeway { + target = 0 + } else { + target = newValue - leeway + } + } + } +} + +extension TaskPriority { + + var jobQueueIndex:Int { + + if self > .high { + 0 + } else if self > .medium { + 1 + } else if self > .low { + 2 + } else if self > .background { + 3 + } else { + 4 + } + } + +} + +@inlinable +@inline(__always) +internal func getDrainIterations(queueIndex: Int) -> Int { + switch queueIndex { + case 0: .max // high + case 1: 128 // medium + case 2: 2 // low + default : 1 // background and lower + } +} + + + +class Backing { + + + + unowned(unsafe) var runLoop:RunLoop? = nil + unowned(unsafe) var source:CFRunLoopSource! = nil + var serialExecutor:UnownedSerialExecutor! = nil + let timers = [ + DispatchSource.makeTimerSource(), + DispatchSource.makeTimerSource(), + DispatchSource.makeTimerSource(), + ] as! [DispatchSource & DispatchSourceTimer] + let store = SlicedJobQueue() + let registry = NSMapTable.weakToStrongObjects() + + open var taskRef:Builtin.Executor? { nil } + + required init() { + + timers.forEach { + $0.setEventHandler { [unowned(unsafe) self] in + if let s = source { + CFRunLoopSourceSignal(s) + withExtendedLifetime(s) { + let valueTypes = Unmanaged.passUnretained(s).toOpaque().assumingMemoryBound(to: CFRunLoopSourceOpaqeueValue.self) + let bag:CFBag + do { + CFRunLoopSourceOpaqeueValue.lock(&valueTypes.pointee.mutex) + if let mutbag = valueTypes.pointee.mutableBag?.takeUnretainedValue() { + bag = CFBagCreateCopy(nil, mutbag) + } else { + + bag = withUnsafePointer(to: kCFTypeBagCallBacks) { + CFBagCreateMutable(nil, 0, $0) + } + } + CFRunLoopSourceOpaqeueValue.unlock(&valueTypes.pointee.mutex) + } + let runloopArrays = withUnsafeTemporaryAllocation(of: UnsafeRawPointer?.self, capacity: CFBagGetCount(bag)) { buffer in + CFBagGetValues(bag, buffer.baseAddress!) + return buffer.withMemoryRebound(to: CFRunLoop.self) { + Array($0) + } + } + + } + + } + if let rl = runLoop?.getCFRunLoop() { + CFRunLoopWakeUp(rl) + } + + } + } + timers.forEach{ $0.activate() } + } + + deinit { + timers.forEach { $0.cancel() } + } + + func dispatch() { + store.runBatch(executor: serialExecutor.unsafelyUnwrapped, taskRef: taskRef) + + let timeout = store.delayedJobs.withLock { + + $0.map(\.min?.timestamp) + } + + for i in timeout.indices { + if var t = timeout[i] { + let s = timers[i] + var start = t.target + if i == 0 { + start |= 1 << 63 + } + if i == 2 { + start = 0 &- start + } + Dispatch.__dispatch_source_set_timer( + s, + start, + DispatchTime.distantFuture.rawValue, + t.leeway + ) + } + } + + } + + func schedule(_ runloop:CFRunLoop, _ mode:CFRunLoopMode) { + registry.object(forKey: runloop)?.add(mode.rawValue) + } + + func cancel(_ runloop:CFRunLoop, _ mode:CFRunLoopMode) { + registry.object(forKey: runloop)?.remove(mode.rawValue) + } + + + class func create() -> CFRunLoopSource { + let ob = Self.init() + NSMapTable.weakToStrongObjects() +// malloc_zone_t + var context = CFRunLoopSourceContext() + context.info = Unmanaged.passUnretained(ob).toOpaque() + context.copyDescription = kCFTypeSetCallBacks.copyDescription + context.equal = kCFTypeSetCallBacks.equal + context.hash = kCFTypeSetCallBacks.hash + context.retain = unsafeBitCast(CFBundleGetFunctionPointerForName(CFBundleGetBundleWithIdentifier("com.apple.CoreFoundation" as CFString), "CFRetain" as CFString), to: (@convention(c) (UnsafeRawPointer?) -> UnsafeRawPointer?).self) + context.release = unsafeBitCast(CFBundleGetFunctionPointerForName(CFBundleGetBundleWithIdentifier("com.apple.CoreFoundation" as CFString), "CFRelease" as CFString), to: (@convention(c) (UnsafeRawPointer?) -> Void).self) + context.perform = { + Unmanaged.fromOpaque($0.unsafelyUnwrapped).takeUnretainedValue().dispatch() + } + context.schedule = { + Unmanaged.fromOpaque($0.unsafelyUnwrapped).takeUnretainedValue() + .schedule($1.unsafelyUnwrapped, $2.unsafelyUnwrapped) + } + context.cancel = { + Unmanaged.fromOpaque($0.unsafelyUnwrapped).takeUnretainedValue() + .cancel($1.unsafelyUnwrapped, $2.unsafelyUnwrapped) + } + let source = CFRunLoopSourceCreate(nil, 0, &context)! + ob.source = source + let bag:CFBag + do { +// CFRunLoopAddSource(CFRunLoopGetMain(), source, .commonModes) +// +// CFRunLoopRemoveSource(CFRunLoopGetMain(), source, .commonModes) + let k2 = Unmanaged.passUnretained(source).toOpaque().assumingMemoryBound(to: CFRunLoopSourceOpaqeueValue.self) + CFRunLoopSourceOpaqeueValue.lock(&k2.pointee.mutex) + if let mutBag = k2.pointee.mutableBag?.takeUnretainedValue() { + bag = CFBagCreateCopy(nil, mutBag) + } else { + bag = withUnsafePointer(to: kCFTypeBagCallBacks) { + CFBagCreate(nil, nil, 0, $0) + } + } + CFRunLoopSourceOpaqeueValue.unlock(&k2.pointee.mutex) + + } + //AutoreleasingUnsafeMutablePointer + let t = withUnsafeTemporaryAllocation(of: UnsafeRawPointer?.self, capacity: CFBagGetCount(bag)) { + CFBagGetValues(bag, $0.baseAddress!) + let count = CFBagGetCount(bag) + let buffer = UnsafeMutableBufferPointer(rebasing: $0[...init(unsafeUninitializedCapacity: buffer.count) { + for i in buffer.indices { + $0.initializeElement(at: i, to: Unmanaged.fromOpaque(buffer[i]!).takeUnretainedValue()) + } + $1 = buffer.count + } + +// return withUnsafePointer(to: kCFTypeArrayCallBacks) { CFArrayCreate(nil, buffer.baseAddress!, CFBagGetCount(bag), $0) +// +// } as! [CFRunLoop] +// return buffer.withMemoryRebound(to: CFRunLoop.self) { +// Array($0) +// } + } + print(t) + return source + } +} + +struct CFRunLoopSourceOpaqeueValue:BitwiseCopyable { + var runtime:(UnsafeMutableRawPointer?, UnsafeMutableRawPointer?) + + var mutex:RecursiveMutex + var order:CFIndex + var signalTime:UInt64 + var mutableBag:Unmanaged! + + var context0:CFRunLoopSourceContext + + #if false && canImport(Darwin) + struct RecursiveMutex:BitwiseCopyable { + var lock:os_unfair_lock + var count:UInt32 + } + @_silgen_name("os_unfair_recursive_lock_lock_with_options") + static func lock_options(_ lock: inout RecursiveMutex, options:UInt32) + + static func lock(_ lock: inout RecursiveMutex) { + lock_options(&lock, options: 0) + } + @_silgen_name("os_unfair_recursive_lock_unlock") + static func unlock(_ lock: inout RecursiveMutex) + + #elseif canImport(WinSDK) + typealias RecursiveMutex = SWIFT_CRITICAL_SECTION + static func lock(_ lock: inout RecursiveMutex) { + EnterCriticalSection(&lock) + } + static func unlock(_ lock: inout RecursiveMutex) { + LeaveCriticalSection(&lock) + } + #elseif canImport(pthread) + + typealias RecursiveMutex = pthread_mutex_t + static func lock(_ lock: inout RecursiveMutex) { + pthread_mutex_lock(&lock) + } + static func unlock(_ lock: inout RecursiveMutex) { + pthread_mutex_unlock(&lock) + } + #else + #error("RecursiveMutex can not be inferred") + + #endif + + +} + +@available(macOS 15.0, iOS 18.0, watchOS 11.0, tvOS 18.0, visionOS 2.0, *) +class Backing2: Backing { + + + var taskExecutor:UnownedTaskExecutor! + + override var taskRef: Builtin.Executor { + taskExecutor!._executor + } + + +} + + +@usableFromInline +struct ForwardItemT: RawRepresentable, AtomicValue, AtomicOptionalWrappable { + + @usableFromInline + @inline(__always) + var rawValue: UnsafeMutableRawPointer + + @usableFromInline + @_transparent + @inline(__always) + func load(_ type:T.Type = T.self) -> UnsafeMutablePointer { + rawValue.assumingMemoryBound(to: type) + } + + @_transparent + @inline(__always) + @usableFromInline + init(rawValue: UnsafeMutableRawPointer) { + self.rawValue = rawValue + } + +} diff --git a/Sources/TetraRunLoopConcurrency/TetraRunLoopExecutor.swift b/Sources/TetraRunLoopConcurrency/TetraRunLoopExecutor.swift deleted file mode 100644 index fddd3d3..0000000 --- a/Sources/TetraRunLoopConcurrency/TetraRunLoopExecutor.swift +++ /dev/null @@ -1,209 +0,0 @@ -// -// File.swift -// Tetra -// -// Created by 박병관 on 1/27/25. -// - -import Foundation -@preconcurrency import CoreFoundation -@_implementationOnly private import TetraConcurrentQueueShim -//private import CriticalSection -//private import Atomics - -struct StateStorage { - -// fileprivate let lock: some UnfairLockProtocol = createUnfairLock() -// var registry = [CFRunLoop: Set]() - var serialRef: UnownedSerialExecutor -// fileprivate let reference = ManagedAtomicLazyReference() - - var taskRef:AnyObject? = nil - -} - - -fileprivate actor DummyActor { - let unownedExecutor: UnownedSerialExecutor - init(unownedExecutor: UnownedSerialExecutor) { - self.unownedExecutor = unownedExecutor - } - - func run(_ block: @convention(block) () -> Void) { block() } - -} - -final class RunLoopStorageBufferHolder {} - -final package class TetraRunLoopExecutor: NSObject { - - - let source: CFRunLoopSource - - deinit { - CFRunLoopSourceInvalidate(source) -// runLoop.perform {} - - } - - @objc - package override convenience init() { - self.init(name: nil) - } - - @nonobjc - package init( - name: String? - ) { - - - var bufferPtr = ManagedBufferPointer(bufferClass: RunLoopStorageBufferHolder.self, minimumCapacity: 0) { buffer, capacity in - .init(serialRef: MainActor.sharedUnownedExecutor) - } - - self.source = withUnsafePointer(to: TetraContextData( - perform: tetra_runLoop_drainSource, - schedule: tetra_runLoop_schedule_cb, - cancel: tetra_runLoop_cancel_cb - )) { - create_tetra_runLoop_executor(bufferPtr.buffer, $0) - } - super.init() - // override serialExecutor to the correct value - if #available(macOS 15.0, iOS 18.0, watchOS 11.0, tvOS 18.0, visionOS 2.0, *) { - bufferPtr.header.taskRef = ManagedBufferPointer(bufferClass: RunLoopStorageBufferHolder.self, minimumCapacity: 0, makingHeaderWith: { buffer, capacity in - return asUnownedTaskExecutor() - }).buffer - } - bufferPtr.header.serialRef = asUnownedSerialExecutor() - let thread = Thread(block: runLoopThreadRun) - thread.threadDictionary["source"] = self.source - if let name = name { - thread.name = name - } - thread.qualityOfService = .default - thread.start() - - } - -} - - - - -extension TetraRunLoopExecutor : SerialExecutor {} - -extension TetraRunLoopExecutor: TaskExecutor {} - -package extension TetraRunLoopExecutor { - - @objc - nonisolated func copyCurrentRegistry() -> [CFRunLoop:Set] { - return copy_tetra_runLoop_registry(source) as! [CFRunLoop:Set] - } - - @available(macOS 14.0, iOS 17.0, watchOS 10.0, tvOS 17.0, *) - nonisolated func enqueue(_ job: consuming ExecutorJob) { - enqueue(UnownedJob(job)) - } - - nonisolated func enqueue(_ job: UnownedJob) { - tetra_enqueue_and_signal(source, job as AnyObject) - let _ = ManagedBufferPointer(unsafeBufferObject: tetra_get_stateInfo(source)) - } - - nonisolated func asUnownedSerialExecutor() -> UnownedSerialExecutor { - if #available(macOS 14.0, iOS 17.0, watchOS 10.0, tvOS 17.0, *) { - return .init(complexEquality: self) - } - return .init(ordinary: self) - } - - nonisolated func checkIsolated() { -// let currentMode = RunLoop.current.currentMode.flatMap{ $0.rawValue -// as CFString }.flatMap{ CFRunLoopMode($0)} - precondition( -// currentMode != nil && - CFRunLoopContainsSource(CFRunLoopGetCurrent(), source, CFRunLoopMode.defaultMode), - "TetraRunLoopExecutor must be called from the RunLoop that it was created on." - ) - } - - func isSameExclusiveExecutionContext(other: TetraRunLoopExecutor) -> Bool { - CFEqual(source, other.source) - } - -} - -extension TetraRunLoopExecutor { - - nonisolated package func register(_ runLoop: CFRunLoop) { - CFRunLoopAddSource(runLoop, source, .defaultMode) - } - - nonisolated package func register(_ runLoop:RunLoop) { - CFRunLoopAddSource(runLoop.getCFRunLoop(), source, .defaultMode) - } - -// @available(swift, obsoleted: 1.0) -// @objc -// package func schedule( _ block: @convention(block) () -> Void) async { -// await DummyActor(unownedExecutor: asUnownedSerialExecutor()) -// .run(block) -// } - -} - - -fileprivate nonisolated func runLoopThreadRun() { - let source = Thread.current.threadDictionary["source"] as! CFRunLoopSource - Thread.current.threadDictionary["source"] = nil - CFRunLoopAddSource(CFRunLoopGetCurrent(), source, CFRunLoopMode.defaultMode) - while CFRunLoopSourceIsValid(source), RunLoop.current.run(mode: .default, before: .distantFuture) { - - } - -} - -private func tetra_runLoop_drainSource(_ stateRef:CFTypeRef, _ jobRef:CFArray) { - let buffPtr = ManagedBufferPointer(unsafeBufferObject: stateRef) - let serials = buffPtr.header.serialRef - let jobArray = jobRef as! [UnownedJob] - if #available(macOS 15.0, iOS 18.0, watchOS 11.0, tvOS 18.0, visionOS 2.0, *), let taskRef = buffPtr.header.taskRef { - let taskExecutor = ManagedBufferPointer(unsafeBufferObject: taskRef).header - jobArray.forEach{ - $0.runSynchronously(isolatedTo: serials, taskExecutor: taskExecutor) - } - } else { - jobArray.forEach{ - $0.runSynchronously(on: serials) - } - } -} - -private func tetra_runLoop_schedule_cb(_ state:CFTypeRef, _ runLoop:CFRunLoop, _ mode:CFRunLoopMode) { -// let buffPtr = ManagedBufferPointer(unsafeBufferObject: state) -// let lock = buffPtr.header.lock -// let _ = buffPtr.header.reference.storeIfNilThenLoad(runLoop) -// lock.withLockUnchecked { -// buffPtr.withUnsafeMutablePointerToHeader{ -// if ($0.pointee.registry[runLoop] == nil) { -// $0.pointee.registry[runLoop] = [mode.rawValue as String] -// } else { -// $0.pointee.registry[runLoop]?.insert(mode.rawValue as String) -// } -// } -// } -} - -private func tetra_runLoop_cancel_cb(_ state:CFTypeRef, _ runLoop:CFRunLoop, _ mode:CFRunLoopMode) { -// let buffPtr = ManagedBufferPointer(unsafeBufferObject: state) -// let lock = buffPtr.header.lock -// lock.withLockUnchecked { -// buffPtr.withUnsafeMutablePointerToHeader{ -// if ($0.pointee.registry[runLoop] != nil) { -// $0.pointee.registry[runLoop]?.remove(mode.rawValue as String) -// } -// } -// } -} diff --git a/Sources/TetraRunLoopConcurrency/_MPSCQueue.swift b/Sources/TetraRunLoopConcurrency/_MPSCQueue.swift new file mode 100644 index 0000000..c87d4e3 --- /dev/null +++ b/Sources/TetraRunLoopConcurrency/_MPSCQueue.swift @@ -0,0 +1,162 @@ +// +// BufferNode.swift +// Tetra +// +// Created by 박병관 on 2/12/26. +// + +import CriticalSection +import Atomics +//import Synchronization + + + +internal struct _MPSCQueue: ~Copyable, @unchecked Sendable { + + + @usableFromInline + struct ForwardItem: RawRepresentable, AtomicValue, AtomicOptionalWrappable { + @usableFromInline + var rawValue: UnsafeMutableRawPointer + @usableFromInline + var pointer:UnsafeMutablePointer { + _read { + yield rawValue.assumingMemoryBound(to: BufferNode.self) + } + set { + rawValue = .init(newValue) + } + @storageRestrictions(initializes: rawValue) + init(newValue) { + rawValue = .init(newValue) + } + } + + @usableFromInline + init(rawValue: UnsafeMutableRawPointer) { + self.rawValue = rawValue + } + @usableFromInline + init(_ pointer:UnsafeMutablePointer) { + self.pointer = pointer + } + } + + @usableFromInline + internal struct BufferNode: ~Copyable { + @usableFromInline + internal var data: Element? + + @usableFromInline + internal let next: AtomicStore = .init(nil) + + @inlinable + internal init(data: consuming Element?) { + self.data = data + } + } + + + @usableFromInline + internal struct Header:Copyable { + @usableFromInline + internal var capacity:Int + } + + @usableFromInline + internal let head: AtomicStore + + @usableFromInline + internal let tail: AtomicStore + + @usableFromInline + internal let cache: _SPMCBoundedQueue + + @inlinable + public var wasFull: Bool { false } + + @inlinable + public init(cacheSize: Int = 1024) { + + let node = UnsafeMutablePointer.allocate(capacity: 1) + node.initialize(to: BufferNode(data: nil)) + self.head = AtomicStore(.init(node)) + self.tail = AtomicStore(.init(node)) + self.cache = _SPMCBoundedQueue(size: cacheSize) + } + + deinit { + while let _ = dequeue() {} + while let node = cache.dequeue() { + node.pointer.deinitialize(count: 1) + node.pointer.deallocate() + } + tail.load(ordering: .relaxed).pointer.deinitialize(count: 1) + tail.load(ordering: .relaxed).pointer.deallocate() + } + + @inlinable + public func flushCache() { + while let node = cache.dequeue() { + node.pointer.deinitialize(count: 1) + node.pointer.deallocate() + } + } + + @inlinable + public func enqueue(_ value: consuming sending Element) -> sending Element? { + let bufferNode = allocateNode() + bufferNode.pointee.data = consume value + let previous = tail.exchange(.init(bufferNode), ordering: .acquiringAndReleasing).pointer + previous.pointee.next.store(.init(bufferNode), ordering: .releasing) + return nil + } + + @inlinable + public func dequeue() -> sending Element? { + let currentHead = head.load(ordering: .relaxed) + guard let next = currentHead.pointer.pointee.next.load(ordering: .acquiring)?.pointer else { + return nil + } + let result = next.pointee.data.take() + + head.store(.init(next), ordering: .releasing) + let dummy = Int(bitPattern: currentHead.rawValue) + if let dropped = cache.enqueue(.init(rawValue: .init(bitPattern: dummy)!))?.pointer { + dropped.deinitialize(count: 1) + dropped.deallocate() + } + return result + } + + @inline(__always) + public func withFirst(_ body: (borrowing Element?) throws(Failure) -> T) throws(Failure) -> T { + let currentHead = head.load(ordering: .relaxed).pointer + guard let next = currentHead.pointee.next.load(ordering: .acquiring)?.pointer else { + return try body(nil) + } + let first = next.pointee.data.take() + let result = try body(first) + next.pointee.data = first + return result + } + + @inline(__always) + public func dequeueAll(_ closure: (consuming sending Element) -> Void) { + while let element = dequeue() { + closure(element) + } + } + + @inlinable + internal func allocateNode() -> UnsafeMutablePointer { + if let node = cache.dequeue()?.pointer { + node.pointee.next.store(nil, ordering: .relaxed) + return node + } + let node: UnsafeMutablePointer = .allocate(capacity: 1) + node.initialize(to: BufferNode(data: nil)) + return node + } +} + diff --git a/Sources/TetraRunLoopConcurrency/_SPMCBoundedQueue.swift b/Sources/TetraRunLoopConcurrency/_SPMCBoundedQueue.swift new file mode 100644 index 0000000..84e81fa --- /dev/null +++ b/Sources/TetraRunLoopConcurrency/_SPMCBoundedQueue.swift @@ -0,0 +1,159 @@ +// +// _SPMCBoundedQueue.swift +// Tetra +// +// Created by 박병관 on 2/12/26. +// +import Atomics +import CriticalSection + + +internal struct _SPMCBoundedQueue: ~Copyable, @unchecked Sendable { + + @usableFromInline + internal struct Header { + + @usableFromInline + var capacity:Int + + + @usableFromInline + var tail = 0 + + } + + @usableFromInline + internal struct BufferNode: ~Copyable { + @usableFromInline + internal var data: Element? + + @usableFromInline + internal let sequence: AtomicStore = .init(0) + + @inlinable + init(data: consuming Element) { + self.data = consume data + } + + init() { + self.data = nil + } + } + + @usableFromInline + internal let mask: Int + + @usableFromInline + internal let _buffer: ManagedBuffer + + @usableFromInline + internal let head: AtomicStore = .init(0) + + public var count: Int { + let headIndex = head.load(ordering: .relaxed) + let tailIndex = _buffer.header.tail + return tailIndex < headIndex ? (_buffer.header.capacity - headIndex + tailIndex) : (tailIndex - headIndex) + } + + public var wasFull: Bool { + _buffer.header.capacity - count == 1 + } + + public init(size: Int) { + let size = size.nextPowerOf2() + self.mask = size - 1 + self._buffer = .create(minimumCapacity: size, makingHeaderWith: { ref in + .init(capacity: size, tail: 0) + }) + _buffer.withUnsafeMutablePointers { headPtr, buffPtr in + let buffer = UnsafeMutableBufferPointer(start: buffPtr, count: headPtr.pointee.capacity) + for i in 0.. sending Element? { + let pos = _buffer.header.tail + var result:Element? = consume value + _buffer.withUnsafeMutablePointers { headPtr, buffPtr in + let buffer = UnsafeMutableBufferPointer(start: buffPtr, count: headPtr.pointee.capacity) + let node: UnsafeMutablePointer = buffer.baseAddress!.advanced(by: pos & mask) + let seq = node.pointee.sequence.load(ordering: .acquiring) + let difference = seq - pos + + if difference == 0 { + headPtr.pointee.tail += 1 + } else if difference < 0 { + return + } + node.pointee.data = consume result + result = nil + node.pointee.sequence.store(pos + 1, ordering: .releasing) + return + } + return result + } + + @inlinable + public func dequeue() -> sending Element? { + return _buffer.withUnsafeMutablePointers { headPtr, buffPtr in + let buffer = UnsafeMutableBufferPointer(start: buffPtr, count: headPtr.pointee.capacity) + var node: UnsafeMutablePointer! + var pos = head.load(ordering: .relaxed) + + while true { + + node = buffer.baseAddress!.advanced(by: pos & mask) + let seq = node.pointee.sequence.load(ordering: .acquiring) + let difference = seq - (pos + 1) + + if difference == 0 { + if head.weakCompareExchange(expected: pos, desired: pos + 1, successOrdering: .relaxed, failureOrdering: .relaxed).exchanged { + break + } + } else if difference < 0 { + return nil + } else { + pos = head.load(ordering: .relaxed) + } + } + var result:Element? = nil + swap(&result, &node.pointee.data) + node.pointee.sequence.store(pos + mask + 1, ordering: .releasing) + return result + } + + } + + @inline(__always) + public func dequeueAll(_ closure: (consuming sending Element) -> Void) { + while let element = dequeue() { + closure(element) + } + } + +} +extension FixedWidthInteger { + /// Returns the next power of two. + @inlinable + @_transparent + func nextPowerOf2() -> Self { + guard self != 0 else { + return 1 + } + return 1 << (Self.bitWidth - (self - 1).leadingZeroBitCount) + } +} From 3cd5f9921a3002c387cec2c1a9898f394e092ef6 Mon Sep 17 00:00:00 2001 From: pbk Date: Mon, 6 Jul 2026 09:50:09 +0900 Subject: [PATCH 62/63] Add TetraRunLoopConcurrency kqueue run-loop executor (iOS 13) - StackBoundRunLoopExecutor (SerialExecutor facade) + KQScheduler engine over the ported SlicedJobQueue, wrapping a kqueue in a CFFileDescriptor run-loop source. Facade owns the run frame; stack-bound engine lifetime, pendingJobPop wake elision, producerGate quiescence. - KQueueSelector (EVFILT_USER wake + 3-clock-domain EVFILT_TIMER) and SerialExecutorRef (current-executor peek via swift_task_getCurrentExecutor). - Timer path: owning-thread ThreeArray armed set with arm-if-earlier; QoS-once boost release; per-lane drain fairness cap in runBatch. - Gate SchedulingExecutor/RunLoopExecutor/MainExecutor SPI conformances behind the opt-in `SchedulingExecutorSPI` package trait (off by default so release toolchains build; TaskExecutor stays unconditional). - Fix ThreeArray init precondition (5 -> 3) and SlicedJobQueue QoS-override install condition (was inverted). - Tests: 14 (executor behavior, kqueue selector, engine lifetime/dealloc). Also carries in-progress swift6 branch changes (CoreData/Combine/SwiftUI). Co-Authored-By: Claude Opus 4.8 (1M context) --- .../xcshareddata/xcschemes/Tetra.xcscheme | 9 + Package.resolved | 10 +- Package.swift | 26 +- Sources/CriticalSection/BackportedCell.swift | 6 +- .../Combine/Publishers+AsyncFlatMap.swift | 4 +- .../CoreDataStack+Concurrency.swift | 6 +- .../Tetra/SwiftUI/RefreshableScrollView.swift | 6 +- .../TetraRunLoopConcurrency/KQScheduler.swift | 381 +++++++++++++ .../KQueueSelector.swift | 207 +++++++ .../SerialExecutorRef.swift | 54 ++ .../SlicedJobQueue.swift | 229 +++----- .../StackBoundRunLoopExecutor.swift | 515 ++++++++++++++++++ .../KQueueSelectorTests.swift | 20 + ...ackBoundRunLoopExecutorLifetimeTests.swift | 93 ++++ .../StackBoundRunLoopExecutorTests.swift | 260 +++++++++ .../TestSupport.swift | 15 + 16 files changed, 1678 insertions(+), 163 deletions(-) create mode 100644 Sources/TetraRunLoopConcurrency/KQScheduler.swift create mode 100644 Sources/TetraRunLoopConcurrency/KQueueSelector.swift create mode 100644 Sources/TetraRunLoopConcurrency/SerialExecutorRef.swift create mode 100644 Sources/TetraRunLoopConcurrency/StackBoundRunLoopExecutor.swift create mode 100644 Tests/TetraRunLoopConcurrencyTests/KQueueSelectorTests.swift create mode 100644 Tests/TetraRunLoopConcurrencyTests/StackBoundRunLoopExecutorLifetimeTests.swift create mode 100644 Tests/TetraRunLoopConcurrencyTests/StackBoundRunLoopExecutorTests.swift create mode 100644 Tests/TetraRunLoopConcurrencyTests/TestSupport.swift diff --git a/.swiftpm/xcode/xcshareddata/xcschemes/Tetra.xcscheme b/.swiftpm/xcode/xcshareddata/xcschemes/Tetra.xcscheme index 2302888..41a9216 100644 --- a/.swiftpm/xcode/xcshareddata/xcschemes/Tetra.xcscheme +++ b/.swiftpm/xcode/xcshareddata/xcschemes/Tetra.xcscheme @@ -38,6 +38,15 @@ ReferencedContainer = "container:"> + + + + =x)` + // can tell the two apart. NOT in the default set → OFF by default, so release/standard + // toolchains build cleanly. Enable with `swift build --traits SchedulingExecutorSPI` on + // a toolchain that exposes the SPI; an enabled trait becomes the `#if SchedulingExecutorSPI` + // compilation condition. + traits: [ + .trait( + name: "SchedulingExecutorSPI", + description: "Conform StackBoundRunLoopExecutor to the SchedulingExecutor/RunLoopExecutor/MainExecutor _Concurrency SPI protocols (only compilable where the stdlib exposes them)." + ), + ], dependencies: [ // Dependencies declare other packages that this package depends on. // .package(url: /* package url */, from: "1.0.0"), @@ -36,7 +50,7 @@ let package = Package( .upToNextMajor(from: "1.3.0"), traits: [.defaults], ), - .package(url: "https://github.com/apple/swift-async-algorithms", from: "1.1.1"), + .package(url: "https://github.com/apple/swift-async-algorithms", from: "1.1.5"), ], targets: [ @@ -134,6 +148,14 @@ let package = Package( swiftSettings: [ .swiftLanguageMode(.v5) ] - ) + ), + .testTarget( + name: "TetraRunLoopConcurrencyTests", + dependencies: ["TetraRunLoopConcurrency"], + swiftSettings: [ + .swiftLanguageMode(.v6), + .unsafeFlags(["-Xfrontend", "-disable-availability-checking"]), + ] + ), ], ) diff --git a/Sources/CriticalSection/BackportedCell.swift b/Sources/CriticalSection/BackportedCell.swift index accc74d..1ef6649 100644 --- a/Sources/CriticalSection/BackportedCell.swift +++ b/Sources/CriticalSection/BackportedCell.swift @@ -69,13 +69,13 @@ package struct ThreeArray:~Copyable { package var _address: UnsafeMutableBufferPointer { .init(start: .init(_rawAddress), count: 3) } - + public init(initializingWith initializer: (inout OutputSpan) throws(E) -> Void) throws(E) where E : Error { var span = unsafe OutputSpan(buffer: _address, initializedCount: 0) try initializer(&span) let count = span.finalize(for: _address) - - precondition(5 == count) + + precondition(3 == count) } deinit { diff --git a/Sources/Tetra/Combine/Publishers+AsyncFlatMap.swift b/Sources/Tetra/Combine/Publishers+AsyncFlatMap.swift index 3ce24f9..39a9af7 100644 --- a/Sources/Tetra/Combine/Publishers+AsyncFlatMap.swift +++ b/Sources/Tetra/Combine/Publishers+AsyncFlatMap.swift @@ -15,7 +15,7 @@ struct AsyncFlatMap: Publisher where typealias Output = Segment.Element typealias Failure = Upstream.Failure - typealias Transform = @Sendable (Upstream.Output) async throws(Failure) -> Segment + typealias Transform = @Sendable @isolated(any) (Upstream.Output) async throws(Failure) -> sending Segment var priority: TaskPriority? = nil let taskExecutor: (any Executor)? var maxTasks:Subscribers.Demand @@ -89,7 +89,7 @@ extension AsyncFlatMap { struct Inner: Subscriber, Sendable, Subscription, CustomStringConvertible, CustomPlaygroundDisplayConvertible where Segment.Element == Down.Input, Down.Failure == AsyncFlatMap.Failure { - typealias Transformer = @Sendable (Upstream.Output) async throws(Failure) -> Segment + typealias Transformer = @isolated(any) @Sendable (Upstream.Output) async throws(Failure) -> sending Segment typealias Input = Upstream.Output typealias Failure = Upstream.Failure diff --git a/Sources/Tetra/Concurrency/CoreDataStack+Concurrency.swift b/Sources/Tetra/Concurrency/CoreDataStack+Concurrency.swift index ee37570..f621ec8 100644 --- a/Sources/Tetra/Concurrency/CoreDataStack+Concurrency.swift +++ b/Sources/Tetra/Concurrency/CoreDataStack+Concurrency.swift @@ -41,7 +41,8 @@ extension TetraExtension where Base: NSPersistentStoreCoordinator { withExtendedLifetime(block, {}) } return await base.perform{ [unowned block] in - block() + let result:Result = block() + return result } }.get() } else { @@ -209,7 +210,8 @@ extension TetraExtension where Base: NSPersistentContainer { let block = CoreDataContextClosureHolder(closure: $0) defer { withExtendedLifetime(block, {}) } return await base.performBackgroundTask{ [unowned block] in - block($0) + let result:Result = block($0) + return result } }.get() } else { diff --git a/Sources/Tetra/SwiftUI/RefreshableScrollView.swift b/Sources/Tetra/SwiftUI/RefreshableScrollView.swift index ccda360..52a432d 100644 --- a/Sources/Tetra/SwiftUI/RefreshableScrollView.swift +++ b/Sources/Tetra/SwiftUI/RefreshableScrollView.swift @@ -18,10 +18,10 @@ import UIKit @available(macOS, deprecated: 13, renamed: "ScrollView") @available(watchOS, deprecated: 9, renamed: "ScrollView") public struct RefreshableScrollView: View { - +#if os(iOS) || targetEnvironment(macCatalyst) @State private var task:Task? = nil @State private var flag = false - +#endif public var content:Content public var axes: Axis.Set = .vertical public var showsIndicators: Bool = true @@ -42,9 +42,11 @@ public struct RefreshableScrollView: View { } .onDisappear{ +#if os(iOS) || targetEnvironment(macCatalyst) task?.cancel() task = nil flag = false +#endif } } diff --git a/Sources/TetraRunLoopConcurrency/KQScheduler.swift b/Sources/TetraRunLoopConcurrency/KQScheduler.swift new file mode 100644 index 0000000..8384427 --- /dev/null +++ b/Sources/TetraRunLoopConcurrency/KQScheduler.swift @@ -0,0 +1,381 @@ +// +// KQScheduler.swift +// TetraRunLoopConcurrency +// +// The kqueue run-loop ENGINE. Ported from swift-platform-executors' +// `KQScheduler2` (the engine half of `StackBoundRunLoopExecutor2`), adapted to +// Tetra's primitives: +// +// * `AtomicStore` (not Synchronization `Atomic`) for the phase / wake / stop flags. +// * `SlicedJobQueue` (ready lanes only) for the 5-QoS ready queues. +// * The timer state lives HERE (DESIGN A): three `Heap` domains +// behind one `createCheckedStateLock`, replacing the reference's `MinMaxHeap` +// `TimerState`. The queue no longer owns any timer state. +// +// The engine wraps the kqueue in a CFFileDescriptor run-loop source, drains the +// ready lanes in `handleReadable`, fires due timers into the ready lanes, and arms +// the earliest pending deadline per domain via `EVFILT_TIMER`. +// + +#if canImport(Darwin) +import Darwin +import Dispatch +import Foundation +import CoreFoundation +import CriticalSection +import HeapModule +import Builtin + +@available(macOS 10.15, iOS 13, *) +final class KQScheduler: @unchecked Sendable { + + /// Lifecycle phase. Stored as a raw `UInt8` in an `AtomicStore` (Tetra's atomic + /// primitive), since `AtomicStore` requires `Value: AtomicValue`. + enum Phase: UInt8 { case live = 0, closing = 1, dead = 2 } + + // MARK: Stored state + + private let facade: StackBoundRunLoopExecutor + private let serial: UnownedSerialExecutor + /// Optional task-executor reference threaded through to `runBatch`. + private let taskRefOrNil: Builtin.Executor? + let cfRunLoop: CFRunLoop + let thread: pthread_t + /// Raw kqueue descriptor, valid from init; exposed for cross-thread wakeups. + let kqueueFD: Int32 + + /// The 5-QoS ready lanes. Drained (only) by `runBatch` on the owning thread. + let ready = SlicedJobQueue(cacheSize: 2048) + + /// Timer state, MOVED here from `SlicedJobQueue` (DESIGN A). One min-heap per + /// clock domain — continuous / suspending / wall — behind a single state lock. + let delayedJobs: some UnfairStateLock>> = + createCheckedStateLock(checkedState: .init(repeating: .init(), count: 3)) + + /// The deadline currently armed on the kqueue per clock domain (`nil` = unarmed). + /// OWNING-THREAD-ONLY — mutated only by the pump (`handleReadable` and its + /// `fireDueTimers`/`armNextDeadlines`), never by producers — so it needs no lock + /// (and can therefore be the noncopyable `ThreeArray`, which `UnfairStateLock`'s + /// copyable `State` could not hold). Arming is deferred from producers to the pump; + /// `armNextDeadlines` re-arms a domain only when its heap min is strictly earlier + /// than what is armed, and one-shot fires clear the entry — eliminating the + /// per-pump re-arm `kevent64` thrash of the previous unconditional arming. + private nonisolated(unsafe) var armed = ThreeArray(initializingWith: { + while !$0.isFull { $0.append(nil) } + }) + + /// True while a drain is pending/imminent; producers elide the wakeup when they + /// lose the false→true race. RMW-only (see `handleReadable`). + private let pendingJobPop = AtomicStore(false) + private let stopFlag = AtomicStore(false) + private let phaseStorage = AtomicStore(Phase.live.rawValue) + + nonisolated(unsafe) var deferredReadable = false + /// True once the CFFileDescriptor exists: from then it owns the kqueue fd + /// (closeOnInvalidate) and the frame invalidates it, so `deinit` must not close. + private nonisolated(unsafe) var didCreateFileDescriptor = false + + // MARK: Init / deinit + + /// - Parameter taskRef: the facade's task-executor reference (its + /// `asUnownedTaskExecutor()._executor`) on the iOS-18 `TaskExecutor` path, or + /// `nil` on the iOS-13 `SerialExecutor`-only path. When non-nil, `runBatch` + /// runs jobs via `runSynchronously(isolatedTo:taskExecutor:)` so a task's + /// preferred task executor is respected (Task 5). + init(facade: StackBoundRunLoopExecutor, serial: UnownedSerialExecutor, + taskRef: Builtin.Executor?, cfRunLoop: CFRunLoop) { + self.facade = facade + self.serial = serial + self.taskRefOrNil = taskRef + self.cfRunLoop = cfRunLoop + self.thread = pthread_self() + self.kqueueFD = KQueueSelector.makeKQueue() + } + + deinit { + // Only the never-mounted path (no descriptor ever created) closes the fd; + // otherwise the CFFileDescriptor closes it on invalidation. + if !didCreateFileDescriptor { + close(kqueueFD) + } + } + + // MARK: Phase helpers + + private var phase: Phase { Phase(rawValue: phaseStorage.load(ordering: .acquiring))! } + + var isStopRequested: Bool { stopFlag.load(ordering: .acquiring) } + + // MARK: Enqueue (producer side — the facade holds `producerGate` across this) + + func enqueueReady(_ job: UnownedJob) { + let phase = self.phase + if phase == .dead { + preconditionFailure("StackBoundRunLoopExecutor.enqueue(_:) called after its run loop finished.") + } + if phase == .closing, pthread_equal(thread, pthread_self()) == 0 { + preconditionFailure("StackBoundRunLoopExecutor.enqueue(_:) raced with its run loop shutting down.") + } + ready.enqueue(job, thread) + if phase == .live { + // Deliberately an unconditional RMW, never load-then-exchange: the RMW + // chain on this variable is what guarantees the pump's clearing + // `exchange(false)` synchronizes-with our lane push when we skip the wake. + if pendingJobPop.exchange(true, ordering: .acquiringAndReleasing) == false { + KQueueSelector.wakeup(fileDescriptor: kqueueFD) + } + } + } + + func enqueueBatch(_ jobs: ContiguousArray) { + precondition(phase == .live, + "StackBoundRunLoopExecutor.enqueueBatch(_:) called after its run loop finished.") + for job in jobs { + ready.enqueue(job, thread) + } + } + + /// Inserts a delayed job and, if it became a strictly-earlier deadline for its + /// domain, arms that domain's kqueue timer and posts a wakeup so the pump re-arms. + /// The timestamp computation was MOVED here from `SlicedJobQueue.enqueue(_:after:...)`. + @available(iOS 16, macOS 13, watchOS 9, tvOS 16, visionOS 1, *) + func enqueueTimer(_ job: UnownedJob, after delay: Duration, tolerance: Duration?, index: SlicedJobQueue.ClockIndex) { + guard phase == .live else { + preconditionFailure("StackBoundRunLoopExecutor.enqueue(_:after:...) called while shutting down.") + } + let timestamp = Self.timestamp(after: delay, tolerance: tolerance, index: index) + let becameNewMin: Bool = delayedJobs.withLock { heaps in + let raw = index.rawValue + let oldStamp = heaps[raw].min?.timestamp + heaps[raw].insert(TimestampJob(job: job, timestamp: timestamp)) + let newStamp = heaps[raw].min?.timestamp + return oldStamp != newStamp + } + if becameNewMin { + // Arming is owning-thread-only; just wake the pump, which re-arms via + // `armNextDeadlines`. (No producer-side `kevent64` / `armed` mutation.) + KQueueSelector.wakeup(fileDescriptor: kqueueFD) + } + } + + /// Resolves a delay + tolerance into a domain fire deadline in the same mach units + /// `KQueueSelector.now(index:)` reports. Moved verbatim from the old + /// `SlicedJobQueue.enqueue(_:after:tolerance:index:)`. + @available(iOS 16, macOS 13, watchOS 9, tvOS 16, visionOS 1, *) + private static func timestamp(after delay: Duration, tolerance: Duration?, index: SlicedJobQueue.ClockIndex) -> Timestamp { + let (delaySec, delayAtto) = delay.components + let dispatch_now: UInt64 + switch index { + case .continuous: + let mask = 1 << 63 as dispatch_time_t + dispatch_now = mask + case .suspending: + dispatch_now = 0 + case .walltime: + dispatch_now = .init(DISPATCH_WALLTIME_NOW) + } + let dispatch_target = Dispatch.__dispatch_time( + dispatch_now, + delaySec * Int64(Dispatch.NSEC_PER_SEC) + Int64(delayAtto / 1_000_000_000) + ) + let dispatch_deadline: dispatch_time_t + if let tolerance { + let (tol_sec, tol_atto) = tolerance.components + dispatch_deadline = Dispatch.__dispatch_time( + dispatch_target, + Int64(Dispatch.NSEC_PER_SEC) * tol_sec + Int64(tol_atto / 1_000_000_000) + ) + } else { + dispatch_deadline = dispatch_target + } + switch index { + case .continuous: + return .init(target: dispatch_target & ~dispatch_now, + leeway: (dispatch_deadline & ~dispatch_now) - (dispatch_target & ~dispatch_now)) + case .suspending: + return .init(target: dispatch_target, leeway: dispatch_deadline - dispatch_target) + case .walltime: + return .init(target: 0 &- dispatch_target, + leeway: (0 &- dispatch_deadline) - (0 &- dispatch_target)) + } + } + + // MARK: Stop + + func requestStop() { + stopFlag.store(true, ordering: .releasing) + KQueueSelector.wakeup(fileDescriptor: kqueueFD) + } + func clearStopRequested() { stopFlag.store(false, ordering: .releasing) } + + // MARK: Drain pump (owning thread, from the CFFileDescriptor callout) + + func handleReadable(_ fd: CFFileDescriptor) { + if withUnsafeCurrentTask(body: { $0 != nil }) { + deferredReadable = true + // Leave the callback disabled; the beforeWaiting observer re-enables it. + return + } + deferredReadable = false + pendingJobPop.store(true, ordering: .relaxed) + let fired = KQueueSelector.drainEvents(fileDescriptor: kqueueFD) // consume wake + timer fires + // A one-shot EVFILT_TIMER that fired is now disarmed in the kernel; clear our + // record so `armNextDeadlines` re-arms the domain's next deadline. + if fired.continuous { armed[0] = nil } + if fired.suspending { armed[1] = nil } + if fired.wall { armed[2] = nil } + fireDueTimers() // pop due timers -> ready lanes + ready.runBatch(executor: serial, taskRef: taskRefOrNil) // drain the 5 ready lanes only + armNextDeadlines() // arm EVFILT_TIMER from delayedJobs mins + var refire = !readyLanesEmpty() + if !refire { + _ = pendingJobPop.exchange(false, ordering: .acquiringAndReleasing) + refire = !readyLanesEmpty() + } + if refire { + KQueueSelector.wakeup(fileDescriptor: kqueueFD) + } else { + // Busy period ended (I4a): release the QoS overrides here, once, instead of + // per drain. Mirrors libdispatch's runloop-queue discipline. + ready.endAllBoosts() + } + // Predicate unwinding (Task 3 concern #1): the facade binds `currentPredicate` + // around a `runUntil` frame. After draining, evaluate it; if satisfied, stop + // the run loop so the frame unwinds. `run()` binds it to nil, so this is a + // no-op there. + let predicateSatisfied: Bool = StackBoundRunLoopExecutor.currentPredicate?.block() ?? false + if stopFlag.load(ordering: .acquiring) { + CFRunLoopStop(CFRunLoopGetCurrent()) + } else if predicateSatisfied { + stopFlag.store(true, ordering: .releasing) + CFRunLoopStop(CFRunLoopGetCurrent()) + } + CFFileDescriptorEnableCallBacks(fd, kCFFileDescriptorReadCallBack) + } + + /// Pop due entries (per domain, `timestamp.target <= now`) and feed them to the + /// ready lanes. + private func fireDueTimers() { + delayedJobs.withLockUnchecked { heaps in + for raw in 0..<3 { + let now = KQueueSelector.now(index: SlicedJobQueue.ClockIndex(rawValue: raw)!) + var popped = false + while let box = heaps[raw].min, box.timestamp.target <= now { + heaps[raw].removeMin(); ready.enqueue(box.job, thread); popped = true + } + // The armed min was just consumed — clear so `armNextDeadlines` re-arms + // the new min (guards against a stale `armed` skipping the next deadline). + if popped { armed[raw] = nil } + } + } + } + + /// Arm the earliest not-yet-due deadline per domain — but only when it is strictly + /// earlier than what is already armed (`armed[raw]`), so a steady state with an + /// unchanged min issues no `kevent64` per pump pass. + private func armNextDeadlines() { + delayedJobs.withLockUnchecked { heaps in + for raw in 0..<3 { + let index = SlicedJobQueue.ClockIndex(rawValue: raw)! + guard let stamp = heaps[raw].min?.timestamp else { continue } + if let armedStamp = armed[raw], armedStamp.deadline <= stamp.deadline { continue } + KQueueSelector.armTimer(fileDescriptor: kqueueFD, index: index, + target: stamp.target, leeway: stamp.leeway, + now: KQueueSelector.now(index: index)) + armed[raw] = stamp + } + } + } + + private func readyLanesEmpty() -> Bool { + for i in 0..<5 where !ready.jobs[i].isEmpty { return false } + return true + } + + // MARK: Shutdown + + func beginClosing() { + let old = phaseStorage.exchange(Phase.closing.rawValue, ordering: .acquiringAndReleasing) + precondition(old == Phase.live.rawValue, "KQScheduler.beginClosing() called in an invalid phase.") + } + + /// Drains the ready lanes to empty once the facade's producer gate quiesces, drops + /// any not-yet-due timers, and transitions to `.dead`. + func finishAndDie() { + while true { + drainReadyLanes() + if facade.producerGate.load(ordering: .acquiring) == 0 { + drainReadyLanes() + if readyLanesEmpty() { + // Final teardown release of any QoS overrides (I4a backstop before + // the SlicedJobQueue.deinit backstop). + ready.endAllBoosts() + dropPendingTimers() + phaseStorage.store(Phase.dead.rawValue, ordering: .releasing) + break + } + } else { + _ = sched_yield() + } + } + } + + private func drainReadyLanes() { + ready.runBatch(executor: serial, taskRef: taskRefOrNil) + } + + /// Not-yet-due timers are dropped at unwind (lifecycle contract). The kqueue + /// timers themselves die when the descriptor is invalidated in the frame's defer. + private func dropPendingTimers() { + delayedJobs.withLockUnchecked { heaps in + for raw in 0..<3 { + heaps[raw] = .init() + } + } + } + + // MARK: Run-loop source + + /// Creates the CFFileDescriptor wrapping the kqueue. Its context retains this + /// scheduler; the facade's frame owns the descriptor and invalidates it on exit + /// (closeOnInvalidate closes the kqueue then). Called exactly once, from the + /// facade's outermost run frame. + func makeFileDescriptor() -> CFFileDescriptor { + precondition(!didCreateFileDescriptor, "makeFileDescriptor() called more than once") + let names = ["CFRetain", "CFRelease", "CFCopyDescription"] + var buffer = Array(repeating: nil, count: names.count) + CFBundleGetFunctionPointersForNames( + CFBundleGetBundleWithIdentifier("com.apple.CoreFoundation" as CFString), + names as CFArray, &buffer + ) + let _CFRetain = unsafeBitCast(buffer[0], to: (@convention(c) (UnsafeMutableRawPointer?) -> UnsafeMutableRawPointer?).self) + let _CFRelease = unsafeBitCast(buffer[1], to: (@convention(c) (UnsafeMutableRawPointer?) -> Void).self) + let _CFCopyDescription = unsafeBitCast(buffer[2], to: (@convention(c) (UnsafeMutableRawPointer?) -> Unmanaged?).self) + + var context = CFFileDescriptorContext( + version: 0, + info: Unmanaged.passUnretained(self).toOpaque(), + retain: _CFRetain, + release: _CFRelease, + copyDescription: _CFCopyDescription + ) + let callback: CFFileDescriptorCallBack = { fd, _, info in + guard let info, let fd else { return } + Unmanaged.fromOpaque(info).takeUnretainedValue().handleReadable(fd) + } + guard let created = CFFileDescriptorCreate(nil, kqueueFD, true, callback, &context) else { + preconditionFailure("Failed to create CFFileDescriptor for kqueue") + } + didCreateFileDescriptor = true + return created + } + + /// Owning-thread teardown, run in the facade frame's defer. The kqueue's one-shot + /// timers are torn down with the descriptor (closeOnInvalidate); nothing to disarm + /// here beyond clearing the deferred-readable latch. + func teardown() { + deferredReadable = false + } +} + +#endif diff --git a/Sources/TetraRunLoopConcurrency/KQueueSelector.swift b/Sources/TetraRunLoopConcurrency/KQueueSelector.swift new file mode 100644 index 0000000..35637d2 --- /dev/null +++ b/Sources/TetraRunLoopConcurrency/KQueueSelector.swift @@ -0,0 +1,207 @@ +// +// KQueueSelector.swift +// TetraRunLoopConcurrency +// +// A STATELESS kqueue syscall wrapper. Holds NO state — every function takes the +// raw descriptor. Ported from swift-platform-executors KQueueSelector2.swift. +// +// * Wakeups use an `EVFILT_USER` event (ident 0). +// * Timers use one-shot `EVFILT_TIMER` events in the mach clock domains +// (idents 1/2/3 = continuous/suspending/wall, matching ClockIndex rawValues 0/1/2). +// + +#if canImport(Darwin) +import Darwin +import Dispatch + +/// Minimal error type for `kevent64(2)` failures, available at iOS 13 / macOS 10.15. +/// Callers only `assertionFailure`/`preconditionFailure` on catch, so the exact type +/// is cosmetic — `System.Errno` (macOS 11+) is intentionally avoided here. +struct KQueueSyscallError: Error { + let code: Int32 +} + +/// Thin `kqueue` descriptor wrapper: issues `kevent64(2)` and surfaces failures as +/// `KQueueSyscallError`. `rawValue` is the kqueue file descriptor. Stateless — safe +/// to construct on demand around a descriptor from any thread. +struct KQueueHelper: RawRepresentable, @unchecked Sendable { + var rawValue: Int32 + init(rawValue: Int32) { self.rawValue = rawValue } + + /// Direct `kevent64(2)` call. Returns the number of events placed in `eventlist` + /// (0 for a pure registration); throws `Errno` on a -1 return. + @discardableResult + func kevent64( + changelist: UnsafePointer?, nchanges: CInt, + eventlist: UnsafeMutablePointer?, nevents: CInt, + flags: UInt32, timeout: UnsafePointer? + ) throws -> CInt { + let result = Darwin.kevent64(rawValue, changelist, nchanges, eventlist, nevents, flags, timeout) + if result == -1 { throw KQueueSyscallError(code: errno) } + return result + } + + /// Registers a change set (no events collected): a pure `kevent64` registration. + func applyEventChangeSet(_ changes: UnsafeMutableBufferPointer) throws { + _ = try kevent64( + changelist: UnsafePointer(changes.baseAddress), nchanges: CInt(changes.count), + eventlist: nil, nevents: 0, flags: 0, timeout: nil + ) + } +} + +enum KQueueSelector { + + /// Which timer domains fired in a `drainEvents` pass. The engine uses this to + /// clear the corresponding armed deadline under its timer mutex. + struct FiredDomains { + var continuous = false + var suspending = false + var wall = false + } + + // MARK: Setup + + /// Creates a kqueue and registers the `EVFILT_USER` wakeup channel (ident 0). + static func makeKQueue() -> Int32 { + let fd = Darwin.kqueue() + precondition(fd >= 0, "kqueue() failed: \(String(cString: strerror(errno)))") + var event = kevent64_s() + event.ident = 0 + event.filter = Int16(EVFILT_USER) + event.fflags = UInt32(bitPattern: NOTE_FFNOP) + event.flags = UInt16(EV_ADD | EV_ENABLE | EV_CLEAR) + do { + try withUnsafeMutablePointer(to: &event) { + try KQueueHelper(rawValue: fd).applyEventChangeSet( + UnsafeMutableBufferPointer(start: $0, count: 1) + ) + } + } catch { + preconditionFailure("Failed to install kqueue user event: \(error)") + } + return fd + } + + // MARK: Wakeup + + /// Posts the `EVFILT_USER` wakeup, making the kqueue readable. Safe from any + /// thread; touches no shared state beyond the descriptor. + static func wakeup(fileDescriptor: Int32) { + var event = kevent64_s() + event.ident = 0 + event.filter = Int16(EVFILT_USER) + event.fflags = UInt32(NOTE_TRIGGER | NOTE_FFNOP) + do { + _ = try withUnsafePointer(to: &event) { + try KQueueHelper(rawValue: fileDescriptor).kevent64( + changelist: $0, nchanges: 1, eventlist: nil, nevents: 0, + flags: UInt32(KEVENT_FLAG_IMMEDIATE), timeout: nil + ) + } + } catch { + preconditionFailure("Failed to signal kqueue wakeup: \(error)") + } + } + + // MARK: Drain + + /// Consumes pending events non-blocking; returns which timer domains fired. + /// The `EVFILT_USER` wakeup is consumed and reported by no flag (it only + /// exists to make the kqueue readable). + static func drainEvents(fileDescriptor: Int32, maxEvents: Int = 8) -> FiredDomains { + var fired = FiredDomains() + withUnsafeTemporaryAllocation(of: kevent64_s.self, capacity: maxEvents) { buffer in + do { + let count = try KQueueHelper(rawValue: fileDescriptor).kevent64( + changelist: nil, nchanges: 0, + eventlist: buffer.baseAddress!, nevents: CInt(maxEvents), + flags: UInt32(KEVENT_FLAG_IMMEDIATE), timeout: nil + ) + for index in 0.. UInt64 { + switch index { + case .continuous: + // continuous: __dispatch_time(1<<63, 0) & ~(1<<63) + Dispatch.__dispatch_time(1 << 63, 0) & ~(1 << 63) + case .suspending: + // suspending: __dispatch_time(0, 0) + Dispatch.__dispatch_time(0, 0) + case .walltime: + // walltime: 0 &- __dispatch_walltime(nil, 0) + 0 &- Dispatch.__dispatch_walltime(nil, 0) + } + } + + // MARK: Arm + + /// Arms (or replaces) the one-shot timer for `index`. Re-arming the same + /// ident replaces the prior arming, so a producer installing an earlier + /// deadline overwrites a later one. `data` is the interval in the domain's + /// mach units; a past deadline arms `0`, firing immediately. + /// + /// - Parameters: + /// - fileDescriptor: The kqueue file descriptor. + /// - index: The clock domain (`SlicedJobQueue.ClockIndex`). + /// - target: The earliest acceptable instant (same units as `now(index:)`). + /// - leeway: Leeway window for timer coalescing (0 = no leeway). + /// - now: The current instant from `now(index:)` for computing the interval. + static func armTimer( + fileDescriptor: Int32, + index: SlicedJobQueue.ClockIndex, + target: UInt64, + leeway: UInt64, + now: UInt64 + ) { + var event = kevent64_s() + event.filter = Int16(EVFILT_TIMER) + event.flags = UInt16(EV_ADD | EV_ENABLE | EV_ONESHOT) + event.data = target > now ? Int64(target - now) : 0 + switch index { + case .continuous: + // ident 1, fflags: NOTE_MACH_CONTINUOUS_TIME | NOTE_MACHTIME + event.ident = 1 + event.fflags = UInt32(NOTE_MACH_CONTINUOUS_TIME | NOTE_MACHTIME) + case .suspending: + // ident 2, fflags: NOTE_MACHTIME + event.ident = 2 + event.fflags = UInt32(NOTE_MACHTIME) + case .walltime: + // ident 3, fflags: NOTE_NSECONDS | NOTE_MACH_CONTINUOUS_TIME + event.ident = 3 + event.fflags = UInt32(NOTE_NSECONDS | NOTE_MACH_CONTINUOUS_TIME) + } + if leeway != 0 { + event.fflags |= UInt32(NOTE_LEEWAY) + event.ext = (event.ext.0, leeway) + } + do { + try withUnsafeMutablePointer(to: &event) { + try KQueueHelper(rawValue: fileDescriptor).applyEventChangeSet( + UnsafeMutableBufferPointer(start: $0, count: 1) + ) + } + } catch { + assertionFailure("Failed to arm kqueue timer: \(error)") + } + } +} +#endif diff --git a/Sources/TetraRunLoopConcurrency/SerialExecutorRef.swift b/Sources/TetraRunLoopConcurrency/SerialExecutorRef.swift new file mode 100644 index 0000000..53ec353 --- /dev/null +++ b/Sources/TetraRunLoopConcurrency/SerialExecutorRef.swift @@ -0,0 +1,54 @@ +// +// SerialExecutorRef.swift +// TetraRunLoopConcurrency +// +// Reconstructs the concrete `any SerialExecutor` backing the *currently running* +// task, so `StackBoundRunLoopExecutor.peek()` can identify itself even when it is +// not the `currentOnLocal` TaskLocal (e.g. while running a job whose task is +// isolated to this executor). Reads the runtime's current-executor ref via +// `swift_task_getCurrentExecutor` and bit-casts it to the `(Identity, Implementation)` +// layout of `UnownedSerialExecutor`. Ported from swift-platform-executors. +// + +struct SerialExecutorRef: BitwiseCopyable { + var Identity: UnsafeRawPointer? + var Implementation: UnsafeRawPointer? + + func isGeneric() -> Bool { + return Identity == nil + } + + func isDefaultActor() -> Bool { + return !isGeneric() && Implementation == nil + } + + nonisolated + private static var WitnessTableMask: UInt { + unsafe ~(UInt(MemoryLayout.alignment) - 1) + } + + /// Rebuild the `any SerialExecutor` existential from the ref's identity + witness + /// table pointers. Returns nil for the generic/default-actor executors (no concrete + /// custom executor to recover). + nonisolated + func unsafeConvert() -> (any SerialExecutor)? { + guard !isGeneric() else { return nil } + guard !isDefaultActor() else { return nil } + let transformed = Implementation.flatMap(UInt.init) ?? 0 + let alignedExecutor = (Identity, transformed & Self.WitnessTableMask) + return unsafe unsafeBitCast(alignedExecutor, to: (any SerialExecutor)?.self) + } + + /// The concrete executor the current task is running on, or nil if not in a task. + nonisolated + static func peek() -> (any Executor)? { + return withUnsafeCurrentTask { + if $0 == nil { return nil } + let t = _task_getCurrentExecutor() + return unsafeBitCast(t, to: SerialExecutorRef.self).unsafeConvert() + } + } + + @_silgen_name("swift_task_getCurrentExecutor") + private nonisolated static func _task_getCurrentExecutor() -> UnownedSerialExecutor +} diff --git a/Sources/TetraRunLoopConcurrency/SlicedJobQueue.swift b/Sources/TetraRunLoopConcurrency/SlicedJobQueue.swift index 45ace73..1288e98 100644 --- a/Sources/TetraRunLoopConcurrency/SlicedJobQueue.swift +++ b/Sources/TetraRunLoopConcurrency/SlicedJobQueue.swift @@ -19,13 +19,10 @@ internal struct SlicedJobQueue: ~Copyable, Sendable { let jobs:FiveArray<__MPSCQueue> nonisolated(unsafe) let boost:FiveArray> -// nonisolated(unsafe) - let delayedJobs: some UnfairStateLock>> = createCheckedStateLock(checkedState: ContiguousArray>.init(repeating: .init(), count: 3)) -// let lock = NSRecursiveLock() - + let cache1:__MPSCQueue.NodeCache // let cache2:__MPSCQueue.NodeCache - + init(cacheSize:Int = 2048) { let cache1 = __MPSCQueue.NodeCache(size: cacheSize) // let cache2 = __MPSCQueue.NodeCache(size: cacheSize / 2) @@ -43,12 +40,24 @@ internal struct SlicedJobQueue: ~Copyable, Sendable { }) } + /// Drains the 5 ready lanes on the owning thread, highest-priority-first. + /// + /// Two disciplines ported from the reference engine: + /// * I4a (QoS-once): the per-lane `pthread_override` boosts are NOT ended + /// here. They are held across drains and released ONCE by the engine's + /// `endAllBoosts()` on the idle path (no refire) / at teardown — avoiding + /// per-drain start/end syscall thrash and a mid-busy priority drop. This + /// method only sets the *thread* QoS floor (`pthread_set_qos_class_self_np`) + /// per lane so the owning thread runs each lane's jobs at that lane's class. + /// * I4b (fairness cap): each lane's dequeue loop is capped at + /// `getDrainIterations(queueIndex:)`, so a high-priority flood cannot starve + /// the run loop's other CFRunLoop sources. A capped lane may leave jobs; the + /// engine pump re-checks `readyLanesEmpty()` and re-fires until empty. internal func runBatch( executor:UnownedSerialExecutor, taskRef: Builtin.Executor? = nil ) { - - var currentJobs = ContiguousArray>.init(repeating: [], count: 5) + let qos:DispatchQoS do { var _qos = QOS_CLASS_UNSPECIFIED @@ -60,71 +69,61 @@ internal struct SlicedJobQueue: ~Copyable, Sendable { defer { pthread_set_qos_class_self_np(qos.qosClass.rawValue, .init(qos.relativePriority)) } - repeat { - + do { + for i in 0..<5 { - while let t = self.jobs[i].dequeue() { - currentJobs[i].append(t) - } - var buffer = ContiguousArray() - buffer.reserveCapacity(currentJobs.capacity) - swap(&buffer, ¤tJobs[i]) - - do { - let qos = switch i { - case 0: - QOS_CLASS_USER_INTERACTIVE - case 1: - QOS_CLASS_USER_INITIATED - case 2: - QOS_CLASS_DEFAULT - case 3: - QOS_CLASS_UTILITY - case 4: - fallthrough - default: - QOS_CLASS_BACKGROUND - } - if qos != currentQos, !buffer.isEmpty { - pthread_set_qos_class_self_np(qos, 0) - currentQos = qos - } - } - do { - if let override = boost[i].exchange(nil, ordering: .relaxed) { - pthread_override_qos_class_end_np(.init(override)) - } + let laneQos = switch i { + case 0: + QOS_CLASS_USER_INTERACTIVE + case 1: + QOS_CLASS_USER_INITIATED + case 2: + QOS_CLASS_DEFAULT + case 3: + QOS_CLASS_UTILITY + case 4: + fallthrough + default: + QOS_CLASS_BACKGROUND } + let iterations = getDrainIterations(queueIndex: i) + var count = 0 if #available(macOS 15.0, iOS 18.0, watchOS 11.0, tvOS 18.0, visionOS 2.0, *), let t = taskRef { let taskExecutor = UnownedTaskExecutor(t) - for j in buffer { + while count < iterations, let j = self.jobs[i].dequeue() { + if count == 0, laneQos != currentQos { + pthread_set_qos_class_self_np(laneQos, 0) + currentQos = laneQos + } j.runSynchronously(isolatedTo: executor, taskExecutor: taskExecutor) + count &+= 1 } } else { - for j in buffer { + while count < iterations, let j = self.jobs[i].dequeue() { + if count == 0, laneQos != currentQos { + pthread_set_qos_class_self_np(laneQos, 0) + currentQos = laneQos + } j.runSynchronously(on: executor) + count &+= 1 } } } - let times = [ - Dispatch.__dispatch_time(1 << 63,0) & ~(1 << 63), - Dispatch.__dispatch_time(0,0), - 0 &- Dispatch.__dispatch_walltime(nil,0) - ] - self.delayedJobs.withLockUnchecked { - for i in 0..<3 { - while let jobBox = $0[i].min, jobBox.timestamp.target <= times[i] { - $0[i].removeMin() - let index = if #available(macOS 14.0, iOS 17.0, watchOS 10.0, tvOS 17.0, *) { - TaskPriority(jobBox.job.priority)?.jobQueueIndex ?? 4 - } else { - 2 - } - currentJobs[index].append(jobBox.job) - } - } + } + } + + /// Release every active QoS override. Called by the engine ONLY when the busy + /// period ends (no refire) or at teardown — NOT per drain. Holding overrides + /// across drains mirrors libdispatch's runloop-queue discipline + /// (`_dispatch_runloop_queue_wakeup`: end the override only when the queue drains + /// empty), avoiding per-drain start/end syscall thrash and the mid-busy priority + /// drop that would otherwise open an inversion window. + internal func endAllBoosts() { + for i in 0..<5 { + if let override = boost[i].exchange(nil, ordering: .acquiring) { + pthread_override_qos_class_end_np(.init(override)) } - } while !currentJobs.allSatisfy(\.isEmpty) + } } nonisolated func enqueue(_ job:UnownedJob, _ thread:pthread_t) { @@ -132,7 +131,10 @@ internal struct SlicedJobQueue: ~Copyable, Sendable { let priority = TaskPriority(job.priority) let index = priority?.jobQueueIndex ?? 4 jobs[index].enqueue(job) - if boost[index].load(ordering: .acquiring) != nil { + // Install a QoS override for this lane only when none is active yet — the + // hot path (a burst of same-priority jobs) then costs one acquiring load and + // no syscall. (Was inverted `!= nil`, which never installed an override.) + if boost[index].load(ordering: .acquiring) == nil { let qos = switch index { case 0: QOS_CLASS_USER_INTERACTIVE @@ -159,63 +161,6 @@ internal struct SlicedJobQueue: ~Copyable, Sendable { } } - @available(macOS 14.0, iOS 17.0, watchOS 10.0, tvOS 17.0, *) - nonisolated - internal func enqueue( - _ job:UnownedJob, - after delay: Swift.Duration, - tolerance: Swift.Duration? = nil, - index:ClockIndex - ) -> Bool { - let (delaySec, delayAtto) = delay.components - let dispatch_now:UInt64 - - switch index { - case .continuous: - let mask = 1 << 63 as dispatch_time_t - dispatch_now = mask - case .suspending: - dispatch_now = 0 - case .walltime: - dispatch_now = .init(DISPATCH_WALLTIME_NOW) - } - let dispatch_target = Dispatch.__dispatch_time( - dispatch_now, - delaySec * Int64(Dispatch.NSEC_PER_SEC) + Int64(delayAtto / 1_000_000_000) - ) - let dispatch_deadline:dispatch_time_t - if let tolerance { - let (tol_sec, tol_atto) = tolerance.components - dispatch_deadline = Dispatch.__dispatch_time( - dispatch_target, - Int64(Dispatch.NSEC_PER_SEC) * tol_sec + Int64(tol_atto / 1_000_000_000) - ) - } else { - dispatch_deadline = dispatch_target - } - let timestamp:Timestamp - switch index { - case .continuous: - timestamp = .init(target: dispatch_target & ~dispatch_now, leeway: (dispatch_deadline & ~dispatch_now) - (dispatch_target & ~dispatch_now)) - break - case .suspending: - timestamp = .init(target: dispatch_target, leeway: dispatch_deadline - dispatch_target) - break - case .walltime: - timestamp = .init(target: 0 &- dispatch_target, leeway: (0 &- dispatch_deadline) - (0 &- dispatch_target)) - break - } - let needsWakeup = delayedJobs.withLock { - let oldStamp = $0[index.rawValue].min?.timestamp - $0[index.rawValue].insert( - .init(job: job, timestamp: timestamp) - ) - let newStamp = $0[index.rawValue].min?.timestamp - return oldStamp != newStamp - } - return needsWakeup - } - internal enum ClockIndex:Int, Sendable, BitwiseCopyable { case continuous case suspending @@ -358,6 +303,16 @@ internal struct SlicedJobQueue: ~Copyable, Sendable { return result } + /// Consumer-side emptiness check: true when there is no dequeuable node. + /// Mirrors `dequeue`'s producer-published-`next` protocol (an in-flight + /// producer that has swung `tail` but not yet published `next` reads as empty, + /// same as `dequeue` returning nil). + @inline(__always) + public var isEmpty: Bool { + let currentHead = head.load(ordering: .relaxed).load(BufferNode.self) + return currentHead.pointee.next.load(ordering: .acquiring) == nil + } + @inline(__always) public func withFirst(_ body: (borrowing Element?) throws(Failure) -> T) throws(Failure) -> T { let currentHead = head.load(ordering: .relaxed).load(BufferNode.self) @@ -465,14 +420,19 @@ extension TaskPriority { } +/// Per-lane drain cap (I4b fairness). Ported from the reference engine's exact +/// values: userInteractive(0) is uncapped; high(1) and default(2) get 128; utility(3) +/// gets 2; background(4) gets 1. A capped lane leaves residual jobs, which the engine +/// pump drains on subsequent re-fires (see `handleReadable`'s `readyLanesEmpty` recheck). @inlinable @inline(__always) internal func getDrainIterations(queueIndex: Int) -> Int { switch queueIndex { - case 0: .max // high - case 1: 128 // medium - case 2: 2 // low - default : 1 // background and lower + case 0: .max // userInteractive + case 1: 128 // high + case 2: 128 // default + case 3: 2 // utility + default: 1 // background and lower } } @@ -541,31 +501,6 @@ class Backing { func dispatch() { store.runBatch(executor: serialExecutor.unsafelyUnwrapped, taskRef: taskRef) - - let timeout = store.delayedJobs.withLock { - - $0.map(\.min?.timestamp) - } - - for i in timeout.indices { - if var t = timeout[i] { - let s = timers[i] - var start = t.target - if i == 0 { - start |= 1 << 63 - } - if i == 2 { - start = 0 &- start - } - Dispatch.__dispatch_source_set_timer( - s, - start, - DispatchTime.distantFuture.rawValue, - t.leeway - ) - } - } - } func schedule(_ runloop:CFRunLoop, _ mode:CFRunLoopMode) { diff --git a/Sources/TetraRunLoopConcurrency/StackBoundRunLoopExecutor.swift b/Sources/TetraRunLoopConcurrency/StackBoundRunLoopExecutor.swift new file mode 100644 index 0000000..76498e1 --- /dev/null +++ b/Sources/TetraRunLoopConcurrency/StackBoundRunLoopExecutor.swift @@ -0,0 +1,515 @@ +// +// StackBoundRunLoopExecutor.swift +// TetraRunLoopConcurrency +// +// The FACADE half of the kqueue/CFFileDescriptor run-loop executor. A +// `SerialExecutor` that mounts a stack-bound `KQScheduler` engine on the +// current CFRunLoop thread, buffers jobs while dormant, and delegates the run +// frame (`run`/`runUntil`/`stop`) to the engine. +// +// Ported from swift-platform-executors' `StackBoundRunLoopExecutor2` facade +// half, adapted to Tetra: +// +// * iOS 13 / macOS 10.15 (the reference's macOS 15 / iOS 18 gate is dropped; +// the `ExecutorJob` / timer surfaces keep their own finer availability). +// * `AtomicStore` (not Synchronization `Atomic`) for `producerGate` and the +// published-scheduler pointer; `createCheckedStateLock` for the mount FSM. +// * The engine (`KQScheduler`) already owns the run frame + mount lifetime, so +// this facade only publishes a freshly mounted engine and forwards to it. +// * No `TaskExecutor` conformance here — Task 5 supplies the gated task ref. +// `taskRef` stays nil (the iOS 13 `SerialExecutor` path). +// +// Concern wiring (Task 3 hand-offs resolved here + in `KQScheduler.swift`): +// 1. Predicate unwinding: `currentPredicate` is bound around the `runUntil` +// frame; the engine's `handleReadable` consults it after each drain and +// stops the run loop when it returns true. +// 2. `current()` / mount: `currentOnLocal` is bound around the run frame so a +// nested `current()` on the same thread recovers this facade. +// 3. Nested re-entry: the engine's own `enterRunLoop` installs the +// `beforeWaiting` observer + `deferredReadable` guard. +// + +#if canImport(Darwin) +import Darwin +import Dispatch +import CoreFoundation +import CriticalSection +import Builtin +// SchedulingExecutor / RunLoopExecutor / MainExecutor are SPI on `_Concurrency`. +// The SPI groups match the ones the test target imports. +@_spi(ExperimentalScheduling) @_spi(ConcurrencyExecutors) @_spi(ExperimentalCustomExecutors) import _Concurrency + +@available(macOS 10.15, iOS 13, *) +public final class StackBoundRunLoopExecutor: SerialExecutor, @unchecked Sendable { + + /// Dormant (buffering) → live (engine mounted & published) → dead (unwound). + private enum MountState { + case dormant(ContiguousArray) + case live + case dead + } + private let mount: some UnfairStateLock = + createCheckedStateLock(checkedState: MountState.dormant([])) + + /// The published live engine, dereferenced by producers while `producerGate` + /// is held. `nil` unless mounted. Stored as a raw pointer inside an + /// `AtomicStore` (Tetra has no `AtomicStore?>`). + private let liveSchedulerBits = AtomicStore?>(.none) + + /// Producers in flight. Held across every dereference of the live engine and + /// doubles as the engine's shutdown quiescence count (`finishAndDie`). + let producerGate = AtomicStore(0) + + nonisolated(unsafe) let cfRunLoop: CFRunLoop + nonisolated(unsafe) let thread: pthread_t + + // MARK: Task locals (concern #1 predicate, #2 mount recovery) + + @TaskLocal static var currentOnLocal: Unmanaged? + @TaskLocal static var currentPredicate: UnsafeBlockBox? + + struct UnsafeBlockBox: @unchecked Sendable { + nonisolated(unsafe) let block: () -> Bool + } + + #if DEBUG + /// Test-only hook, invoked with a freshly mounted engine (before the run frame + /// spins) so a lifetime test can capture it in a `weak` box and later assert it + /// deallocated once `run()` returned. Not compiled into release. + nonisolated(unsafe) static var _debugOnMountEngine: ((KQScheduler) -> Void)? + #endif + + // MARK: Init / mount lookup + + private init() { + self.thread = pthread_self() + self.cfRunLoop = CFRunLoopGetCurrent() + } + + /// Recover the facade mounted on the current thread, if any. + static func peek() -> StackBoundRunLoopExecutor? { + if let existing = currentOnLocal?.takeUnretainedValue(), + pthread_equal(existing.thread, pthread_self()) != 0 { + return existing + } + // Fallback: we may be running inside a job whose task is isolated to this + // executor without `currentOnLocal` being bound. Recover the concrete executor + // from the current task's serial-executor ref. + return withUnsafeCurrentTask { + if $0 != nil, let serial = SerialExecutorRef.peek() as? StackBoundRunLoopExecutor { + return serial + } + return nil + } + } + + /// Mount an engine on the current CFRunLoop thread (or recover the one already + /// mounted here). + public static func current() -> StackBoundRunLoopExecutor { + if let existing = peek() { return existing } + return StackBoundRunLoopExecutor() + } + + public var isMainExecutor:Bool { + false + } + + // MARK: Published-engine helpers + + private func publishedScheduler() -> Unmanaged? { + return liveSchedulerBits.load(ordering: .acquiring) + } + + private func publish(_ scheduler: KQScheduler) { +// let raw = Unmanaged.passUnretained(scheduler).toOpaque() + liveSchedulerBits.store(.passUnretained(scheduler), ordering: .releasing) + } + + /// Retire the published engine: no producer may dereference it after this + /// returns, and the mount FSM is dead. Spins until every in-flight producer + /// releases the gate. + private func retireScheduler() { + liveSchedulerBits.store(nil, ordering: .releasing) + mount.withLock { $0 = .dead } + while producerGate.load(ordering: .acquiring) != 0 { _ = sched_yield() } + } + + // MARK: producerGate counter (AtomicStore has no wrapping add/sub) + + private func gateEnter() { + while true { + let old = producerGate.load(ordering: .relaxed) + let (ok, _) = producerGate.compareExchange( + expected: old, desired: old &+ 1, ordering: .acquiring) + if ok { return } + } + } + private func gateLeave() { + while true { + let old = producerGate.load(ordering: .relaxed) + let (ok, _) = producerGate.compareExchange( + expected: old, desired: old &- 1, ordering: .releasing) + if ok { return } + } + } + + // MARK: SerialExecutor + + /// Primary `SerialExecutor` conformance method, available at iOS 13 / macOS 10.15. + public func enqueue(_ job: UnownedJob) { + gateEnter() + defer { gateLeave() } + + if let ref = publishedScheduler() { + ref._withUnsafeGuaranteedRef { $0.enqueueReady(job) } + return + } + let scheduler = mount.withLock { state -> Unmanaged? in + switch state { + case .dormant(var buffer): + state = .dormant([]) + buffer.append(job) + state = .dormant(buffer) + return nil + case .live: + return publishedScheduler() + case .dead: + preconditionFailure("StackBoundRunLoopExecutor.enqueue(_:) called after its run loop finished.") + } + } + scheduler?._withUnsafeGuaranteedRef { $0.enqueueReady(job) } + } + + /// `ExecutorJob` overload available on newer platforms; forwards to the primary path. + @available(macOS 14, iOS 17, watchOS 10, tvOS 17, visionOS 1, *) + public func enqueue(_ job: consuming ExecutorJob) { + enqueue(UnownedJob(job)) + } + + public func asUnownedSerialExecutor() -> UnownedSerialExecutor { + if #available(macOS 14.0, iOS 17.0, watchOS 10.0, tvOS 17.0, visionOS 1.0, *) { + return .init(complexEquality: self) + } else { + return .init(ordinary: self) + } + } + + public static func == (lhs: StackBoundRunLoopExecutor, rhs: StackBoundRunLoopExecutor) -> Bool { + pthread_equal(lhs.thread, rhs.thread) != 0 + } + + public func checkIsolated() { + precondition(isIsolatingCurrentContext() == true, + "Caller is not isolated to this executor's thread.") + } + public func isIsolatingCurrentContext() -> Bool? { + pthread_equal(thread, pthread_self()) != 0 + } + + // MARK: Timer enqueue (delegates to the engine's clock-domain heaps) + + /// `ClockIndex` is module-internal, so this timer surface is `internal` even + /// though the type is public (mirrors the reference's `SchedulingExecutor` + /// surface, reduced to the concrete Tetra domain index). +// @available(iOS 16, macOS 13, watchOS 9, tvOS 16, *) + @available(macOS 14.0, iOS 17.0, watchOS 10.0, tvOS 17.0, *) + func enqueue(_ job: consuming ExecutorJob, after: Duration, + tolerance: Duration? = nil, clock: SlicedJobQueue.ClockIndex) { + let unowned = UnownedJob(job) + gateEnter() + defer { gateLeave() } + + if let ref = publishedScheduler() { + ref._withUnsafeGuaranteedRef { + $0.enqueueTimer(unowned, after: after, tolerance: tolerance, index: clock) + } + return + } + // Dormant / not-yet-published: buffer the job on the ready lanes (fires + // ASAP once mounted). The engine owns the delay heaps and is only reachable + // once live; timers scheduled before mount degrade to immediate. + let scheduler = mount.withLock { state -> Unmanaged? in + switch state { + case .dormant(var buffer): + state = .dormant([]) + buffer.append(unowned) + state = .dormant(buffer) + return nil + case .live: + return publishedScheduler() + case .dead: + preconditionFailure("StackBoundRunLoopExecutor.enqueue(_:after:...) called after its run loop finished.") + } + } + scheduler?._withUnsafeGuaranteedRef { + $0.enqueueTimer(unowned, after: after, tolerance: tolerance, index: clock) + } + } + + // MARK: Run frame (owned by the facade; drives the engine's pump) + + public func run() throws { + try enterRunLoop(stopScope: true) { engine in + Self.$currentPredicate.withValue(nil) { + while !engine._withUnsafeGuaranteedRef(\.isStopRequested) { + let result = CFRunLoopRunInMode(.defaultMode, 1.0e10, false) + if result == .finished { break } + } + } + } + } + + public func runUntil(_ condition: @escaping () -> Bool) throws { + try enterRunLoop(stopScope: false) { engine in + withoutActuallyEscaping(condition) { escaping in + Self.$currentPredicate.withValue(.init(block: escaping)) { + while !engine._withUnsafeGuaranteedRef(\.isStopRequested) { + let result = CFRunLoopRunInMode(.defaultMode, 1.0e10, false) + if result == .finished { break } + } + } + } + } + } + + public func stop() { + let onOwningThread = pthread_equal(thread, pthread_self()) != 0 + gateEnter() + publishedScheduler()?._withUnsafeGuaranteedRef { $0.requestStop() } + gateLeave() + if onOwningThread { + CFRunLoopStop(cfRunLoop) + } + } + + // MARK: Mount + install run-loop source + spin + + /// Mounts a fresh engine on the current thread (or reuses the one already mounted + /// for nested re-entry), installs the CFFileDescriptor run-loop source + a + /// `beforeWaiting` observer that re-enables the one-shot readable callback deferred + /// while inside a task context, binds the mount task-local, spins the caller's loop, + /// then tears everything down, drains to empty, and retires the engine. + private func enterRunLoop(stopScope: Bool, + _ spin: (Unmanaged) -> Void) throws { + guard pthread_equal(thread, pthread_self()) != 0 else { + preconditionFailure("StackBoundRunLoopExecutor.run()/runUntil(_:) called from a non-owning thread.") + } + guard withUnsafeCurrentTask(body: { $0 == nil }) else { + throw CancellationError() + } + + // Nested re-entry: reuse the already-mounted engine. The fd/observer are + // installed once by the outermost frame. The engine stays alive for the nested + // spin via the outer frame's keep-alive (the CFFileDescriptor context retain + + // the outer frame's strong local), so we hand the spin an unretained + // `Unmanaged` — no extra retain across the loop. + if Self.peek() === self, let ref = publishedScheduler() { + if !stopScope { + // Force one drain pass so the predicate is evaluated on entry. + ref._withUnsafeGuaranteedRef { KQueueSelector.wakeup(fileDescriptor: $0.kqueueFD) } + } + defer { + ref._withUnsafeGuaranteedRef { $0.clearStopRequested() } + } + spin(ref) + return + } + + // Outermost: mount a fresh engine bound to this thread's run loop, drain any + // buffered jobs, publish it, install the run-loop source, spin, then retire it + // (stack-bound lifetime). + // + // On the iOS-18 path this facade is a `TaskExecutor`, so hand the engine a + // task-executor ref; `runBatch` then runs jobs via + // `runSynchronously(isolatedTo:taskExecutor:)`. Below iOS 18 the ref stays nil + // (the `SerialExecutor`-only `runSynchronously(on:)` path). + let taskRef: Builtin.Executor? + if #available(iOS 18, macOS 15, watchOS 11, tvOS 18, visionOS 2, *) { + taskRef = self.asUnownedTaskExecutor()._executor + } else { + taskRef = nil + } + // The frame keeps a plain strong local; the CFFileDescriptor context holds a + // second retain. Both drop on exit — so the engine deallocates when this frame + // returns (stack-bound lifetime; the lifetime test enforces this). + unowned(unsafe) let engine: KQScheduler + let fd: CFFileDescriptor + do { + let _engine = KQScheduler(facade: self, + serial: asUnownedSerialExecutor(), + taskRef: taskRef, + cfRunLoop: cfRunLoop) + let buffered: ContiguousArray = mount.withLock { state in + switch state { + case .dormant(let buffer): + // Publish inside the lock so a producer that observes `.live` here + // never reads a nil published pointer (and drops the job). + publish(_engine) + state = .live + return buffer + case .live, .dead: + return [] + } + } + fd = _engine.makeFileDescriptor() + engine = _engine + _engine.enqueueBatch(buffered) + + #if DEBUG + Self._debugOnMountEngine?(_engine) + #endif + } + + guard let source = CFFileDescriptorCreateRunLoopSource(nil, fd, 0) else { + // Undo the publish so no producer dereferences the about-to-be-freed + // engine, then invalidate the descriptor (closes the kqueue). + retireScheduler() + CFFileDescriptorInvalidate(fd) + throw CancellationError() + } + + let observer = CFRunLoopObserverCreateWithHandler( + nil, ([.beforeWaiting] as CFRunLoopActivity).rawValue, true, 0 + ) { [unowned(unsafe) engine, unowned(unsafe) fd] _, activity in + // Re-arm a readable that was deferred because we were inside a task-context. + // Safe: the observer is removed and invalidated in the same defer that + // precedes the engine's release, and the fd is valid for the frame. + if activity == .beforeWaiting, engine.deferredReadable, withUnsafeCurrentTask(body: { $0 == nil }) { + engine.deferredReadable = false + CFFileDescriptorEnableCallBacks(fd, kCFFileDescriptorReadCallBack) + } + }! + + Self.$currentOnLocal.withValue(Unmanaged.passUnretained(self)) { + CFRunLoopAddSource(cfRunLoop, source, .commonModes) + CFRunLoopAddObserver(cfRunLoop, observer, .commonModes) + CFFileDescriptorEnableCallBacks(fd, kCFFileDescriptorReadCallBack) + + // Initial kick: drain anything buffered before the source was installed. + KQueueSelector.wakeup(fileDescriptor: engine.kqueueFD) + + defer { + CFFileDescriptorDisableCallBacks(fd, kCFFileDescriptorReadCallBack) + CFRunLoopRemoveSource(cfRunLoop, source, .commonModes) + CFRunLoopRemoveObserver(cfRunLoop, observer, .commonModes) + CFRunLoopSourceInvalidate(source) + CFRunLoopObserverInvalidate(observer) + engine.teardown() + // Invalidate the descriptor last: closes the kqueue (closeOnInvalidate) + // and drops the context's retain; the local strong `engine` releases as + // this frame returns (stack-bound lifetime). + CFFileDescriptorInvalidate(fd) + } + + spin(.passUnretained(engine)) + engine.clearStopRequested() + engine.beginClosing() + engine.finishAndDie() + retireScheduler() + } + } +} + +// MARK: - Gated executor-protocol conformances (Task 5) +// +// All four protocols floor at iOS 16 / macOS 13, but this executor supplies a +// task executor to `runBatch` via `runSynchronously(isolatedTo:taskExecutor:)`, +// which is iOS 18 / macOS 15. So the conformances are gated at that higher floor. +// `SchedulingExecutor` / `RunLoopExecutor` / `MainExecutor` are SPI (see the +// `@_spi(...) import _Concurrency` at the top of this file). + +@available(iOS 18, macOS 15, watchOS 11, tvOS 18, visionOS 2, *) +extension StackBoundRunLoopExecutor: TaskExecutor { + // `asUnownedTaskExecutor()` is provided by the protocol's default extension; + // no explicit member needed. The engine reads `asUnownedTaskExecutor()._executor` + // when it mounts (see `enterRunLoop`). +} + +// The SPI executor protocols `SchedulingExecutor` / `RunLoopExecutor` / `MainExecutor` +// are NOT nameable on the standard/release stdlib: even a `@_spi(...) import _Concurrency` +// cannot see them (they are `internal` on 6.2 as `SchedulableExecutor` etc., and the +// release .swiftinterface does not re-export the SPI decls — confirmed: release Swift 6.4 +// reports `cannot find type 'SchedulingExecutor'`). Their visibility depends on the +// TOOLCHAIN FLAVOR (a development snapshot's stdlib exposes them; a release one does not), +// which no `#if compiler(>=x)` can distinguish. So gate on the `SchedulingExecutorSPI` +// package trait (Package.swift): OFF by default (release/standard toolchains → excluded → +// the module compiles as a SerialExecutor + TaskExecutor, clock scheduling falling back to +// the global executor), enabled with `swift build --traits SchedulingExecutorSPI` only on a +// toolchain whose stdlib actually exposes these SPI protocols. +#if SchedulingExecutorSPI +@available(iOS 18, macOS 15, watchOS 11, tvOS 18, visionOS 2, *) +@_spi(ExperimentalCustomExecutors) +extension StackBoundRunLoopExecutor: RunLoopExecutor { + // `run()` / `runUntil(_:)` / `stop()` already exist on the facade. (The facade's + // `runUntil` takes an `@escaping` closure, which satisfies the protocol's + // non-escaping requirement.) +} + +@available(iOS 18, macOS 15, watchOS 11, tvOS 18, visionOS 2, *) +@_spi(ExperimentalScheduling) +extension StackBoundRunLoopExecutor: SchedulingExecutor { + + /// Delay-scheduling entry point. Maps the standard clocks to the engine's + /// `SlicedJobQueue.ClockIndex` domains and delegates to the internal + /// `enqueue(_:after:tolerance:clock:)` timer path (which computes the deadline in + /// the engine — this facade never recomputes timestamps). + /// + /// `ContinuousClock.Duration` and `SuspendingClock.Duration` are both + /// `Swift.Duration`, so the delay/tolerance forward unchanged. + /// + /// For any clock this executor does not model, we prefer to trampoline through the + /// global concurrent executor (so the job still fires on the *right* clock) and then + /// hop back here to run. If that SPI cast is unavailable at runtime we fall back to + /// treating the delay on the suspending (uptime) domain rather than crashing — an + /// approximation, documented, for exotic custom clocks only. + @_spi(ExperimentalScheduling) + public func enqueue( + _ job: consuming ExecutorJob, + after delay: C.Duration, + tolerance: C.Duration? = nil, + clock: C + ) { + if clock is ContinuousClock { + let d = delay as! Swift.Duration + let tol = tolerance as! Swift.Duration? + self.enqueue(job, after: d, tolerance: tol, clock: .continuous) + return + } + if clock is SuspendingClock { + let d = delay as! Swift.Duration + let tol = tolerance as! Swift.Duration? + self.enqueue(job, after: d, tolerance: tol, clock: .suspending) + return + } + // Unmodeled clock: trampoline through the global executor if it schedules, + // otherwise approximate on the suspending domain (Tetra has no WallClock type). + if let global = globalConcurrentExecutor as? (any SchedulingExecutor), !(global === self) { + let trampoline = job.createTrampoline(to: self) + global.enqueue(trampoline, after: delay, tolerance: tolerance, clock: clock) + } else { + // Fallback: run the delay on the suspending (uptime) domain. This is an + // approximation for clocks other than Continuous/Suspending; it never + // crashes on an unknown clock. + let d = (delay as? Swift.Duration) ?? .zero + let tol = tolerance as? Swift.Duration + self.enqueue(job, after: d, tolerance: tol, clock: .suspending) + } + } + + nonisolated public var asSchedulingExecutor: (any SchedulingExecutor)? { self } + + + /// Deadline-scheduling entry point. The `SchedulingExecutor` extension supplies a + /// default that converts `at:` to `after:` via `clock.now`, so no override is + /// required here; it routes through `enqueue(_:after:tolerance:clock:)` above. +} + +@available(iOS 18, macOS 15, watchOS 11, tvOS 18, visionOS 2, *) +@_spi(ExperimentalCustomExecutors) +extension StackBoundRunLoopExecutor: MainExecutor { + // MainExecutor == RunLoopExecutor + SerialExecutor; both are already satisfied. + // `isMainExecutor` is provided by the SerialExecutor default (returns false). +} +#endif // SchedulingExecutorSPI trait — SchedulingExecutor/RunLoopExecutor/MainExecutor + +#endif diff --git a/Tests/TetraRunLoopConcurrencyTests/KQueueSelectorTests.swift b/Tests/TetraRunLoopConcurrencyTests/KQueueSelectorTests.swift new file mode 100644 index 0000000..920fe96 --- /dev/null +++ b/Tests/TetraRunLoopConcurrencyTests/KQueueSelectorTests.swift @@ -0,0 +1,20 @@ +import Testing +import Darwin +@testable import TetraRunLoopConcurrency + +@Suite struct KQueueSelectorTests { + @Test func wakeupMakesKqueueReadable() { + let fd = KQueueSelector.makeKQueue(); defer { close(fd) } + KQueueSelector.wakeup(fileDescriptor: fd) + let fired = KQueueSelector.drainEvents(fileDescriptor: fd) // consumes the EVFILT_USER + #expect(!fired.continuous && !fired.suspending && !fired.wall) // user wake reports no timer domain + } + @Test func suspendingTimerFires() { + let fd = KQueueSelector.makeKQueue(); defer { close(fd) } + let now = KQueueSelector.now(index: .suspending) + KQueueSelector.armTimer(fileDescriptor: fd, index: .suspending, target: now, leeway: 0, now: now) // due immediately + var pollFd = pollfd(fd: fd, events: Int16(POLLIN), revents: 0) + #expect(poll(&pollFd, 1, 500) == 1) // becomes readable within 500ms + #expect(KQueueSelector.drainEvents(fileDescriptor: fd).suspending) + } +} diff --git a/Tests/TetraRunLoopConcurrencyTests/StackBoundRunLoopExecutorLifetimeTests.swift b/Tests/TetraRunLoopConcurrencyTests/StackBoundRunLoopExecutorLifetimeTests.swift new file mode 100644 index 0000000..97d8636 --- /dev/null +++ b/Tests/TetraRunLoopConcurrencyTests/StackBoundRunLoopExecutorLifetimeTests.swift @@ -0,0 +1,93 @@ +// +// StackBoundRunLoopExecutorLifetimeTests.swift +// TetraRunLoopConcurrencyTests +// +// Task C lifetime verification: after `run()`/`runUntil()` returns, the +// stack-bound `KQScheduler` engine (and, transitively, its `SlicedJobQueue`, +// CFFileDescriptor, and run-loop observer) must deallocate promptly — the run +// loop carries no extra retain on the engine. +// +// Observed by capturing the freshly mounted engine through the DEBUG-only +// `_debugOnMountEngine` hook into a `weak` box, then asserting the box is nil +// after `run()` returns. +// +// The hook is a PROCESS-GLOBAL, so while it is installed, engines mounted by +// *other* (parallel) test suites' `current()` calls fire it too. We therefore +// guard the capture on the test's own worker thread (`expectedThread`): only the +// engine mounted on this test's thread is recorded, so a concurrently-running +// suite can never contaminate `box.engine`. `.serialized` keeps this suite's own +// two tests from overlapping on the shared hook. +// + +#if canImport(Darwin) && DEBUG +import Testing +import Foundation +import CoreFoundation +import Dispatch +import Darwin +@testable import TetraRunLoopConcurrency +import CriticalSection + +@Suite(.serialized) +struct StackBoundRunLoopExecutorLifetimeTests { + + /// Holds a weak reference to the engine mounted on `expectedThread`, so the test + /// thread can inspect it after the worker thread's `run()` returns. Test-only; + /// `expectedThread` is written once (on the worker, before mount) and read by the + /// hook on the mounting thread — a benign test-only cross-thread read of a + /// pointer-sized value. + final class WeakEngineBox: @unchecked Sendable { + weak var engine: KQScheduler? + var expectedThread: pthread_t? + } + + /// After a mounted engine's `run()` returns, its `KQScheduler` deallocates: + /// the run loop held no extra retain, so the weak reference reads nil. + @Test(.timeLimit(.minutes(1))) @available(macOS 9999, *) + func engineDeallocatesAfterRunReturns() { + let box = WeakEngineBox() + StackBoundRunLoopExecutor._debugOnMountEngine = { eng in + if let t = box.expectedThread, pthread_equal(eng.thread, t) != 0 { box.engine = eng } + } + defer { StackBoundRunLoopExecutor._debugOnMountEngine = nil } + + onThread { + box.expectedThread = pthread_self() + let executor = StackBoundRunLoopExecutor.current() + executor.enqueue(makeJob(priority: .medium) { executor.stop() }) + try! executor.run() + } + + // `run()` returned on the worker thread: the frame's strong `engine` local + // dropped and the CFFileDescriptor was invalidated, dropping its context + // retain. Nothing else holds the engine, so it must be gone. + #expect(box.engine == nil) + } + + /// The facade can be re-mounted and re-run on a fresh thread cleanly, and that + /// second engine also deallocates — proving no live engine leaked from the first. + @Test(.timeLimit(.minutes(1))) @available(macOS 9999, *) + func reMountAndReRunDeallocatesEachEngine() { + for _ in 0..<2 { + let box = WeakEngineBox() + let ran = ManagedUnfairLock(initialState: false) + StackBoundRunLoopExecutor._debugOnMountEngine = { eng in + if let t = box.expectedThread, pthread_equal(eng.thread, t) != 0 { box.engine = eng } + } + onThread { + box.expectedThread = pthread_self() + let executor = StackBoundRunLoopExecutor.current() + executor.enqueue(makeJob(priority: .medium) { + ran.withLockUnchecked { $0 = true } + executor.stop() + }) + try! executor.run() + } + StackBoundRunLoopExecutor._debugOnMountEngine = nil + #expect(ran.withLockUnchecked { $0 }) + #expect(box.engine == nil) + } + } +} + +#endif // canImport(Darwin) && DEBUG diff --git a/Tests/TetraRunLoopConcurrencyTests/StackBoundRunLoopExecutorTests.swift b/Tests/TetraRunLoopConcurrencyTests/StackBoundRunLoopExecutorTests.swift new file mode 100644 index 0000000..3ea1d2e --- /dev/null +++ b/Tests/TetraRunLoopConcurrencyTests/StackBoundRunLoopExecutorTests.swift @@ -0,0 +1,260 @@ +// +// StackBoundRunLoopExecutorTests.swift +// TetraRunLoopConcurrencyTests +// +// Behavioral-verification suite for StackBoundRunLoopExecutor. +// Ported from swift-platform-executors' +// DarwinRunLoopExecutorTests/StackBoundRunLoopExecutor2Tests.swift (10 tests). +// +// Adaptations: +// * Type: StackBoundRunLoopExecutor (from @testable import TetraRunLoopConcurrency) +// * makeJob / onThread: from TestSupport.swift (Task 1). Not redefined here. +// * Synchronization.Mutex → ManagedUnfairLock (iOS 13+ compatible). +// * Timer clock arg: SlicedJobQueue.ClockIndex (.continuous / .suspending / .walltime). +// * Tests are NOT async: onThread(_:) is synchronous-blocking. +// * Timer tests gated @available(iOS 16, macOS 13, watchOS 9, tvOS 16, *) +// in addition to the @available(macOS 9999, *) makeJob gate. +// + +#if canImport(Darwin) +import Testing +import Foundation +import CoreFoundation +import Dispatch +@testable import TetraRunLoopConcurrency +// CriticalSection is an internal dependency; ManagedUnfairLock is accessible +// because the test target imports TetraRunLoopConcurrency @testable. +import CriticalSection + +@Suite +struct StackBoundRunLoopExecutorTests { + + // MARK: Non-timer tests + + /// Jobs enqueued while the executor is dormant buffer and flush once run() mounts. + @Test(.timeLimit(.minutes(1))) @available(macOS 9999, *) + func dormantEnqueueBuffersUntilRun() { + let order = ManagedUnfairLock<[Int]>(initialState: []) + onThread { + let executor = StackBoundRunLoopExecutor.current() + executor.enqueue(makeJob(priority: .medium) { order.withLockUnchecked { $0.append(1) } }) + executor.enqueue(makeJob(priority: .medium) { order.withLockUnchecked { $0.append(2) } }) + executor.enqueue(makeJob(priority: .medium) { + order.withLockUnchecked { $0.append(3) } + executor.stop() + }) + try! executor.run() + } + #expect(order.withLockUnchecked { $0 } == [1, 2, 3]) + } + + /// Stress: many threads enqueue concurrently; every job runs exactly once. + @Test(.timeLimit(.minutes(1))) @available(macOS 9999, *) + func concurrentEnqueueFromManyThreadsRunsEveryJob() { + let producers = 6 + let perProducer = 200 + let target = producers * perProducer + let ran = ManagedUnfairLock(initialState: 0) + onThread { + let executor = StackBoundRunLoopExecutor.current() + CFRunLoopPerformBlock(CFRunLoopGetCurrent(), CFRunLoopMode.defaultMode.rawValue) { + let priorities: [TaskPriority] = [.high, .medium, .low, .background, .high, .medium] + DispatchQueue.global(qos: .userInteractive).async { + DispatchQueue.concurrentPerform(iterations: producers) { p in + let priority = priorities[p % priorities.count] + for _ in 0..(initialState: 0) + onThread { + let executor = StackBoundRunLoopExecutor.current() + func hop() { + var n = 0 + count.withLockUnchecked { n = $0 + 1; $0 = n } + if n < hops { + executor.enqueue(makeJob(priority: .medium) { hop() }) + } else { + executor.stop() + } + } + CFRunLoopPerformBlock(CFRunLoopGetCurrent(), CFRunLoopMode.defaultMode.rawValue) { + executor.enqueue(makeJob(priority: .medium) { hop() }) + } + CFRunLoopWakeUp(CFRunLoopGetCurrent()) + try! executor.run() + } + #expect(count.withLockUnchecked { $0 } == hops) + } + + /// Jobs drain highest-priority-first (FIFO within a priority); `stop()` unwinds. + @Test(.timeLimit(.minutes(1))) @available(macOS 9999, *) + func serialPriorityOrderingAndStop() { + let order = ManagedUnfairLock<[Int]>(initialState: []) + onThread { + let executor = StackBoundRunLoopExecutor.current() + CFRunLoopPerformBlock(CFRunLoopGetCurrent(), CFRunLoopMode.defaultMode.rawValue) { + func job(_ n: Int, _ priority: TaskPriority) -> ExecutorJob { + makeJob(priority: priority) { order.withLockUnchecked { $0.append(n) } } + } + executor.enqueue(job(3, .medium)) + executor.enqueue(job(5, .low)) + executor.enqueue(job(1, .high)) + executor.enqueue(job(6, .low)) + executor.enqueue(job(4, .medium)) + executor.enqueue(job(2, .high)) + executor.enqueue(makeJob(priority: .low) { + order.withLockUnchecked { $0.append(7) } + executor.stop() + }) + } + CFRunLoopWakeUp(CFRunLoopGetCurrent()) + try! executor.run() + } + #expect(order.withLockUnchecked { $0 } == [1, 2, 3, 4, 5, 6, 7]) + } + + /// `runUntil` unwinds as soon as its predicate turns true after a drain. + @Test(.timeLimit(.minutes(1))) @available(macOS 9999, *) + func runUntilPredicateUnwinds() { + let counter = ManagedUnfairLock(initialState: 0) + onThread { + let executor = StackBoundRunLoopExecutor.current() + CFRunLoopPerformBlock(CFRunLoopGetCurrent(), CFRunLoopMode.defaultMode.rawValue) { + for _ in 0..<5 { + executor.enqueue(makeJob(priority: .medium) { + counter.withLockUnchecked { $0 += 1 } + }) + } + } + CFRunLoopWakeUp(CFRunLoopGetCurrent()) + try! executor.runUntil { counter.withLockUnchecked { $0 } >= 5 } + } + #expect(counter.withLockUnchecked { $0 } == 5) + } + + /// After quiescence the executor re-arms for a fresh timer correctly. + @Test(.timeLimit(.minutes(1))) @available(macOS 9999, *) @available(iOS 16, macOS 13, watchOS 9, tvOS 16, *) + func rearmAfterQuiescence() { + let fires = ManagedUnfairLock(initialState: 0) + onThread { + let executor = StackBoundRunLoopExecutor.current() + CFRunLoopPerformBlock(CFRunLoopGetCurrent(), CFRunLoopMode.defaultMode.rawValue) { + executor.enqueue(makeJob(priority: .medium) { + fires.withLockUnchecked { $0 += 1 } + executor.enqueue(makeJob(priority: .medium) { + fires.withLockUnchecked { $0 += 1 } + executor.stop() + }, after: .milliseconds(30), clock: .continuous) + }, after: .milliseconds(30), clock: .continuous) + } + CFRunLoopWakeUp(CFRunLoopGetCurrent()) + try! executor.run() + } + #expect(fires.withLockUnchecked { $0 } == 2) + } + + // MARK: Timer tests + + /// A continuous-clock delayed job fires no earlier than its deadline. + @Test(.timeLimit(.minutes(1))) @available(macOS 9999, *) @available(iOS 16, macOS 13, watchOS 9, tvOS 16, *) + func scheduledJobFiresAfterDelay() { + let fired = ManagedUnfairLock(initialState: false) + let elapsed = ManagedUnfairLock(initialState: nil) + onThread { + let executor = StackBoundRunLoopExecutor.current() + let start = ContinuousClock.now + CFRunLoopPerformBlock(CFRunLoopGetCurrent(), CFRunLoopMode.defaultMode.rawValue) { + let job = makeJob(priority: .medium) { + fired.withLockUnchecked { $0 = true } + elapsed.withLockUnchecked { $0 = start.duration(to: ContinuousClock.now) } + executor.stop() + } + executor.enqueue(job, after: .milliseconds(50), clock: .continuous) + } + CFRunLoopWakeUp(CFRunLoopGetCurrent()) + try! executor.run() + } + #expect(fired.withLockUnchecked { $0 }) + if let actual = elapsed.withLockUnchecked({ $0 }) { + #expect(actual >= .milliseconds(50)) + #expect(actual < .milliseconds(650)) + } + } + + /// Timers inserted out of deadline order fire in deadline order (re-arm path). + @Test(.timeLimit(.minutes(1))) @available(macOS 9999, *) @available(iOS 16, macOS 13, watchOS 9, tvOS 16, *) + func multipleTimersFireInDeadlineOrder() { + let order = ManagedUnfairLock<[Int]>(initialState: []) + onThread { + let executor = StackBoundRunLoopExecutor.current() + CFRunLoopPerformBlock(CFRunLoopGetCurrent(), CFRunLoopMode.defaultMode.rawValue) { + func fire(_ n: Int, last: Bool = false) -> ExecutorJob { + makeJob(priority: .medium) { + order.withLockUnchecked { $0.append(n) } + if last { executor.stop() } + } + } + executor.enqueue(fire(90, last: true), after: .milliseconds(90), clock: .continuous) + executor.enqueue(fire(30), after: .milliseconds(30), clock: .continuous) + executor.enqueue(fire(60), after: .milliseconds(60), clock: .continuous) + } + CFRunLoopWakeUp(CFRunLoopGetCurrent()) + try! executor.run() + } + #expect(order.withLockUnchecked { $0 } == [30, 60, 90]) + } + + /// The suspending-clock domain works (uptime deadline). + @Test(.timeLimit(.minutes(1))) @available(macOS 9999, *) @available(iOS 16, macOS 13, watchOS 9, tvOS 16, *) + func suspendingClockTimerFires() { + let fired = ManagedUnfairLock(initialState: false) + onThread { + let executor = StackBoundRunLoopExecutor.current() + CFRunLoopPerformBlock(CFRunLoopGetCurrent(), CFRunLoopMode.defaultMode.rawValue) { + executor.enqueue(makeJob(priority: .medium) { + fired.withLockUnchecked { $0 = true } + executor.stop() + }, after: .milliseconds(30), clock: .suspending) + } + CFRunLoopWakeUp(CFRunLoopGetCurrent()) + try! executor.run() + } + #expect(fired.withLockUnchecked { $0 }) + } + + /// The wall-clock domain works. + @Test(.timeLimit(.minutes(1))) @available(macOS 9999, *) @available(iOS 16, macOS 13, watchOS 9, tvOS 16, *) + func wallClockTimerFires() { + let fired = ManagedUnfairLock(initialState: false) + onThread { + let executor = StackBoundRunLoopExecutor.current() + CFRunLoopPerformBlock(CFRunLoopGetCurrent(), CFRunLoopMode.defaultMode.rawValue) { + executor.enqueue(makeJob(priority: .medium) { + fired.withLockUnchecked { $0 = true } + executor.stop() + }, after: .milliseconds(30), clock: .walltime) + } + CFRunLoopWakeUp(CFRunLoopGetCurrent()) + try! executor.run() + } + #expect(fired.withLockUnchecked { $0 }) + } +} + +#endif // canImport(Darwin) diff --git a/Tests/TetraRunLoopConcurrencyTests/TestSupport.swift b/Tests/TetraRunLoopConcurrencyTests/TestSupport.swift new file mode 100644 index 0000000..b927989 --- /dev/null +++ b/Tests/TetraRunLoopConcurrencyTests/TestSupport.swift @@ -0,0 +1,15 @@ +import Darwin +import Dispatch +import Foundation +@testable import TetraRunLoopConcurrency +@_spi(ExperimentalScheduling) @_spi(ConcurrencyExecutors) @_spi(ExperimentalCustomExecutors) import _Concurrency + +@available(macOS 9999, *) +func makeJob(priority: TaskPriority = .medium, _ body: @escaping () -> Void) -> ExecutorJob { + _swift_createJobForTestingOnly(priority: priority, body) +} + +func onThread(_ body: @escaping @Sendable () -> Void) { + let done = DispatchSemaphore(value: 0) + let t = Thread { body(); done.signal() }; t.stackSize = 1 << 22; t.start(); done.wait() +} From 3180e74e9d21a0c9ee6dd113c306642b5f9e5d22 Mon Sep 17 00:00:00 2001 From: pbk Date: Tue, 7 Jul 2026 13:53:41 +0900 Subject: [PATCH 63/63] Align KQScheduler drain/timer protocol with the reference engine MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Syscall and protocol work on the kqueue run-loop engine, ported from swift-platform-executors' StackBoundRunLoopExecutor2: * Drop the per-lane thread QoS floor in the drain (platform convention: effective priority is thread base + producer overrides, as in libdispatch runloop queues; jobs are never demoted below the thread's base). Removes up to 6 __bsdthread_ctl traps per pump pass; background chain throughput improved ~3x in A/B measurement. * Producer-arms timer protocol: enqueueTimer arms EVFILT_TIMER directly under the timer lock (armIfEarlierLocked) — no EVFILT_USER wake per timer enqueue, and the inPump owning-thread special case is gone. * Merge armed deadlines into the TimerState lock (heaps + armed + FIFO sequence behind one createCheckedStateLock), replacing the unlocked owning-thread-only armed record. * timerCount fast path: skip the fire/arm passes (no lock, no clock reads) when no timers exist; clock reads are lazy per non-empty heap. * Move fired timer jobs into the ready lanes outside the timer lock via a reused dueBuffer, with thread:nil so the pump installs no override for jobs it is about to drain itself. * Equal-deadline timers now pop in enqueue (FIFO) order via a sequence tie-break in TimestampJob. * Priority->lane mapping uses the reference's inclusive band boundaries (>= 33/25/21/17); named priorities are unchanged. * Wrap the drain pass in an explicit autoreleasepool (defensive for the bare-thread run loops on older OS floors). * Move the capped lane drain from SlicedJobQueue.runBatch into the engine (drainReadyJobs/processLane), leaving SlicedJobQueue as a pure lanes+boost container; inline ThreeElement/FiveElement replace the rawLayout arrays. Behavior pins in DrainSyscallDisciplineTests: QoS discipline, mid-pump re-arm without producer wake, autorelease draining per pass, FIFO tie-break, and the lane band mapping. Co-Authored-By: Claude Fable 5 --- Package.swift | 1 + Sources/CriticalSection/BackportedCell.swift | 5 + .../FixedSizedElement.swift | 103 +++++++++ .../TetraRunLoopConcurrency/KQScheduler.swift | 214 ++++++++++++------ .../KQueueSelector.swift | 3 +- .../SlicedJobQueue.swift | 159 +++++-------- .../StackBoundRunLoopExecutor.swift | 4 +- .../DrainSyscallDisciplineTests.swift | 189 ++++++++++++++++ 8 files changed, 504 insertions(+), 174 deletions(-) create mode 100644 Sources/TetraRunLoopConcurrency/FixedSizedElement.swift create mode 100644 Tests/TetraRunLoopConcurrencyTests/DrainSyscallDisciplineTests.swift diff --git a/Package.swift b/Package.swift index c77d965..c30df1f 100644 --- a/Package.swift +++ b/Package.swift @@ -155,6 +155,7 @@ let package = Package( swiftSettings: [ .swiftLanguageMode(.v6), .unsafeFlags(["-Xfrontend", "-disable-availability-checking"]), + ] ), ], diff --git a/Sources/CriticalSection/BackportedCell.swift b/Sources/CriticalSection/BackportedCell.swift index 1ef6649..0f64790 100644 --- a/Sources/CriticalSection/BackportedCell.swift +++ b/Sources/CriticalSection/BackportedCell.swift @@ -57,6 +57,11 @@ package struct FiveArray:~Copyable { } +extension FiveArray: @unchecked Sendable where T:~Copyable, T:Sendable {} +extension ThreeArray: @unchecked Sendable where T:~Copyable, T:Sendable {} + + + @_rawLayout(likeArrayOf: T, count: 3, movesAsLike) package struct ThreeArray:~Copyable { @_transparent diff --git a/Sources/TetraRunLoopConcurrency/FixedSizedElement.swift b/Sources/TetraRunLoopConcurrency/FixedSizedElement.swift new file mode 100644 index 0000000..309b08a --- /dev/null +++ b/Sources/TetraRunLoopConcurrency/FixedSizedElement.swift @@ -0,0 +1,103 @@ +// +// ThreeElement.swift +// Tetra +// +// Created by 박병관 on 7/7/26. +// + + +struct ThreeElement:~Copyable { + var e0: T + var e1: T + var e2: T + + init(_ body: (Int) -> T) { + e0 = body(0) + e1 = body(1) + e2 = body(2) + } + + + + init(repeating value: T) where T:Copyable{ + self.init { _ in value } + } + + var indices: Range { 0..<3 } + + subscript(i: Int) -> T { + _read { + switch i { + case 0: yield e0 + case 1: yield e1 + case 2: yield e2 + default: preconditionFailure("ThreeElement index out of bounds: \(i)") + } + } + _modify { + switch i { + case 0: yield &e0 + case 1: yield &e1 + case 2: yield &e2 + default: preconditionFailure("ThreeElement index out of bounds: \(i)") + } + } + } +} + +extension ThreeElement: Sendable where T: Sendable {} +extension ThreeElement:Copyable where T:Copyable {} +extension ThreeElement:BitwiseCopyable where T:BitwiseCopyable {} + +/// Five inline elements, indexable `0...4` — the QoS ready lanes. Supports noncopyable +/// `T` (the per-lane MPSC queues and QoS-override atomics). +internal struct FiveElement: ~Copyable { + var e0: T + var e1: T + var e2: T + var e3: T + var e4: T + + public init(_ body: (Int) -> T) { + e0 = body(0) + e1 = body(1) + e2 = body(2) + e3 = body(3) + e4 = body(4) + } + init(repeating value: T) where T:Copyable{ + self.init { _ in value } + } + + public var indices: Range { 0..<5 } + public var startIndex: Int { 0 } + public var endIndex: Int { 5 } + + public subscript(i: Int) -> T { + _read { + switch i { + case 0: yield e0 + case 1: yield e1 + case 2: yield e2 + case 3: yield e3 + case 4: yield e4 + default: preconditionFailure("FiveElement index out of bounds: \(i)") + } + } + _modify { + switch i { + case 0: yield &e0 + case 1: yield &e1 + case 2: yield &e2 + case 3: yield &e3 + case 4: yield &e4 + default: preconditionFailure("FiveElement index out of bounds: \(i)") + } + } + + } +} + +extension FiveElement: Sendable where T: Sendable & ~Copyable {} +extension FiveElement: Copyable where T:Copyable {} +extension FiveElement: BitwiseCopyable where T:BitwiseCopyable {} diff --git a/Sources/TetraRunLoopConcurrency/KQScheduler.swift b/Sources/TetraRunLoopConcurrency/KQScheduler.swift index 8384427..4478bee 100644 --- a/Sources/TetraRunLoopConcurrency/KQScheduler.swift +++ b/Sources/TetraRunLoopConcurrency/KQScheduler.swift @@ -18,6 +18,7 @@ // #if canImport(Darwin) +import Atomics import Darwin import Dispatch import Foundation @@ -37,32 +38,55 @@ final class KQScheduler: @unchecked Sendable { private let facade: StackBoundRunLoopExecutor private let serial: UnownedSerialExecutor - /// Optional task-executor reference threaded through to `runBatch`. + /// Optional task-executor reference threaded through to `processLane`. private let taskRefOrNil: Builtin.Executor? let cfRunLoop: CFRunLoop let thread: pthread_t /// Raw kqueue descriptor, valid from init; exposed for cross-thread wakeups. let kqueueFD: Int32 - /// The 5-QoS ready lanes. Drained (only) by `runBatch` on the owning thread. + /// The 5-QoS ready lanes. Drained (only) by `drainReadyJobs` on the owning thread. let ready = SlicedJobQueue(cacheSize: 2048) - /// Timer state, MOVED here from `SlicedJobQueue` (DESIGN A). One min-heap per - /// clock domain — continuous / suspending / wall — behind a single state lock. - let delayedJobs: some UnfairStateLock>> = - createCheckedStateLock(checkedState: .init(repeating: .init(), count: 3)) - - /// The deadline currently armed on the kqueue per clock domain (`nil` = unarmed). - /// OWNING-THREAD-ONLY — mutated only by the pump (`handleReadable` and its - /// `fireDueTimers`/`armNextDeadlines`), never by producers — so it needs no lock - /// (and can therefore be the noncopyable `ThreeArray`, which `UnfairStateLock`'s - /// copyable `State` could not hold). Arming is deferred from producers to the pump; - /// `armNextDeadlines` re-arms a domain only when its heap min is strictly earlier - /// than what is armed, and one-shot fires clear the entry — eliminating the - /// per-pump re-arm `kevent64` thrash of the previous unconditional arming. - private nonisolated(unsafe) var armed = ThreeArray(initializingWith: { - while !$0.isFull { $0.append(nil) } - }) + /// Pending-timer heaps and the armed-deadline record, per clock domain, together + /// behind one lock — the reference engine's `TimerState` shape. Producers arm the + /// kqueue `EVFILT_TIMER` directly under this lock (`kevent64` is thread-safe), so + /// there is no producer/owning-thread split-brain over the armed idents and no + /// `EVFILT_USER` wakeup per timer enqueue. + struct TimerState { + /// Monotonic insertion counter — the FIFO tie-break for equal deadlines. + var sequence: UInt64 = 0 + /// One min-heap per clock domain — continuous / suspending / wall. MOVED here + /// from `SlicedJobQueue` (DESIGN A). + var heaps: ThreeElement> = .init(repeating: .init()) + /// The deadline currently armed on the kqueue per domain (`nil` = unarmed). + /// `armIfEarlierLocked` re-arms a domain only when the candidate is strictly + /// earlier than what is armed, and one-shot fires clear the entry — + /// eliminating the per-pump re-arm `kevent64` thrash of unconditional arming. + var armed: ThreeElement = .init(repeating: nil) + } + + let timers: some UnfairStateLock = + createCheckedStateLock(checkedState: TimerState()) + + /// Timers currently in the heaps. Lets the drain pump skip the whole timer pass + /// (no lock, no clock reads) when zero — the common, timer-free case. + private let timerCount = ManagedAtomic(0) + + /// Owning-thread-only scratch for due jobs, so fired timer jobs are moved into + /// the ready lanes OUTSIDE the timer lock; reused so steady state allocates nothing. + private nonisolated(unsafe) var dueBuffer = ContiguousArray() + + /// Arm `index` if `candidate` is strictly earlier than what is armed (or nothing + /// is). Caller holds `timers`. Reads the domain clock only when it actually arms. + private func armIfEarlierLocked(_ state: inout TimerState, index: SlicedJobQueue.ClockIndex, candidate: Timestamp) { + let raw = index.rawValue + if let armedStamp = state.armed[raw], candidate.deadline >= armedStamp.deadline { return } + KQueueSelector.armTimer(fileDescriptor: kqueueFD, index: index, + target: candidate.target, leeway: candidate.leeway, + now: KQueueSelector.now(index: index)) + state.armed[raw] = candidate + } /// True while a drain is pending/imminent; producers elide the wakeup when they /// lose the false→true race. RMW-only (see `handleReadable`). @@ -79,7 +103,7 @@ final class KQScheduler: @unchecked Sendable { /// - Parameter taskRef: the facade's task-executor reference (its /// `asUnownedTaskExecutor()._executor`) on the iOS-18 `TaskExecutor` path, or - /// `nil` on the iOS-13 `SerialExecutor`-only path. When non-nil, `runBatch` + /// `nil` on the iOS-13 `SerialExecutor`-only path. When non-nil, `processLane` /// runs jobs via `runSynchronously(isolatedTo:taskExecutor:)` so a task's /// preferred task executor is respected (Task 5). init(facade: StackBoundRunLoopExecutor, serial: UnownedSerialExecutor, @@ -135,26 +159,24 @@ final class KQScheduler: @unchecked Sendable { } } - /// Inserts a delayed job and, if it became a strictly-earlier deadline for its - /// domain, arms that domain's kqueue timer and posts a wakeup so the pump re-arms. - /// The timestamp computation was MOVED here from `SlicedJobQueue.enqueue(_:after:...)`. + /// Inserts a delayed job and, if it is a strictly-earlier deadline for its domain, + /// arms that domain's kqueue timer right here, under the lock — no `EVFILT_USER` + /// wakeup round trip (an already-due deadline arms `0` and fires immediately, so + /// the kernel itself wakes the pump). The timestamp computation was MOVED here + /// from `SlicedJobQueue.enqueue(_:after:...)`. @available(iOS 16, macOS 13, watchOS 9, tvOS 16, visionOS 1, *) func enqueueTimer(_ job: UnownedJob, after delay: Duration, tolerance: Duration?, index: SlicedJobQueue.ClockIndex) { guard phase == .live else { preconditionFailure("StackBoundRunLoopExecutor.enqueue(_:after:...) called while shutting down.") } let timestamp = Self.timestamp(after: delay, tolerance: tolerance, index: index) - let becameNewMin: Bool = delayedJobs.withLock { heaps in - let raw = index.rawValue - let oldStamp = heaps[raw].min?.timestamp - heaps[raw].insert(TimestampJob(job: job, timestamp: timestamp)) - let newStamp = heaps[raw].min?.timestamp - return oldStamp != newStamp - } - if becameNewMin { - // Arming is owning-thread-only; just wake the pump, which re-arms via - // `armNextDeadlines`. (No producer-side `kevent64` / `armed` mutation.) - KQueueSelector.wakeup(fileDescriptor: kqueueFD) + timerCount.wrappingIncrement(ordering: .releasing) + timers.withLock { state in + state.sequence &+= 1 + state.heaps[index.rawValue].insert( + TimestampJob(job: job, sequence: state.sequence, timestamp: timestamp) + ) + armIfEarlierLocked(&state, index: index, candidate: timestamp) } } @@ -218,15 +240,22 @@ final class KQScheduler: @unchecked Sendable { } deferredReadable = false pendingJobPop.store(true, ordering: .relaxed) - let fired = KQueueSelector.drainEvents(fileDescriptor: kqueueFD) // consume wake + timer fires - // A one-shot EVFILT_TIMER that fired is now disarmed in the kernel; clear our - // record so `armNextDeadlines` re-arms the domain's next deadline. - if fired.continuous { armed[0] = nil } - if fired.suspending { armed[1] = nil } - if fired.wall { armed[2] = nil } - fireDueTimers() // pop due timers -> ready lanes - ready.runBatch(executor: serial, taskRef: taskRefOrNil) // drain the 5 ready lanes only - armNextDeadlines() // arm EVFILT_TIMER from delayedJobs mins + // The explicit pool bounds job-autoreleased objects to this pass — a bare + // thread's CFRunLoop is not guaranteed to push one of its own on every OS + // Tetra supports. + autoreleasepool { + let fired = KQueueSelector.drainEvents(fileDescriptor: kqueueFD) // consume wake + timer fires + let hadTimers = timerCount.load(ordering: .acquiring) > 0 + // Fire the due pass whenever timers exist OR the kqueue reported a timer + // firing (so a fire is never dropped even if the count just changed). + if hadTimers || fired.continuous || fired.suspending || fired.wall { + fireDueTimers(fired) // pop due timers -> ready lanes + } + drainReadyJobs() // drain the 5 ready lanes only + if timerCount.load(ordering: .acquiring) > 0 { + armNextDeadlines() // arm EVFILT_TIMER from heap mins + } + } var refire = !readyLanesEmpty() if !refire { _ = pendingJobPop.exchange(false, ordering: .acquiringAndReleasing) @@ -253,36 +282,90 @@ final class KQScheduler: @unchecked Sendable { CFFileDescriptorEnableCallBacks(fd, kCFFileDescriptorReadCallBack) } - /// Pop due entries (per domain, `timestamp.target <= now`) and feed them to the - /// ready lanes. - private func fireDueTimers() { - delayedJobs.withLockUnchecked { heaps in + /// Pop due entries (per domain, `timestamp.target <= now`) into `dueBuffer` and + /// feed them to the ready lanes OUTSIDE the lock, so no MPSC push (or override + /// syscall) ever happens while producers wait on `timers`. Also clears the armed + /// record for domains the kqueue reported as fired — a one-shot EVFILT_TIMER that + /// fired is disarmed in the kernel, so the next arm pass must re-arm that domain. + /// Clock reads happen lazily, only for non-empty heaps. + private func fireDueTimers(_ fired: KQueueSelector.FiredDomains) { + dueBuffer.removeAll(keepingCapacity: true) + var firedCount = 0 + timers.withLockUnchecked { state in + if fired.continuous { state.armed[0] = nil } + if fired.suspending { state.armed[1] = nil } + if fired.wall { state.armed[2] = nil } for raw in 0..<3 { + guard state.heaps[raw].min != nil else { continue } let now = KQueueSelector.now(index: SlicedJobQueue.ClockIndex(rawValue: raw)!) var popped = false - while let box = heaps[raw].min, box.timestamp.target <= now { - heaps[raw].removeMin(); ready.enqueue(box.job, thread); popped = true + while let box = state.heaps[raw].min, box.timestamp.target <= now { + state.heaps[raw].removeMin() + dueBuffer.append(box.job) + firedCount &+= 1 + popped = true } - // The armed min was just consumed — clear so `armNextDeadlines` re-arms + // The armed min was just consumed — clear so the next arm pass re-arms // the new min (guards against a stale `armed` skipping the next deadline). - if popped { armed[raw] = nil } + if popped { state.armed[raw] = nil } } } + if firedCount > 0 { + timerCount.wrappingDecrement(by: firedCount, ordering: .releasing) + } + // `thread: nil`: the pump is about to drain these itself — no boost install. + for job in dueBuffer { + ready.enqueue(job, nil) + } + dueBuffer.removeAll(keepingCapacity: true) } /// Arm the earliest not-yet-due deadline per domain — but only when it is strictly - /// earlier than what is already armed (`armed[raw]`), so a steady state with an - /// unchanged min issues no `kevent64` per pump pass. + /// earlier than what is already armed, so a steady state with an unchanged min + /// issues no `kevent64` per pump pass. private func armNextDeadlines() { - delayedJobs.withLockUnchecked { heaps in + timers.withLockUnchecked { state in for raw in 0..<3 { let index = SlicedJobQueue.ClockIndex(rawValue: raw)! - guard let stamp = heaps[raw].min?.timestamp else { continue } - if let armedStamp = armed[raw], armedStamp.deadline <= stamp.deadline { continue } - KQueueSelector.armTimer(fileDescriptor: kqueueFD, index: index, - target: stamp.target, leeway: stamp.leeway, - now: KQueueSelector.now(index: index)) - armed[raw] = stamp + guard let stamp = state.heaps[raw].min?.timestamp else { continue } + armIfEarlierLocked(&state, index: index, candidate: stamp) + } + } + } + + /// Drains the 5 ready lanes on the owning thread, highest-priority-first. + /// + /// Two disciplines ported from the reference engine: + /// * I4a (QoS-once): the per-lane `pthread_override` boosts are NOT ended + /// here. They are held across drains and released ONCE by `endAllBoosts()` + /// on the idle path (no refire) / at teardown — avoiding per-drain start/end + /// syscall thrash and a mid-busy priority drop. The drain never alters the + /// owning thread's own QoS: effective priority is thread base + overrides, + /// matching libdispatch's runloop-queue discipline — low-QoS jobs are not + /// demoted below the thread's base. + /// * I4b (fairness cap): each lane's dequeue loop is capped at + /// `getDrainIterations(queueIndex:)`, so a high-priority flood cannot starve + /// the run loop's other CFRunLoop sources. A capped lane may leave jobs; the + /// pump re-checks `readyLanesEmpty()` and re-fires until empty. + private func drainReadyJobs() { + for index in 0..<5 { + processLane(index) + } + } + + private func processLane(_ index: Int) { + let iterations = getDrainIterations(queueIndex: index) + var count = 0 + if #available(macOS 15.0, iOS 18.0, watchOS 11.0, tvOS 18.0, visionOS 2.0, *), let t = taskRefOrNil { + let taskExecutor = UnownedTaskExecutor(t) + while count < iterations, let j = ready.jobs[index].dequeue() { + j.runSynchronously(isolatedTo: serial, taskExecutor: taskExecutor) + count &+= 1 + } + } else { + while count < iterations, let j = ready.jobs[index].dequeue() { + j.runSynchronously(on: serial) + count &+= 1 } } } @@ -303,9 +386,9 @@ final class KQScheduler: @unchecked Sendable { /// any not-yet-due timers, and transitions to `.dead`. func finishAndDie() { while true { - drainReadyLanes() + drainReadyJobs() if facade.producerGate.load(ordering: .acquiring) == 0 { - drainReadyLanes() + drainReadyJobs() if readyLanesEmpty() { // Final teardown release of any QoS overrides (I4a backstop before // the SlicedJobQueue.deinit backstop). @@ -320,18 +403,17 @@ final class KQScheduler: @unchecked Sendable { } } - private func drainReadyLanes() { - ready.runBatch(executor: serial, taskRef: taskRefOrNil) - } /// Not-yet-due timers are dropped at unwind (lifecycle contract). The kqueue /// timers themselves die when the descriptor is invalidated in the frame's defer. private func dropPendingTimers() { - delayedJobs.withLockUnchecked { heaps in + timers.withLockUnchecked { state in for raw in 0..<3 { - heaps[raw] = .init() + state.heaps[raw] = .init() + state.armed[raw] = nil } } + timerCount.store(0, ordering: .releasing) } // MARK: Run-loop source diff --git a/Sources/TetraRunLoopConcurrency/KQueueSelector.swift b/Sources/TetraRunLoopConcurrency/KQueueSelector.swift index 35637d2..bbcab06 100644 --- a/Sources/TetraRunLoopConcurrency/KQueueSelector.swift +++ b/Sources/TetraRunLoopConcurrency/KQueueSelector.swift @@ -135,8 +135,7 @@ enum KQueueSelector { // MARK: Clock - /// The domain's current instant in the same units that `SlicedJobQueue.Timestamp.target` - /// is stored in, matching `SlicedJobQueue.runBatch`'s `times` array. + /// The domain's current instant in the same units that `Timestamp.target` is stored in. static func now(index: SlicedJobQueue.ClockIndex) -> UInt64 { switch index { case .continuous: diff --git a/Sources/TetraRunLoopConcurrency/SlicedJobQueue.swift b/Sources/TetraRunLoopConcurrency/SlicedJobQueue.swift index 1288e98..5ba3eb3 100644 --- a/Sources/TetraRunLoopConcurrency/SlicedJobQueue.swift +++ b/Sources/TetraRunLoopConcurrency/SlicedJobQueue.swift @@ -15,10 +15,10 @@ import Builtin internal struct SlicedJobQueue: ~Copyable, Sendable { + + let jobs:FiveElement<__MPSCQueue> nonisolated(unsafe) - let jobs:FiveArray<__MPSCQueue> - nonisolated(unsafe) - let boost:FiveArray> + let boost:FiveElement> let cache1:__MPSCQueue.NodeCache // let cache2:__MPSCQueue.NodeCache @@ -28,90 +28,14 @@ internal struct SlicedJobQueue: ~Copyable, Sendable { // let cache2 = __MPSCQueue.NodeCache(size: cacheSize / 2) self.cache1 = cache1 // self.cache2 = cache2 - boost = .init(initializingWith: { - while !$0.isFull { - $0.append(.init(nil)) - } + boost = .init({ _ in + .init(nil) }) - jobs = .init(initializingWith: { - while !$0.isFull { - $0.append(.init(cache: cache1)) - } + jobs = .init({ _ in + .init(cache: cache1) }) } - /// Drains the 5 ready lanes on the owning thread, highest-priority-first. - /// - /// Two disciplines ported from the reference engine: - /// * I4a (QoS-once): the per-lane `pthread_override` boosts are NOT ended - /// here. They are held across drains and released ONCE by the engine's - /// `endAllBoosts()` on the idle path (no refire) / at teardown — avoiding - /// per-drain start/end syscall thrash and a mid-busy priority drop. This - /// method only sets the *thread* QoS floor (`pthread_set_qos_class_self_np`) - /// per lane so the owning thread runs each lane's jobs at that lane's class. - /// * I4b (fairness cap): each lane's dequeue loop is capped at - /// `getDrainIterations(queueIndex:)`, so a high-priority flood cannot starve - /// the run loop's other CFRunLoop sources. A capped lane may leave jobs; the - /// engine pump re-checks `readyLanesEmpty()` and re-fires until empty. - internal func runBatch( - executor:UnownedSerialExecutor, - taskRef: Builtin.Executor? = nil - ) { - - let qos:DispatchQoS - do { - var _qos = QOS_CLASS_UNSPECIFIED - var priority = Int32(0) - pthread_get_qos_class_np(pthread_self(), &_qos, &priority) - qos = .init(qosClass: .init(rawValue: _qos)!, relativePriority: .init(priority)) - } - var currentQos = qos.qosClass.rawValue - defer { - pthread_set_qos_class_self_np(qos.qosClass.rawValue, .init(qos.relativePriority)) - } - do { - - for i in 0..<5 { - let laneQos = switch i { - case 0: - QOS_CLASS_USER_INTERACTIVE - case 1: - QOS_CLASS_USER_INITIATED - case 2: - QOS_CLASS_DEFAULT - case 3: - QOS_CLASS_UTILITY - case 4: - fallthrough - default: - QOS_CLASS_BACKGROUND - } - let iterations = getDrainIterations(queueIndex: i) - var count = 0 - if #available(macOS 15.0, iOS 18.0, watchOS 11.0, tvOS 18.0, visionOS 2.0, *), let t = taskRef { - let taskExecutor = UnownedTaskExecutor(t) - while count < iterations, let j = self.jobs[i].dequeue() { - if count == 0, laneQos != currentQos { - pthread_set_qos_class_self_np(laneQos, 0) - currentQos = laneQos - } - j.runSynchronously(isolatedTo: executor, taskExecutor: taskExecutor) - count &+= 1 - } - } else { - while count < iterations, let j = self.jobs[i].dequeue() { - if count == 0, laneQos != currentQos { - pthread_set_qos_class_self_np(laneQos, 0) - currentQos = laneQos - } - j.runSynchronously(on: executor) - count &+= 1 - } - } - } - } - } - /// Release every active QoS override. Called by the engine ONLY when the busy /// period ends (no refire) or at teardown — NOT per drain. Holding overrides /// across drains mirrors libdispatch's runloop-queue discipline @@ -126,7 +50,10 @@ internal struct SlicedJobQueue: ~Copyable, Sendable { } } - nonisolated func enqueue(_ job:UnownedJob, _ thread:pthread_t) { + /// `thread == nil` skips the QoS-override install: the pump passes nil when moving + /// its own fired timer jobs into the lanes (it is about to drain them itself, so + /// boosting its own thread would only buy a wasted syscall pair). + nonisolated func enqueue(_ job:UnownedJob, _ thread:pthread_t?) { if #available(macOS 14.0, iOS 17.0, watchOS 10.0, tvOS 17.0, *){ let priority = TaskPriority(job.priority) let index = priority?.jobQueueIndex ?? 4 @@ -134,7 +61,7 @@ internal struct SlicedJobQueue: ~Copyable, Sendable { // Install a QoS override for this lane only when none is active yet — the // hot path (a burst of same-priority jobs) then costs one acquiring load and // no syscall. (Was inverted `!= nil`, which never installed an override.) - if boost[index].load(ordering: .acquiring) == nil { + if let thread, boost[index].load(ordering: .acquiring) == nil { let qos = switch index { case 0: QOS_CLASS_USER_INTERACTIVE @@ -345,25 +272,28 @@ internal struct SlicedJobQueue: ~Copyable, Sendable { } +/// A delayed job, ordered by fire deadline; `sequence` breaks deadline ties so +/// equal-deadline timers pop in enqueue (FIFO) order. struct TimestampJob: Comparable { - + static func < (lhs: Self, rhs: Self) -> Bool { - lhs.timestamp.deadline < rhs.timestamp.deadline - } - - static func > (lhs: Self, rhs: Self) -> Bool { - lhs.timestamp.deadline > rhs.timestamp.deadline + if lhs.timestamp.deadline != rhs.timestamp.deadline { + return lhs.timestamp.deadline < rhs.timestamp.deadline + } + return lhs.sequence < rhs.sequence } - + static func == (lhs: Self, rhs: Self) -> Bool { - lhs.timestamp == rhs.timestamp + lhs.sequence == rhs.sequence } - + let timestamp:Timestamp + let sequence:UInt64 let job:UnownedJob - - init(job:consuming UnownedJob, timestamp:Timestamp) { + + init(job:consuming UnownedJob, sequence:UInt64, timestamp:Timestamp) { self.job = job + self.sequence = sequence self.timestamp = timestamp } } @@ -402,22 +332,25 @@ struct Timestamp:BitwiseCopyable, Hashable, Sendable, Copyable { } extension TaskPriority { - + + /// Reference-engine band mapping (`>=` boundaries): a priority lands in the lane + /// of the highest named priority it meets or exceeds. `.userInteractive` (33) is + /// spelled via rawValue — the named stdlib symbol is newer than Tetra's iOS 13 floor. var jobQueueIndex:Int { - - if self > .high { + + if self >= .init(rawValue: 33) { 0 - } else if self > .medium { + } else if self >= .high { 1 - } else if self > .low { + } else if self >= .medium { 2 - } else if self > .background { + } else if self >= .low { 3 } else { 4 } } - + } /// Per-lane drain cap (I4b fairness). Ported from the reference engine's exact @@ -500,7 +433,25 @@ class Backing { } func dispatch() { - store.runBatch(executor: serialExecutor.unsafelyUnwrapped, taskRef: taskRef) + // Same capped lane drain as the engine's `drainReadyJobs`/`processLane`, + // inlined: this experimental CFRunLoopSource backing has no engine to host it. + let executor = serialExecutor.unsafelyUnwrapped + for index in 0..<5 { + let iterations = getDrainIterations(queueIndex: index) + var count = 0 + if #available(macOS 15.0, iOS 18.0, watchOS 11.0, tvOS 18.0, visionOS 2.0, *), let t = taskRef { + let taskExecutor = UnownedTaskExecutor(t) + while count < iterations, let j = store.jobs[index].dequeue() { + j.runSynchronously(isolatedTo: executor, taskExecutor: taskExecutor) + count &+= 1 + } + } else { + while count < iterations, let j = store.jobs[index].dequeue() { + j.runSynchronously(on: executor) + count &+= 1 + } + } + } } func schedule(_ runloop:CFRunLoop, _ mode:CFRunLoopMode) { diff --git a/Sources/TetraRunLoopConcurrency/StackBoundRunLoopExecutor.swift b/Sources/TetraRunLoopConcurrency/StackBoundRunLoopExecutor.swift index 76498e1..05d7291 100644 --- a/Sources/TetraRunLoopConcurrency/StackBoundRunLoopExecutor.swift +++ b/Sources/TetraRunLoopConcurrency/StackBoundRunLoopExecutor.swift @@ -320,7 +320,7 @@ public final class StackBoundRunLoopExecutor: SerialExecutor, @unchecked Sendabl // (stack-bound lifetime). // // On the iOS-18 path this facade is a `TaskExecutor`, so hand the engine a - // task-executor ref; `runBatch` then runs jobs via + // task-executor ref; `processLane` then runs jobs via // `runSynchronously(isolatedTo:taskExecutor:)`. Below iOS 18 the ref stays nil // (the `SerialExecutor`-only `runSynchronously(on:)` path). let taskRef: Builtin.Executor? @@ -413,7 +413,7 @@ public final class StackBoundRunLoopExecutor: SerialExecutor, @unchecked Sendabl // MARK: - Gated executor-protocol conformances (Task 5) // // All four protocols floor at iOS 16 / macOS 13, but this executor supplies a -// task executor to `runBatch` via `runSynchronously(isolatedTo:taskExecutor:)`, +// task executor to `processLane` via `runSynchronously(isolatedTo:taskExecutor:)`, // which is iOS 18 / macOS 15. So the conformances are gated at that higher floor. // `SchedulingExecutor` / `RunLoopExecutor` / `MainExecutor` are SPI (see the // `@_spi(...) import _Concurrency` at the top of this file). diff --git a/Tests/TetraRunLoopConcurrencyTests/DrainSyscallDisciplineTests.swift b/Tests/TetraRunLoopConcurrencyTests/DrainSyscallDisciplineTests.swift new file mode 100644 index 0000000..12f4ce7 --- /dev/null +++ b/Tests/TetraRunLoopConcurrencyTests/DrainSyscallDisciplineTests.swift @@ -0,0 +1,189 @@ +// +// DrainSyscallDisciplineTests.swift +// TetraRunLoopConcurrencyTests +// +// Characterization tests pinning the behaviors at stake in the drain-path +// syscall optimizations: +// +// * QoS discipline (platform convention): `runBatch` never alters the owning +// thread's own QoS — no per-lane demotion while a job runs, and the entry +// class AND relative priority are intact after any drain. Effective priority +// is thread base + producer-installed overrides, as in libdispatch's +// runloop queues and the reference engine. +// * A strictly-earlier timer enqueued from INSIDE a job (owning thread, mid-pump) +// must be armed by the end of that pump pass — it may not wait for the +// previously-armed later deadline. This is what allows `enqueueTimer` to skip +// the producer self-wake on the owning-thread-in-pump path. +// + +#if canImport(Darwin) +import Testing +import Foundation +import CoreFoundation +import Dispatch +import HeapModule +@testable import TetraRunLoopConcurrency +import CriticalSection + +private final class WeakRef: @unchecked Sendable { + weak var value: T? +} +private final class Canary {} + +@Suite +struct DrainSyscallDisciplineTests { + + /// After a mixed-lane drain the owning thread still has its entry QoS class AND + /// relative priority — the drain leaves the thread's own QoS untouched. + @Test(.timeLimit(.minutes(1))) @available(macOS 9999, *) + func threadQoSClassAndRelativePriorityIntactAfterMixedLaneDrain() { + let restored = ManagedUnfairLock<(UInt32, Int32)?>(initialState: nil) + onThread { + pthread_set_qos_class_self_np(QOS_CLASS_USER_INITIATED, -4) + let executor = StackBoundRunLoopExecutor.current() + CFRunLoopPerformBlock(CFRunLoopGetCurrent(), CFRunLoopMode.defaultMode.rawValue) { + // .medium -> DEFAULT lane, .low -> UTILITY lane: both differ from the + // entry class, exercising the lanes that used to demote the thread. + executor.enqueue(makeJob(priority: .medium) { }) + executor.enqueue(makeJob(priority: .low) { executor.stop() }) + } + CFRunLoopWakeUp(CFRunLoopGetCurrent()) + try! executor.run() + var qos = QOS_CLASS_UNSPECIFIED + var relative = Int32(0) + pthread_get_qos_class_np(pthread_self(), &qos, &relative) + restored.withLockUnchecked { $0 = (qos.rawValue, relative) } + } + let result = restored.withLockUnchecked { $0 } + #expect(result?.0 == QOS_CLASS_USER_INITIATED.rawValue) + #expect(result?.1 == -4) + } + + /// Platform-convention QoS discipline (libdispatch runloop queues, the reference + /// engine): the drain never demotes the owning thread below its base QoS — a + /// background-lane job observes the thread's own requested QoS class, not + /// QOS_CLASS_BACKGROUND. Priority-inversion avoidance is the overrides' job. + @Test(.timeLimit(.minutes(1))) @available(macOS 9999, *) + func jobRunsAtThreadBaseQoSNotLaneQoS() { + let observed = ManagedUnfairLock(initialState: nil) + onThread { + pthread_set_qos_class_self_np(QOS_CLASS_USER_INITIATED, 0) + let executor = StackBoundRunLoopExecutor.current() + CFRunLoopPerformBlock(CFRunLoopGetCurrent(), CFRunLoopMode.defaultMode.rawValue) { + executor.enqueue(makeJob(priority: .background) { + var qos = QOS_CLASS_UNSPECIFIED + var relative = Int32(0) + pthread_get_qos_class_np(pthread_self(), &qos, &relative) + observed.withLockUnchecked { $0 = qos.rawValue } + executor.stop() + }) + } + CFRunLoopWakeUp(CFRunLoopGetCurrent()) + try! executor.run() + } + #expect(observed.withLockUnchecked { $0 } == QOS_CLASS_USER_INITIATED.rawValue) + } + + /// A 30ms timer enqueued from inside a job while a 200ms timer is already armed + /// fires near its own deadline (not at the stale 200ms arming), proving the pump + /// re-arms the earlier deadline by the end of the pass without a producer wake. + @Test(.timeLimit(.minutes(1))) @available(macOS 9999, *) @available(iOS 16, macOS 13, watchOS 9, tvOS 16, *) + func earlierTimerEnqueuedMidPumpRearmsWithoutProducerWake() { + let order = ManagedUnfairLock<[Int]>(initialState: []) + let earlyElapsed = ManagedUnfairLock(initialState: nil) + onThread { + let executor = StackBoundRunLoopExecutor.current() + let start = ContinuousClock.now + CFRunLoopPerformBlock(CFRunLoopGetCurrent(), CFRunLoopMode.defaultMode.rawValue) { + executor.enqueue(makeJob(priority: .medium) { + order.withLockUnchecked { $0.append(200) } + executor.stop() + }, after: .milliseconds(200), clock: .continuous) + executor.enqueue(makeJob(priority: .medium) { + // Mid-pump, owning thread: this becomes the strictly-earlier min. + executor.enqueue(makeJob(priority: .medium) { + order.withLockUnchecked { $0.append(30) } + earlyElapsed.withLockUnchecked { $0 = start.duration(to: ContinuousClock.now) } + }, after: .milliseconds(30), clock: .continuous) + }) + } + CFRunLoopWakeUp(CFRunLoopGetCurrent()) + try! executor.run() + } + #expect(order.withLockUnchecked { $0 } == [30, 200]) + if let actual = earlyElapsed.withLockUnchecked({ $0 }) { + #expect(actual >= .milliseconds(30)) + // Firing anywhere near the stale 200ms arming means the re-arm was lost. + #expect(actual < .milliseconds(150)) + } else { + Issue.record("early timer never fired") + } + } + + /// Objects autoreleased by a job must be released by the end of that pump pass — + /// the drain runs inside its own autorelease pool (a bare-thread CFRunLoop pushes + /// none of its own, so without one they pile up until the thread exits). + @Test(.timeLimit(.minutes(1))) @available(macOS 9999, *) @available(iOS 16, macOS 13, watchOS 9, tvOS 16, *) + func autoreleasedObjectsDrainBetweenPumpPasses() { + let stillAlive = ManagedUnfairLock(initialState: nil) + let ref = WeakRef() + onThread { + let executor = StackBoundRunLoopExecutor.current() + CFRunLoopPerformBlock(CFRunLoopGetCurrent(), CFRunLoopMode.defaultMode.rawValue) { + executor.enqueue(makeJob(priority: .medium) { + let canary = Canary() + ref.value = canary + Unmanaged.passRetained(canary).autorelease() + // A later, separate pump pass observes whether the pool drained. + executor.enqueue(makeJob(priority: .medium) { + stillAlive.withLockUnchecked { $0 = ref.value != nil } + executor.stop() + }, after: .milliseconds(50), clock: .continuous) + }) + } + CFRunLoopWakeUp(CFRunLoopGetCurrent()) + try! executor.run() + } + #expect(stillAlive.withLockUnchecked { $0 } == false) + } + + /// Delayed jobs with IDENTICAL deadlines pop in enqueue (FIFO) order — the heap + /// ordering must carry a sequence tie-break, not leave equal keys unordered. + @Test @available(macOS 9999, *) + func equalDeadlineTimerJobsPopInFifoOrder() { + var heap = Heap() + let stamp = Timestamp(target: 1_000, leeway: 0) + var insertion: [UnsafeRawPointer] = [] + for i in 0..<8 { + // Identity-only jobs: never run, deliberately leaked (test process only). + let job = UnownedJob(makeJob(priority: .medium) { }) + insertion.append(unsafeBitCast(job, to: UnsafeRawPointer.self)) + heap.insert(TimestampJob(job: job, sequence: UInt64(i), timestamp: stamp)) + } + var popped: [UnsafeRawPointer] = [] + while let min = heap.popMin() { + popped.append(unsafeBitCast(min.job, to: UnsafeRawPointer.self)) + } + #expect(popped == insertion) + } + + /// Priority→lane mapping uses the reference engine's `>=` band boundaries: a raw + /// priority maps to the lane of the highest named priority it meets or exceeds + /// (33/25/21/17, else background). Named priorities land where they always did; + /// this pins the in-between raw values. + @Test func priorityLaneMappingUsesInclusiveBandBoundaries() { + let expected: [(UInt8, Int)] = [ + (33, 0), // userInteractive + (26, 1), (25, 1), // (high, userInteractive) band + high itself + (24, 2), (21, 2), // (medium, high) band + medium itself + (20, 3), (17, 3), // (low, medium) band + low itself + (16, 4), (10, 4), (9, 4), (1, 4), // below low -> background lane + ] + for (raw, lane) in expected { + #expect(TaskPriority(rawValue: raw).jobQueueIndex == lane, + "rawValue \(raw) should map to lane \(lane)") + } + } +} + +#endif // canImport(Darwin)