1

audio (连续信号)  →  text (离散 token)

1

Encoder → Attention → Decoder → Output

1
2
3
4
5
6

输出: "hello world"
生成过程:
  1. 生成 "h"
  2. 基于 "h" 生成 "he"
  3. 基于 "he" 生成 "hel"
  4. ...

1
2
3

输出: "hello world"
生成过程:
  1. 直接输出完整句子 "hello world"

1
2
3
4
5
6
7

<key>NSExtension</key>
<dict>
    <key>NSExtensionPointIdentifier</key>
    <string>com.apple.broadcast-services-upload</string>
    <key>NSExtensionPrincipalClass</key>
    <string>$(PRODUCT_MODULE_NAME).SampleHandler</string>
</dict>

1
2
3
4

<key>UIBackgroundModes</key>
<array>
    <string>audio</string>
</array>

1
2
3
4
5
6

let picker = RPSystemBroadcastPickerView(
    frame: CGRect(x: 0, y: 0, width: 44, height: 44)
)
picker.preferredExtension = "com.xxx.broadcast"
picker.showsMicrophoneButton = true
view.addSubview(picker)

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18

override func processSampleBuffer(_ sampleBuffer: CMSampleBuffer, with sampleBufferType: RPSampleBufferType) {
    guard case .audioApp = sampleBufferType else { return }

    guard let formatDescription = CMSampleBufferGetFormatDescription(sampleBuffer),
          let streamDesc = CMAudioFormatDescriptionGetStreamBasicDescription(formatDescription)?.pointee else {
        return
    }

    let inputSampleRate = streamDesc.mSampleRate
    let channelCount = Int(streamDesc.mChannelsPerFrame)
    let bitsPerChannel = streamDesc.mBitsPerChannel
    let formatFlags = streamDesc.mFormatFlags
    let isFloat = (formatFlags & kAudioFormatFlagIsFloat) != 0
    let isNonInterleaved = (formatFlags & kAudioFormatFlagIsNonInterleaved) != 0
    let isBigEndian = (formatFlags & kAudioFormatFlagIsBigEndian) != 0

    // 提取音频数据...
}

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146

private let targetSampleRate: Double = 16000.0

/// 处理音频样本buffer
private func processAudioBuffer(_ sampleBuffer: CMSampleBuffer, source: String) {
    guard let formatDescription = CMSampleBufferGetFormatDescription(sampleBuffer),
          let streamDesc = CMAudioFormatDescriptionGetStreamBasicDescription(formatDescription)?.pointee else {
        return
    }

    let inputSampleRate = streamDesc.mSampleRate
    let channelCount = Int(streamDesc.mChannelsPerFrame)
    let bitsPerChannel = streamDesc.mBitsPerChannel
    let formatFlags = streamDesc.mFormatFlags
    let isFloat = (formatFlags & kAudioFormatFlagIsFloat) != 0
    let isNonInterleaved = (formatFlags & kAudioFormatFlagIsNonInterleaved) != 0
    let isBigEndian = (formatFlags & kAudioFormatFlagIsBigEndian) != 0

    // 获取 AudioBufferList
    var audioBufferList = AudioBufferList()
    var blockBuffer: CMBlockBuffer?

    let status = CMSampleBufferGetAudioBufferListWithRetainedBlockBuffer(
        sampleBuffer,
        bufferListSizeNeededOut: nil,
        bufferListOut: &audioBufferList,
        bufferListSize: MemoryLayout<AudioBufferList>.size,
        blockBufferAllocator: nil,
        blockBufferMemoryAllocator: nil,
        flags: kCMSampleBufferFlag_AudioBufferList_Assure16ByteAlignment,
        blockBufferOut: &blockBuffer
    )

    guard status == noErr else { return }

    let audioBufferListPointer = UnsafeMutableAudioBufferListPointer(
        UnsafeMutablePointer<AudioBufferList>.allocate(capacity: 1)
    )
    defer {
        audioBufferListPointer.unsafeMutablePointer.deallocate()
    }

    let numBuffers = audioBufferListPointer.count
    let frameCount = CMSampleBufferGetNumSamples(sampleBuffer)
    var floatSamples: [Float] = []

    // 处理非交错格式
    if isNonInterleaved {
        var channelData: [[Float]] = []

        for bufferIndex in 0..<numBuffers {
            let buffer = audioBufferListPointer[bufferIndex]
            guard let data = buffer.mData else { continue }

            let dataByteSize = Int(buffer.mDataByteSize)
            var channelSamples: [Float] = []

            if isFloat && bitsPerChannel == 32 {
                let floatPtr = data.assumingMemoryBound(to: Float.self)
                let count = dataByteSize / MemoryLayout<Float>.size
                for i in 0..<count {
                    var value = floatPtr[i]
                    if isBigEndian {
                        value = Float(bitPattern: value.bitPattern.bigEndian)
                    }
                    channelSamples.append(value)
                }
            } else if bitsPerChannel == 16 {
                let int16Ptr = data.assumingMemoryBound(to: Int16.self)
                let count = dataByteSize / MemoryLayout<Int16>.size
                for i in 0..<count {
                    var value = int16Ptr[i]
                    if isBigEndian { value = value.bigEndian }
                    channelSamples.append(Float(value) / 32768.0)
                }
            }

            channelData.append(channelSamples)
        }

        // 混音为单声道
        if let firstChannel = channelData.first {
            if channelData.count == 1 {
                floatSamples = firstChannel
            } else {
                for i in 0..<firstChannel.count {
                    var sum: Float = 0
                    for ch in channelData where i < ch.count {
                        sum += ch[i]
                    }
                    floatSamples.append(sum / Float(channelData.count))
                }
            }
        }
    } else {
        // 交错格式处理...
    }

    // 重采样到16kHz
    if inputSampleRate != targetSampleRate {
        floatSamples = resample(floatSamples, from: inputSampleRate, to: targetSampleRate)
    }

    sharedBuffer.writeAudioSamples(floatSamples)
}

/// 使用AVAudioConverter重采样
private func resample(_ samples: [Float], from inputRate: Double, to outputRate: Double) -> [Float] {
    guard inputRate > 0 && outputRate > 0, inputRate != outputRate, !samples.isEmpty else {
        return samples
    }

    guard let inputFormat = AVAudioFormat(
        commonFormat: .pcmFormatFloat32, sampleRate: inputRate, channels: 1, interleaved: false
    ),
    let outputFormat = AVAudioFormat(
        commonFormat: .pcmFormatFloat32, sampleRate: outputRate, channels: 1, interleaved: false
    ),
    let converter = AVAudioConverter(from: inputFormat, to: outputFormat) else {
        return samples
    }

    guard let inputBuffer = AVAudioPCMBuffer(pcmFormat: inputFormat, frameCapacity: AVAudioFrameCount(samples.count)),
          let outputBuffer = AVAudioPCMBuffer(pcmFormat: outputFormat) else {
        return samples
    }

    inputBuffer.frameLength = AVAudioFrameCount(samples.count)
    let inputData = inputBuffer.floatChannelData!
    for i in 0..<samples.count { inputData[0][i] = samples[i] }

    let ratio = outputRate / inputRate
    let outputFrameCount = Int(ceil(Double(samples.count) * ratio))
    outputBuffer.frameCapacity = AVAudioFrameCount(outputFrameCount)

    var error: NSError?
    let status = converter.convert(to: outputBuffer, error: &error) { _, outStatus in
        outStatus.pointee = .haveData
        return inputBuffer
    }

    guard status == .haveData, error == nil else { return samples }

    let outputData = outputBuffer.floatChannelData!
    let outputLength = Int(outputBuffer.frameLength)
    return (0..<outputLength).map { outputData[0][$0] }
}

1
2
3
4

<key>com.apple.security.application-groups</key>
<array>
    <string>group.com.xxx.shared</string>
</array>

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44

class AudioSharedBuffer {
    static let appGroupId = "group.com.xxx.shared"
    private let sharedContainerURL = FileManager.default.containerURL(
        forSecurityApplicationGroupIdentifier: Self.appGroupId
    )

    // Extension写入处理后的音频
    func writeAudioSamples(_ samples: [Float]) {
        guard let url = sharedContainerURL?.appendingPathComponent("audio.raw") else { return }

        let data = samples.withUnsafeBufferPointer { Data(buffer: $0) }

        if FileManager.default.fileExists(atPath: url.path) {
            let handle = try? FileHandle(forWritingTo: url)
            handle?.seekToEndOfFile()
            handle?.write(data)
            handle?.closeFile()
        } else {
            try? data.write(to: url)
        }

        postDarwinNotification("com.xxx.newAudioData")
    }

    // 主应用读取音频
    func readAudioSamples() -> [Float]? {
        guard let url = sharedContainerURL?.appendingPathComponent("audio.raw"),
              FileManager.default.fileExists(atPath: url.path) else { return nil }

        let data = try? Data(contentsOf: url)
        try? FileManager.default.removeItem(at: url)

        let floatCount = data?.count ?? 0 / MemoryLayout<Float>.size
        var samples = [Float](repeating: 0, count: floatCount)
        data?.copyBytes(to: samples.withUnsafeMutableBufferPointer { $0 })
        return samples
    }

    private func postDarwinNotification(_ name: String) {
        let center = CFNotificationCenterGetDarwinNotifyCenter()
        CFNotificationCenterPostNotification(center, CFNotificationName(name as CFString),
            nil, nil, true)
    }
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16

func startListening() {
    CFNotificationCenterAddObserver(
        CFNotificationCenterGetDarwinNotifyCenter(),
        Unmanaged.passUnretained(self).toOpaque(),
        { _, observer, _, _, _ in
            guard let observer = observer else { return }
            let selfPtr = Unmanaged<YourClass>.fromOpaque(observer).takeUnretainedValue()
            if let samples = selfPtr.audioBuffer.readAudioSamples() {
                selfPtr.onAudioReceived?(samples)
            }
        },
        "com.xxx.newAudioData" as CFString,
        nil,
        .deliverImmediately
    )
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30

class SampleHandler: RPBroadcastSampleHandler {
    private let sharedBuffer = AudioSharedBuffer()
    private let targetSampleRate: Double = 16000.0

    override func broadcastStarted(withSetupInfo setupInfo: [String : NSObject]?) {
        sharedBuffer.clearAudioData()
    }

    override func broadcastPaused() {}

    override func broadcastResumed() {}

    override func broadcastFinished() {}

    override func processSampleBuffer(_ sampleBuffer: CMSampleBuffer,
                                     with sampleBufferType: RPSampleBufferType) {
        switch sampleBufferType {
        case .audioApp:
            // 处理应用音频（系统播放的音频）
            let samples = convertTo16kMono(sampleBuffer)
            sharedBuffer.writeAudioSamples(samples)
        case .audioMic:
            // 忽略麦克风音频
            break
        case .video:
            // 忽略视频
            break
        }
    }
}