Skip to content

Commit b32dd43

Browse files
natario1natario1
committed
New pipeline, fix video issues and transcoding stalls
1 parent cf8b351 commit b32dd43

File tree

14 files changed

+255
-109
lines changed

14 files changed

+255
-109
lines changed

‎lib/src/androidTest/java/com/otaliastudios/transcoder/integration/IssuesTests.kt

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
package com.otaliastudios.transcoder.integration
22

3+
import android.media.MediaFormat
34
import android.media.MediaMetadataRetriever.METADATA_KEY_DURATION
45
import android.media.MediaMetadataRetriever
56
import androidx.test.ext.junit.runners.AndroidJUnit4
@@ -66,7 +67,7 @@ class IssuesTests {
6667
}
6768

6869

69-
@Test(timeout = 3000)
70+
@Test(timeout = 5000)
7071
fun issue137() = with(Helper(137)) {
7172
transcode {
7273
// addDataSource(ClipDataSource(input("main.mp3"), 0L, 200_000L))
@@ -93,7 +94,7 @@ class IssuesTests {
9394
Unit
9495
}
9596

96-
@Test(timeout = 3000)
97+
@Test(timeout = 5000)
9798
fun issue184() = with(Helper(184)) {
9899
transcode {
99100
addDataSource(TrackType.VIDEO, input("transcode.3gp"))

‎lib/src/main/java/com/otaliastudios/transcoder/internal/Codecs.kt

Lines changed: 37 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,19 @@
11
package com.otaliastudios.transcoder.internal
22

33
import android.media.MediaCodec
4+
import android.media.MediaCodecInfo
5+
import android.media.MediaCodecList
46
import android.media.MediaFormat
7+
import android.opengl.EGL14
58
import android.view.Surface
9+
import com.otaliastudios.opengl.core.EglCore
10+
import com.otaliastudios.opengl.surface.EglOffscreenSurface
11+
import com.otaliastudios.opengl.surface.EglWindowSurface
612
import com.otaliastudios.transcoder.common.TrackStatus
713
import com.otaliastudios.transcoder.common.TrackType
8-
import com.otaliastudios.transcoder.internal.media.MediaFormatProvider
14+
import com.otaliastudios.transcoder.internal.media.MediaFormatConstants
915
import com.otaliastudios.transcoder.internal.utils.Logger
1016
import com.otaliastudios.transcoder.internal.utils.TrackMap
11-
import com.otaliastudios.transcoder.internal.utils.trackMapOf
12-
import com.otaliastudios.transcoder.source.DataSource
13-
import com.otaliastudios.transcoder.strategy.TrackStrategy
1417

1518
/**
1619
* Encoders are shared between segments. This is not strictly needed but it is more efficient
@@ -25,6 +28,16 @@ internal class Codecs(
2528
private val current: TrackMap<Int>
2629
) {
2730

31+
internal class Surface(
32+
val context: EglCore,
33+
val window: EglWindowSurface,
34+
) {
35+
fun release() {
36+
window.release()
37+
context.release()
38+
}
39+
}
40+
2841
private val log = Logger("Codecs")
2942

3043
val encoders = object : TrackMap<Pair<MediaCodec, Surface?>> {
@@ -40,9 +53,28 @@ internal class Codecs(
4053

4154
private val lazyVideo by lazy {
4255
val format = tracks.outputFormats.video
56+
val width = format.getInteger(MediaFormat.KEY_WIDTH)
57+
val height = format.getInteger(MediaFormat.KEY_HEIGHT)
58+
log.i("Destination video surface size: ${width}x${height} @ ${format.getInteger(MediaFormatConstants.KEY_ROTATION_DEGREES)}")
59+
log.i("Destination video format: $format")
60+
61+
val allCodecs = MediaCodecList(MediaCodecList.REGULAR_CODECS)
62+
val videoEncoders = allCodecs.codecInfos.filter { it.isEncoder && it.supportedTypes.any { it.startsWith("video/") } }
63+
log.i("Available encoders: ${videoEncoders.joinToString { "${it.name} (${it.supportedTypes.joinToString()})" }}")
64+
65+
// Could consider MediaCodecList(MediaCodecList.REGULAR_CODECS).findEncoderForFormat(format)
66+
// But it's trickier, for example, format should not include frame rate on API 21 and maybe other quirks.
4367
val codec = MediaCodec.createEncoderByType(format.getString(MediaFormat.KEY_MIME)!!)
4468
codec.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE)
45-
codec to codec.createInputSurface()
69+
log.i("Selected encoder ${codec.name}")
70+
val surface = codec.createInputSurface()
71+
72+
log.i("Creating OpenGL context on ${Thread.currentThread()} (${surface.isValid})")
73+
val eglContext = EglCore(EGL14.EGL_NO_CONTEXT, EglCore.FLAG_RECORDABLE)
74+
val eglWindow = EglWindowSurface(eglContext, surface, true)
75+
eglWindow.makeCurrent()
76+
77+
codec to Surface(eglContext, eglWindow)
4678
}
4779

4880
override fun get(type: TrackType) = when (type) {

‎lib/src/main/java/com/otaliastudios/transcoder/internal/Segment.kt

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@ package com.otaliastudios.transcoder.internal
33
import com.otaliastudios.transcoder.common.TrackType
44
import com.otaliastudios.transcoder.internal.pipeline.Pipeline
55
import com.otaliastudios.transcoder.internal.pipeline.State
6-
import com.otaliastudios.transcoder.internal.utils.Logger
76

87
internal class Segment(
98
val type: TrackType,
@@ -27,7 +26,7 @@ internal class Segment(
2726
fun needsSleep(): Boolean {
2827
when(val s = state ?: return false) {
2928
is State.Ok -> return false
30-
is State.Wait -> return s.sleep
29+
is State.Failure -> return s.sleep
3130
}
3231
}
3332

‎lib/src/main/java/com/otaliastudios/transcoder/internal/Segments.kt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ internal class Segments(
2121

2222
fun hasNext(type: TrackType): Boolean {
2323
if (!sources.has(type)) return false
24-
log.v("hasNext($type): segment=${current.getOrNull(type)} lastIndex=${sources.getOrNull(type)?.lastIndex} canAdvance=${current.getOrNull(type)?.canAdvance()}")
24+
// log.v("hasNext($type): segment=${current.getOrNull(type)} lastIndex=${sources.getOrNull(type)?.lastIndex} canAdvance=${current.getOrNull(type)?.canAdvance()}")
2525
val segment = current.getOrNull(type) ?: return true // not started
2626
val lastIndex = sources.getOrNull(type)?.lastIndex ?: return false // no track!
2727
return segment.canAdvance() || segment.index < lastIndex

‎lib/src/main/java/com/otaliastudios/transcoder/internal/audio/AudioEngine.kt

Lines changed: 12 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -6,11 +6,8 @@ import android.view.Surface
66
import com.otaliastudios.transcoder.internal.audio.remix.AudioRemixer
77
import com.otaliastudios.transcoder.internal.codec.*
88
import com.otaliastudios.transcoder.internal.pipeline.*
9-
import com.otaliastudios.transcoder.internal.utils.Logger
10-
import com.otaliastudios.transcoder.internal.utils.trackMapOf
119
import com.otaliastudios.transcoder.resample.AudioResampler
1210
import com.otaliastudios.transcoder.stretch.AudioStretcher
13-
import java.util.concurrent.atomic.AtomicInteger
1411
import kotlin.math.ceil
1512
import kotlin.math.floor
1613

@@ -30,18 +27,18 @@ internal class AudioEngine(
3027
private val MediaFormat.sampleRate get() = getInteger(KEY_SAMPLE_RATE)
3128
private val MediaFormat.channels get() = getInteger(KEY_CHANNEL_COUNT)
3229

30+
private val chunks = ChunkQueue(log)
31+
private var readyToDrain = false
3332
private lateinit var rawFormat: MediaFormat
34-
private lateinit var chunks: ChunkQueue
3533
private lateinit var remixer: AudioRemixer
3634

3735
override fun handleSourceFormat(sourceFormat: MediaFormat): Surface? = null
3836

3937
override fun handleRawFormat(rawFormat: MediaFormat) {
4038
log.i("handleRawFormat($rawFormat)")
41-
check(!::rawFormat.isInitialized) { "handleRawFormat called twice: ${this.rawFormat} => $rawFormat"}
4239
this.rawFormat = rawFormat
43-
remixer = AudioRemixer[rawFormat.channels, targetFormat.channels]
44-
chunks = ChunkQueue(log, rawFormat.sampleRate, rawFormat.channels)
40+
this.remixer = AudioRemixer[rawFormat.channels, targetFormat.channels]
41+
this.readyToDrain = true
4542
}
4643

4744
override fun enqueueEos(data: DecoderData) {
@@ -59,19 +56,24 @@ internal class AudioEngine(
5956
}
6057

6158
override fun drain(): State<EncoderData> {
59+
if (!readyToDrain) {
60+
log.i("drain(): not ready, waiting...")
61+
return State.Retry(false)
62+
}
6263
if (chunks.isEmpty()) {
6364
// nothing was enqueued
6465
log.i("drain(): no chunks, waiting...")
65-
return State.Wait(false)
66+
return State.Retry(false)
6667
}
6768
val (outBytes, outId) = next.buffer() ?: return run {
6869
// dequeueInputBuffer failed
6970
log.i("drain(): no next buffer, waiting...")
70-
State.Wait(true)
71+
State.Retry(true)
7172
}
7273
val outBuffer = outBytes.asShortBuffer()
7374
return chunks.drain(
74-
eos = State.Eos(EncoderData(outBytes, outId, 0))
75+
eos = State.Eos(EncoderData(outBytes, outId, 0)),
76+
format = rawFormat
7577
) { inBuffer, timeUs, stretch ->
7678
val outSize = outBuffer.remaining()
7779
val inSize = inBuffer.remaining()

‎lib/src/main/java/com/otaliastudios/transcoder/internal/audio/chunks.kt

Lines changed: 40 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,10 @@
11
package com.otaliastudios.transcoder.internal.audio
22

3+
import android.media.MediaFormat
4+
import android.media.MediaFormat.KEY_CHANNEL_COUNT
5+
import android.media.MediaFormat.KEY_SAMPLE_RATE
36
import com.otaliastudios.transcoder.internal.utils.Logger
7+
import java.nio.ByteBuffer
48
import java.nio.ShortBuffer
59

610
private data class Chunk(
@@ -20,12 +24,9 @@ private data class Chunk(
2024
* big enough to contain the full processed size, in which case we want to consume only
2125
* part of the input buffer and keep it available for the next cycle.
2226
*/
23-
internal class ChunkQueue(
24-
private val log: Logger,
25-
private val sampleRate: Int,
26-
private val channels: Int
27-
) {
27+
internal class ChunkQueue(private val log: Logger) {
2828
private val queue = ArrayDeque<Chunk>()
29+
private val pool = ShortBufferPool()
2930

3031
fun isEmpty() = queue.isEmpty()
3132

@@ -44,7 +45,7 @@ internal class ChunkQueue(
4445
queue.addLast(Chunk.Eos)
4546
}
4647

47-
fun <T> drain(eos: T, action: (buffer: ShortBuffer, timeUs: Long, timeStretch: Double) -> T): T {
48+
fun <T> drain(format: MediaFormat, eos: T, action: (buffer: ShortBuffer, timeUs: Long, timeStretch: Double) -> T): T {
4849
val head = queue.removeFirst()
4950
if (head === Chunk.Eos) return eos
5051

@@ -54,9 +55,17 @@ internal class ChunkQueue(
5455
// Action can reduce the limit for any reason. Restore it before comparing sizes.
5556
head.buffer.limit(limit)
5657
if (head.buffer.hasRemaining()) {
58+
// We could technically hold onto the same chunk, but in practice it's better to
59+
// release input buffers back to the decoder otherwise it can get stuck
5760
val consumed = size - head.buffer.remaining()
61+
val sampleRate = format.getInteger(KEY_SAMPLE_RATE)
62+
val channelCount = format.getInteger(KEY_CHANNEL_COUNT)
63+
val buffer = pool.take(head.buffer)
64+
head.release()
5865
queue.addFirst(head.copy(
59-
timeUs = shortsToUs(consumed, sampleRate, channels)
66+
timeUs = shortsToUs(consumed, sampleRate, channelCount),
67+
release = { pool.give(buffer) },
68+
buffer = buffer
6069
))
6170
log.v("[ChunkQueue] partially handled chunk at ${head.timeUs}us, ${head.buffer.remaining()} bytes left (${queue.size})")
6271
} else {
@@ -67,3 +76,27 @@ internal class ChunkQueue(
6776
return result
6877
}
6978
}
79+
80+
81+
class ShortBufferPool {
82+
private val pool = mutableListOf<ShortBuffer>()
83+
84+
fun take(original: ShortBuffer): ShortBuffer {
85+
val needed = original.remaining()
86+
val index = pool.indexOfFirst { it.capacity() >= needed }
87+
val memory = when {
88+
index >= 0 -> pool.removeAt(index)
89+
else -> ByteBuffer.allocateDirect(needed.coerceAtLeast(1024))
90+
.order(original.order())
91+
.asShortBuffer()
92+
}
93+
memory.put(original)
94+
memory.flip()
95+
return memory
96+
}
97+
98+
fun give(buffer: ShortBuffer) {
99+
buffer.clear()
100+
pool.add(buffer)
101+
}
102+
}

‎lib/src/main/java/com/otaliastudios/transcoder/internal/codec/Decoder.kt

Lines changed: 3 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -10,11 +10,7 @@ import com.otaliastudios.transcoder.internal.data.ReaderData
1010
import com.otaliastudios.transcoder.internal.pipeline.Channel
1111
import com.otaliastudios.transcoder.internal.pipeline.QueuedStep
1212
import com.otaliastudios.transcoder.internal.pipeline.State
13-
import com.otaliastudios.transcoder.internal.utils.Logger
14-
import com.otaliastudios.transcoder.internal.utils.trackMapOf
1513
import java.nio.ByteBuffer
16-
import java.util.concurrent.atomic.AtomicInteger
17-
import kotlin.properties.Delegates
1814
import kotlin.properties.Delegates.observable
1915

2016

@@ -70,7 +66,7 @@ internal class Decoder(
7066
val buf = checkNotNull(codec.getInputBuffer(id)) { "inputBuffer($id) should not be null." }
7167
buf to id
7268
} else {
73-
log.i("buffer() failed. dequeuedInputs=$dequeuedInputs dequeuedOutputs=$dequeuedOutputs")
69+
log.i("buffer() failed with $id. dequeuedInputs=$dequeuedInputs dequeuedOutputs=$dequeuedOutputs")
7470
null
7571
}
7672
}
@@ -96,7 +92,7 @@ internal class Decoder(
9692
return when (result) {
9793
INFO_TRY_AGAIN_LATER -> {
9894
log.i("drain(): got INFO_TRY_AGAIN_LATER, waiting.")
99-
State.Wait(true)
95+
State.Retry(true)
10096
}
10197
INFO_OUTPUT_FORMAT_CHANGED -> {
10298
log.i("drain(): got INFO_OUTPUT_FORMAT_CHANGED, handling format and retrying. format=${codec.outputFormat}")
@@ -126,7 +122,7 @@ internal class Decoder(
126122
} else {
127123
// frame was dropped, no need to sleep
128124
codec.releaseOutputBuffer(result, false)
129-
State.Wait(false)
125+
State.Retry(false)
130126
}.also {
131127
log.v("drain(): returning $it")
132128
}

‎lib/src/main/java/com/otaliastudios/transcoder/internal/codec/Encoder.kt

Lines changed: 12 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -3,19 +3,15 @@ package com.otaliastudios.transcoder.internal.codec
33
import android.media.MediaCodec
44
import android.media.MediaCodec.*
55
import android.view.Surface
6+
import com.otaliastudios.opengl.surface.EglWindowSurface
67
import com.otaliastudios.transcoder.common.TrackType
7-
import com.otaliastudios.transcoder.common.trackType
88
import com.otaliastudios.transcoder.internal.Codecs
99
import com.otaliastudios.transcoder.internal.data.WriterChannel
1010
import com.otaliastudios.transcoder.internal.data.WriterData
1111
import com.otaliastudios.transcoder.internal.pipeline.Channel
1212
import com.otaliastudios.transcoder.internal.pipeline.QueuedStep
1313
import com.otaliastudios.transcoder.internal.pipeline.State
14-
import com.otaliastudios.transcoder.internal.utils.Logger
15-
import com.otaliastudios.transcoder.internal.utils.trackMapOf
1614
import java.nio.ByteBuffer
17-
import java.util.concurrent.atomic.AtomicInteger
18-
import kotlin.properties.Delegates
1915
import kotlin.properties.Delegates.observable
2016

2117
internal data class EncoderData(
@@ -27,15 +23,15 @@ internal data class EncoderData(
2723
}
2824

2925
internal interface EncoderChannel : Channel {
30-
val surface: Surface?
26+
val surface: Codecs.Surface?
3127
fun buffer(): Pair<ByteBuffer, Int>?
3228
}
3329

3430
internal class Encoder(
35-
private val codec: MediaCodec,
36-
override val surface: Surface?,
37-
ownsCodecStart: Boolean,
38-
private val ownsCodecStop: Boolean,
31+
private val codec: MediaCodec,
32+
override val surface: Codecs.Surface?,
33+
ownsCodecStart: Boolean,
34+
private val ownsCodecStop: Boolean,
3935
) : QueuedStep<EncoderData, EncoderChannel, WriterData, WriterChannel>(
4036
when (surface) {
4137
null -> "AudioEncoder"
@@ -44,13 +40,12 @@ internal class Encoder(
4440
), EncoderChannel {
4541

4642
constructor(codecs: Codecs, type: TrackType) : this(
47-
codecs.encoders[type].first,
48-
codecs.encoders[type].second,
49-
codecs.ownsEncoderStart[type],
50-
codecs.ownsEncoderStop[type]
43+
codecs.encoders[type].first,
44+
codecs.encoders[type].second,
45+
codecs.ownsEncoderStart[type],
46+
codecs.ownsEncoderStop[type]
5147
)
5248

53-
private val type = if (surface != null) TrackType.VIDEO else TrackType.AUDIO
5449
private var dequeuedInputs by observable(0) { _, _, _ -> printDequeued() }
5550
private var dequeuedOutputs by observable(0) { _, _, _ -> printDequeued() }
5651
private fun printDequeued() {
@@ -61,9 +56,8 @@ internal class Encoder(
6156

6257
private var info = BufferInfo()
6358

64-
6559
init {
66-
log.i("Encoder: ownsStart=$ownsCodecStart ownsStop=$ownsCodecStop")
60+
log.i("ownsStart=$ownsCodecStart ownsStop=$ownsCodecStop")
6761
if (ownsCodecStart) {
6862
codec.start()
6963
}
@@ -116,7 +110,7 @@ internal class Encoder(
116110
State.Eos(WriterData(buffer, 0L, 0) {})
117111
} else {
118112
log.i("Can't dequeue output buffer: INFO_TRY_AGAIN_LATER")
119-
State.Wait(true)
113+
State.Retry(true)
120114
}
121115
}
122116
INFO_OUTPUT_FORMAT_CHANGED -> {

0 commit comments

Comments
 (0)