From 9e2a4f4563124ee92596da1b961cdd702b8ce016 Mon Sep 17 00:00:00 2001 From: kyle-fitzp Date: Thu, 23 Apr 2026 13:09:49 -0500 Subject: [PATCH 01/10] Updated ffmpeg object reading/writing, decoder/encoder architecture --- .../video/transcoder/FFMpegTranscoder.java | 221 ++++++----------- .../video/transcoder/coders/Coder.java | 230 +++++++++--------- .../video/transcoder/coders/Decoder.java | 96 +++----- .../video/transcoder/coders/Encoder.java | 105 +++----- .../video/transcoder/coders/SwScaler.java | 95 +++----- .../formatters/AVByteFormatter.java | 2 +- .../transcoder/formatters/FrameFormatter.java | 187 ++++++++++++++ .../formatters/PacketFormatter.java | 2 +- .../transcoder/formatters/RgbFormatter.java | 2 +- .../video/transcoder/helpers/CodecInfo.java | 34 +++ .../transcoder/helpers/CodecOptions.java | 109 +++++++++ .../transcoder/helpers/FullCodecEnum.java | 85 +++++++ .../transcoder/helpers/FullPixelEnum.java | 189 ++++++++++++++ 13 files changed, 899 insertions(+), 458 deletions(-) create mode 100644 processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/formatters/FrameFormatter.java create mode 100644 processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/helpers/CodecInfo.java create mode 100644 processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/helpers/CodecOptions.java create mode 100644 processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/helpers/FullCodecEnum.java create mode 100644 processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/helpers/FullPixelEnum.java diff --git a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/FFMpegTranscoder.java b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/FFMpegTranscoder.java index 9ea520d92..0e02c8a73 100644 --- a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/FFMpegTranscoder.java +++ b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/FFMpegTranscoder.java @@ -17,19 +17,19 @@ import net.opengis.swe.v20.*; import org.bytedeco.ffmpeg.avcodec.AVPacket; import org.bytedeco.ffmpeg.avutil.AVFrame; +import org.bytedeco.ffmpeg.global.avcodec; +import org.bytedeco.ffmpeg.global.avutil; import org.bytedeco.javacpp.BytePointer; -import org.bytedeco.javacpp.DoublePointer; import org.bytedeco.javacpp.Pointer; -import org.sensorhub.api.common.SensorHubException; import org.sensorhub.api.processing.OSHProcessInfo; import org.sensorhub.impl.process.video.transcoder.coders.Coder; import org.sensorhub.impl.process.video.transcoder.coders.Decoder; import org.sensorhub.impl.process.video.transcoder.coders.Encoder; import org.sensorhub.impl.process.video.transcoder.coders.SwScaler; -import org.sensorhub.impl.process.video.transcoder.formatters.AVByteFormatter; -import org.sensorhub.impl.process.video.transcoder.formatters.PacketFormatter; -import org.sensorhub.impl.process.video.transcoder.formatters.RgbFormatter; -import org.sensorhub.impl.process.video.transcoder.formatters.YuvFormatter; +import org.sensorhub.impl.process.video.transcoder.formatters.*; +import org.sensorhub.impl.process.video.transcoder.helpers.CodecInfo; +import org.sensorhub.impl.process.video.transcoder.helpers.CodecOptions; +import org.sensorhub.impl.process.video.transcoder.helpers.FullCodecEnum; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.vast.data.DataBlockCompressed; @@ -39,13 +39,10 @@ import org.vast.swe.helper.RasterHelper; import javax.annotation.Nullable; -import java.nio.ByteBuffer; import java.util.*; import java.util.concurrent.atomic.AtomicBoolean; -import static org.bytedeco.ffmpeg.global.avcodec.*; import static org.bytedeco.ffmpeg.global.avutil.*; -import static org.bytedeco.ffmpeg.global.swscale.*; /** *

@@ -73,18 +70,14 @@ public class FFMpegTranscoder extends ExecutableProcessImpl Text inCodecParam; Text outCodecParam; - List> videoProcs; - AVByteFormatter inputFormatter, outputFormatter; - ArrayDeque inputPackets; - ArrayDeque outputPackets; - Thread outputThread; + List videoProcs; + AVByteFormatter inputFormatter, outputFormatter; - HashMap decOptions = new HashMap<>(); - HashMap encOptions = new HashMap<>(); + CodecOptions decOptions, encOptions; final boolean publish = false; - CodecEnum inCodec; - CodecEnum outCodec; + CodecInfo inCodec; + CodecInfo outCodec; RasterHelper swe = new RasterHelper(); int width, height, outWidth, outHeight; @@ -122,13 +115,13 @@ public FFMpegTranscoder() paramData.add("inCodec", inCodecParam = swe.createText() .definition(SWEHelper.getPropertyUri("Codec")) .label("Input Codec Name") - .addAllowedValues(CodecEnum.class) + //.addAllowedValues(CodecEnum.class) .build()); paramData.add("outCodec", outCodecParam = swe.createText() .definition(SWEHelper.getPropertyUri("Codec")) .label("Output Codec Name") - .addAllowedValues(CodecEnum.class) + //.addAllowedValues(CodecEnum.class) .build()); // outputs @@ -156,13 +149,18 @@ public FFMpegTranscoder() } + private void initFormatters() throws ProcessException { + inputFormatter = getFormatter(inCodec, width, height); + outputFormatter = getFormatter(outCodec, outWidth, outHeight); + } + /** * Initializes all encoder/decoder/swscaler objects. These objects are added to the {@link FFMpegTranscoder#videoProcs} * queue in the order they should process the incoming data. At most, the flow will be Decoder -> SWScale -> Encoder. */ private void initCoders() { try { - stopProcessThreads(); + stopProcessing(); } catch (Exception e){ logger.error("Transcoder could not stop process threads during re-init.", e); } @@ -172,30 +170,31 @@ private void initCoders() { videoProcs = new ArrayList<>(); if (!isUncompressed(inCodec)) { - videoProcs.add(new Decoder(inCodec.ffmpegId, decOptions)); + videoProcs.add(new Decoder(inCodec, outCodec, decOptions)); } if (width != outWidth || height != outHeight || (isUncompressed(inCodec) && isUncompressed(outCodec))) { - int inFmt = isUncompressed(inCodec) ? inCodec.ffmpegId : AV_PIX_FMT_YUV420P; - int outFmt = isUncompressed(outCodec) ? outCodec.ffmpegId : AV_PIX_FMT_YUV420P; - videoProcs.add(new SwScaler(inFmt, outFmt, width, height, outWidth, outHeight)); + //int inFmt = isUncompressed(inCodec) ? inCodec.getCodec().ffmpegId : AV_PIX_FMT_YUV420P; + //int outFmt = isUncompressed(outCodec) ? outCodec.getCodec().ffmpegId : AV_PIX_FMT_YUV420P; + videoProcs.add(new SwScaler(inCodec, outCodec, width, height, outWidth, outHeight)); } if (!isUncompressed(outCodec)) { - videoProcs.add(new Encoder(outCodec.ffmpegId, encOptions)); + Encoder encoder = new Encoder(inCodec, outCodec, decOptions); + videoProcs.add(encoder); } - inputPackets = new ArrayDeque<>(); - if (videoProcs.get(0) != null) { - videoProcs.get(0).setInQueue(inputPackets); - } - for (int i = 1; i < videoProcs.size(); i++) { - videoProcs.get(i).setInQueue(videoProcs.get(i - 1).getOutQueue()); - } - try { - outputPackets = (ArrayDeque) videoProcs.get(videoProcs.size() - 1).getOutQueue(); - } catch (Exception e) { - logger.warn("No processes running on video input. Check codec/resolution settings.", e); - outputPackets = inputPackets; + // Frame pipe between decoder, swscaler, encoder + for (int i = 0; i < videoProcs.size() - 1; i++) { + var nextProc = videoProcs.get(i + 1); + videoProcs.get(i).registerCallback(packet -> { + nextProc.submitInputPacket(packet); + }); } + + // Output + videoProcs.get(videoProcs.size() - 1).registerCallback(packet -> { + publishFrameData(outputFormatter.convertOutput(packet)); + }); + } /** @@ -204,15 +203,10 @@ private void initCoders() { */ private void startProcessThreads() { doRun.set(true); - if (videoProcs == null || videoProcs.isEmpty() || videoProcs.get(0).getState() != Thread.State.NEW) { + if (videoProcs == null || videoProcs.isEmpty()) { initCoders(); } - for (Thread process : videoProcs) { - process.start(); - } - outputThread.start(); - isRunning.set(true); } @@ -220,16 +214,15 @@ private void startProcessThreads() { * Invoked on process stop and init. * Stop all {@link Coder} thread objects stored in {@link FFMpegTranscoder#videoProcs}. */ - private void stopProcessThreads() throws InterruptedException { + private void stopProcessing() throws InterruptedException { doRun.set(false); //TODO These atomic booleans may be entirely unnecessary, remove if (videoProcs != null) { - for (Thread thread : videoProcs) { - thread.interrupt(); - thread.join(); + for (Coder codec : videoProcs) { + codec.close(); } + videoProcs.clear(); } - outputThread.interrupt(); - outputThread.join(); + isRunning.set(false); } @@ -268,7 +261,7 @@ public void notifyParamChange() public void stop() { if (isRunning.get()) { try { - stopProcessThreads(); + stopProcessing(); } catch (InterruptedException e) { logger.warn("Interrupted while stopping process threads"); } @@ -328,19 +321,15 @@ public void init() throws ProcessException // TODO: Automatically detect input codec from compression in data struct? try { - inCodec = CodecEnum.valueOf(inCodecParam.getData().getStringValue().toUpperCase()); - outCodec = CodecEnum.valueOf(outCodecParam.getData().getStringValue().toUpperCase()); + //inCodec = CodecEnum.valueOf(inCodecParam.getData().getStringValue().toUpperCase()); + //outCodec = CodecEnum.valueOf(outCodecParam.getData().getStringValue().toUpperCase()); + inCodec = new CodecInfo(inCodecParam.getData().getStringValue()); + outCodec = new CodecInfo(outCodecParam.getData().getStringValue()); setImgEncoding(); - initCodecOptions(); - - // processThreads are always running, passing available data from decoder to encoder and encoder to output + initFormatters(); initCoders(); - outputThread = new Thread(this::outputProcess); - - inputFormatter = getFormatter(inCodec, width, height); - outputFormatter = getFormatter(outCodec, outWidth, outHeight); imgOut.setData(new DataBlockCompressed()); } @@ -356,36 +345,14 @@ public void init() throws ProcessException * Creates maps of options for the encoder/decoder, including framerate, pixel format, bitrate, and image size. */ private void initCodecOptions() { - decOptions = new HashMap<>(); - encOptions = new HashMap<>(); + var decOptionBuilder = new CodecOptions.Builder(); + var encOptionBuilder = new CodecOptions.Builder(); //DataComponent temp; int fps = safeGetCountVal(inputFps); - if (fps > 0) { - decOptions.put("fps", fps); - encOptions.put("fps", fps); - } - - encOptions.put("pix_fmt", AV_PIX_FMT_YUV420P); - decOptions.put("pix_fmt", AV_PIX_FMT_YUV420P); - int bitrate = safeGetCountVal(inputBitrate); - if (bitrate > 0) { - decOptions.put("bit_rate", bitrate); - encOptions.put("bit_rate", bitrate); // Just assuming input br is the same as out, could this change? - } - width = safeGetCountVal(inputWidth); - if (width <= 0) { - width = imgIn.getComponent("row").getComponentCount(); - } - decOptions.put("width", width); - height = safeGetCountVal(inputHeight); - if (height <= 0) { - height = imgIn.getComponentCount(); - } - decOptions.put("height", height); outHeight = safeGetCountVal(outputHeight); @@ -393,28 +360,25 @@ private void initCodecOptions() { try { outHeight = imgOut.getComponentCount(); } catch (Exception ignored) { - outHeight = 0; + outHeight = height; + logger.warn("Output height not specified, using input height"); } } - if (outHeight > 0) { - encOptions.put("height", outHeight); - } else { - encOptions.put("height", height); - } outWidth = safeGetCountVal(outputWidth); if (outWidth <= 0) { try { outWidth = imgIn.getComponent("row").getComponentCount(); } catch (Exception ignored) { - outWidth = 0; + outWidth = width; + logger.warn("Output width not specified, using input width"); } } - if (outWidth > 0) { - encOptions.put("width", outWidth); - } else { - encOptions.put("width", width); - } + + decOptions = decOptionBuilder.setFps(fps).setBitRate(bitrate) + .setWidth(width).setHeight(height).presetUltraFast().tuneZeroLatency().build(); + encOptions = encOptionBuilder.setFps(fps).setBitRate(bitrate) + .setWidth(outWidth).setHeight(outHeight).presetUltraFast().tuneZeroLatency().build(); } /** @@ -426,13 +390,20 @@ private void initCodecOptions() { * @return Formatter object. * @throws ProcessException Thrown when width and height are not provided for an uncompressed format. */ - private AVByteFormatter getFormatter(CodecEnum codec, @Nullable Integer width, @Nullable Integer height) throws ProcessException { + private AVByteFormatter getFormatter(CodecInfo codec, @Nullable Integer width, @Nullable Integer height) throws ProcessException { try { + int pixFmt; + if ((pixFmt = codec.getPixelFmt().ffmpegId) != AV_PIX_FMT_NONE) + return new FrameFormatter(width, height, pixFmt); + else + return new PacketFormatter(); + /* return switch (codec) { case RGB -> new RgbFormatter(width, height); case YUV -> new YuvFormatter(width, height); default -> new PacketFormatter(); }; + */ } catch (NullPointerException e) { reportError("Raw formatter for " + codec + " requires non-null width and height.", e); } @@ -443,8 +414,8 @@ private AVByteFormatter getFormatter(CodecEnum codec, @Nullable Integer width, @ * @param codec Video codec or uncompressed format. * @return Is the codec {@link CodecEnum#RGB} or {@link CodecEnum#YUV}? */ - private boolean isUncompressed(CodecEnum codec) { - return codec == CodecEnum.RGB || codec == CodecEnum.YUV; + private boolean isUncompressed(CodecInfo codec) { + return codec.getCodec() == FullCodecEnum.RAWVIDEO; } /** @@ -503,42 +474,15 @@ public void execute() throws ProcessException logger.warn("Input image is null"); return; } + // Start the threads if not already started if (!isRunning.get()) { startProcessThreads(); } - inputPackets.add( - inputFormatter.convertInput( - ((DataBlockCompressed)imgIn.getData()).getUnderlyingObject().clone() - )); - - } - - /** - * Runs inside a separate thread. Receives any {@link AVPacket}s or {@link AVFrame}s from the last {@link Coder} - * in the {@link FFMpegTranscoder#videoProcs} queue, converts the struct to bytes, and publishes the data. - * @see FFMpegTranscoder#publishFrameData(byte[]) - */ - private void outputProcess() { - while (doRun.get() && !Thread.currentThread().isInterrupted()) { - while (outputPackets == null || outputPackets.isEmpty()) { - if (Thread.currentThread().isInterrupted()) { - return; - } - Thread.onSpinWait(); - } - if (outputPackets != null && !outputPackets.isEmpty()) { - for (var packet : outputPackets) { - if (Thread.currentThread().isInterrupted()) { - return; - } - publishFrameData(outputFormatter.convertOutput(outputPackets.poll())); - //frameData = null; - //packet = null; - } - } - } + videoProcs.get(0).submitInputPacket( + inputFormatter.convertInput(((DataBlockCompressed)imgIn.getData()).getUnderlyingObject().clone()) + ); } @Override @@ -556,21 +500,8 @@ public void dispose() doRun.set(false); if (videoProcs != null) { - for (Thread t : videoProcs) { - t.interrupt(); - try { - t.join(); - } catch (InterruptedException e) { - logger.error("Error waiting for process thread {} to finish", t.getName()); - } - } - } - if (outputThread != null) { - outputThread.interrupt(); - try { - outputThread.join(); - } catch (InterruptedException e) { - logger.error("Error waiting for process thread {} to finish", outputThread.getName()); + for (Coder proc : videoProcs) { + proc.close(); } } } diff --git a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/coders/Coder.java b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/coders/Coder.java index 5cc663d6d..e2009fef8 100644 --- a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/coders/Coder.java +++ b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/coders/Coder.java @@ -4,188 +4,198 @@ import java.util.ArrayDeque; import java.util.HashMap; +import java.util.Map; import java.util.Queue; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; import java.util.concurrent.atomic.AtomicBoolean; import static org.bytedeco.ffmpeg.global.avcodec.*; +import org.bytedeco.ffmpeg.avcodec.AVCodec; import org.bytedeco.ffmpeg.avcodec.AVCodecContext; import org.bytedeco.ffmpeg.avcodec.AVPacket; import org.bytedeco.ffmpeg.avutil.AVFrame; +import org.bytedeco.javacpp.Pointer; +import org.bytedeco.javacpp.PointerPointer; +import org.sensorhub.impl.process.video.transcoder.helpers.CodecInfo; +import org.sensorhub.impl.process.video.transcoder.helpers.CodecOptions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public abstract class Coder extends Thread { +public abstract class Coder implements AutoCloseable { + + + public interface CoderCallback { + // The recipient does not need to deallocate the output; this is done automatically + public abstract void onPacket(O packet); + } protected static final Logger logger = LoggerFactory.getLogger(Coder.class); - int codecId; - private static final int MAX_QUEUE_SIZE = 500; + private static int coderCount = 0; + private final int coderNum = coderCount++; + private final ExecutorService submitExecutor = Executors.newSingleThreadExecutor(r -> new Thread(r, "ffmpeg-codec-thread-" + coderNum)); + private final ExecutorService outputExecutor = Executors.newSingleThreadExecutor(r -> new Thread(r, "ffmpeg-codec-output-thread-" + coderNum)); + private final Map, ExecutorService> callbackMap = new HashMap<>(); + + CodecInfo inputFormat; + CodecInfo outputFormat; protected AVCodecContext codec_ctx; - protected Queue inPackets; // ONLY allow the main loop to poll - protected final Queue outPackets; // ONLY allow the main loop to add + protected AVCodec codec; protected I inPacket; protected O outPacket; - //volatile boolean isProcessing = false; // Set to true at the start of the loop, false at the end - volatile boolean isGettingPackets = false; - final Object waitingObj = new Object(); + protected final Queue outQueue = new ArrayDeque<>(10); + private AtomicBoolean isProcessing = new AtomicBoolean(true); // Set false to indicate packets should no longer be accepted + final Object contextLock = new Object(); Class inputClass; Class outputClass; - HashMap options; - - public AtomicBoolean doRun = new AtomicBoolean(true); + CodecOptions options; + AtomicBoolean isNotifying = new AtomicBoolean(false); - public Coder(int codecId, Class inputClass, Class outputClass, HashMap options) { + public Coder(CodecInfo inFormatInfo, CodecInfo outFormatInfo, Class inputClass, Class outputClass, CodecOptions options) { super(); assert inputClass == AVPacket.class || inputClass == AVFrame.class; assert outputClass == AVPacket.class || outputClass == AVFrame.class; assert options != null; - this.codecId = codecId; - this.inPackets = new ArrayDeque<>(); - this.outPackets = new ArrayDeque<>(); + this.inputFormat = inFormatInfo; this.inputClass = inputClass; this.outputClass = outputClass; this.options = options; - } - /** - * Gets reference to output queue. Typically used as the input queue for another {@link Coder}. - * @return Output queue containing either {@link AVPacket}s or {@link AVFrame}s. - */ - public Queue getOutQueue() { - return outPackets; + submitExecutor.submit(() -> { + initContext(); + initOptions(); + openContext(); + }); } - /** - * Sets input queue. - * @param inPackets Typically the output queue of another {@link Coder}, containing either - * {@link AVPacket}s or {@link AVFrame}s. - */ - public void setInQueue(Queue inPackets) { - this.inPackets = (Queue) inPackets; + public Class getOutputClass() { + return outputClass; } - // Safety net queue purging. - // Sometimes, queues can grow very large (like when a thread hangs up or is paused in the debugger). - // Can recover, so we don't want memory issues from too many items in the queues. - private void queuePurge() { - - synchronized (outPackets) { - if (outPackets.size() > MAX_QUEUE_SIZE) { logger.warn("Output queue is larger than max ({} > {}). Purging queue.", outPackets.size(), MAX_QUEUE_SIZE); } - while (outPackets.size() > MAX_QUEUE_SIZE) { - deallocateOutputPacket(outPackets.poll()); - } - } + public Class getInputClass() { + return inputClass; } - protected abstract void deallocateInputPacket(I packet); - - protected abstract void deallocateOutputPacket(O packet); - - protected abstract void deallocateOutQueue(); - - // To be implemented by subclasses, encoder/decoder protected abstract void initContext(); - // Take data from input queue and send to encoder/decoder - protected abstract void sendInPacket(); - - // Take data from encoder/decoder and send to output queue - protected abstract void receiveOutPacket(); - - // Allocate packets/frames - protected abstract void allocatePackets(); - - // Deallocate packets/frames - protected abstract void deallocatePackets(); + protected void openContext() { + if (avcodec_open2(codec_ctx, codec, (PointerPointer) null) < 0) { + throw new IllegalStateException("Error opening codec " + codec.name().getString()); + } + } /** * Set certain options in the codec context. * @param codec_ctx Codec context. Context must be allocated first. */ - protected void initOptions(AVCodecContext codec_ctx) { + protected void initOptions() { - codec_ctx.time_base(av_make_q(1, options.getOrDefault("fps", 30))); + codec_ctx.time_base(av_make_q(1, options.getFps())); - if (options.containsKey("bit_rate")) { - codec_ctx.bit_rate(options.get("bit_rate") * 1000); + if (options.getBitRate() > 0) { + codec_ctx.bit_rate(options.getBitRate() * 1000); } else { //codec_ctx.bit_rate(150*1000); } - codec_ctx.width(options.getOrDefault("width", 1920)); - - codec_ctx.height(options.getOrDefault("height", 1080)); + codec_ctx.width(options.getWidth()); + codec_ctx.height(options.getHeight()); + /* if (options.containsKey("pix_fmt")) { codec_ctx.pix_fmt(options.get("pix_fmt")); } else { - if (codecId == AV_CODEC_ID_MJPEG) { + if (inputFormat == AV_CODEC_ID_MJPEG) { codec_ctx.pix_fmt(AV_PIX_FMT_YUVJ420P); } else { codec_ctx.pix_fmt(AV_PIX_FMT_YUV420P); } } + */ + av_opt_set(codec_ctx.priv_data(), "preset", "ultrafast", 0); av_opt_set(codec_ctx.priv_data(), "tune", "zerolatency", 0); codec_ctx.strict_std_compliance(FF_COMPLIANCE_UNOFFICIAL); // Needed so that yuvj420p works (used for mjpeg) } - @Override - public void run() { - initContext(); - inPacket = (I)(new Object()); - outPacket = (O)(new Object()); - - allocatePackets(); - - // init FFMPEG logging - av_log_set_level(logger.isDebugEnabled() ? AV_LOG_INFO : AV_LOG_FATAL); - //av_log_set_level(AV_LOG_DEBUG); - - // Actual start of run - // Codec should be initialized by now - doRun.set(true); + protected abstract void deallocateInputPacket(I packet); + protected abstract void deallocateOutputPacket(O packet); + protected abstract O cloneOutput(O packet); + protected abstract void processInputPacket(I inputPacket); - while (doRun.get() && !Thread.currentThread().isInterrupted()) { - while (inPackets == null || inPackets.isEmpty()) { - if (!doRun.get() || Thread.currentThread().isInterrupted()) { break; } - Thread.onSpinWait(); + // Take data from input queue and send to encoder/decoder + public void submitInputPacket(I inputPacket) { + synchronized (contextLock) { + if (inputPacket == null || !isProcessing.get()) { + return; } - while (inPackets != null && !inPackets.isEmpty()) { - if (!doRun.get() || Thread.currentThread().isInterrupted()) { break; } + submitExecutor.submit(() -> { + // Process the input + processInputPacket(inputPacket); + deallocateInputPacket(inputPacket); + + if (!outQueue.isEmpty() && isNotifying.compareAndSet(false, true)) { + outputExecutor.submit(() -> { + for (var outputPacket : outQueue) { + notifyCallbacks(outputPacket); + } + }); + isNotifying.set(false); + } + }); + } + } - queuePurge(); + private void notifyCallbacks(O outputPacket) { + for (var entry: callbackMap.entrySet()) { + entry.getValue().submit(() -> { + var clonedOutputPacket = cloneOutput(outputPacket); + entry.getKey().onPacket(clonedOutputPacket); + deallocateOutputPacket(clonedOutputPacket); + }); + } + } - //logger.debug("Queue size: {}", inPackets.size()); - // Get data from in queue - // Send data to encoder/decoder - //logger.debug("{}: Sending packet", this.getClass().getName()); - sendInPacket(); + public void registerCallback(CoderCallback callback) { + if (!callbackMap.containsKey(callback)) { + callbackMap.put(callback, Executors.newSingleThreadExecutor(r -> new Thread(r, "ffmpeg-codec-" + coderNum + "-callback-thread"))); + } else { + logger.warn("This callback was already registered for codec " + coderNum); + } + } - // Receive data from encoder/decoder - // Add data to out queue - //logger.debug("{}: Receiving packet", this.getClass().getName()); - receiveOutPacket(); + public void unregisterCallback(CoderCallback callback) { + callbackMap.remove(callback); + } - //isProcessing = false; - // Wake a thread waiting for packets. + public void unregisterAllCallbacks() { + callbackMap.clear(); + } + @Override + public void close() { + synchronized (contextLock) { + if (isProcessing.compareAndSet(true, false)) { + submitExecutor.shutdownNow(); + outputExecutor.shutdownNow(); + + if (codec_ctx != null) { + avcodec_free_context(codec_ctx); + } + codec_ctx = null; + codec = null; + + unregisterAllCallbacks(); + + for (var packet : outQueue) { + deallocateOutputPacket(packet); + } } } - - // End of coding - synchronized (waitingObj) { - waitingObj.notifyAll(); - } - - deallocatePackets(); - if (codec_ctx != null) { - avcodec_free_context(codec_ctx); - } - codec_ctx = null; } } diff --git a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/coders/Decoder.java b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/coders/Decoder.java index fc5709954..9adc2a778 100644 --- a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/coders/Decoder.java +++ b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/coders/Decoder.java @@ -4,99 +4,65 @@ import org.bytedeco.ffmpeg.avcodec.AVPacket; import org.bytedeco.ffmpeg.avutil.AVFrame; import org.bytedeco.javacpp.PointerPointer; +import org.sensorhub.impl.process.video.transcoder.helpers.CodecInfo; +import org.sensorhub.impl.process.video.transcoder.helpers.CodecOptions; +import org.sensorhub.impl.process.video.transcoder.helpers.FullCodecEnum; +import org.sensorhub.impl.process.video.transcoder.helpers.FullPixelEnum; +import java.util.ArrayDeque; import java.util.HashMap; +import java.util.Queue; import static org.bytedeco.ffmpeg.global.avcodec.*; import static org.bytedeco.ffmpeg.global.avutil.*; public class Decoder extends Coder { - public Decoder(int codecId, HashMap options) { - super(codecId, AVPacket.class, AVFrame.class, options); - } - - @Override - protected void initContext() { - AVCodec codec = avcodec_find_decoder(codecId); - codec_ctx = avcodec_alloc_context3(codec); - - initOptions(codec_ctx); - if (avcodec_open2(codec_ctx, codec, (PointerPointer)null) < 0) { - throw new IllegalStateException("Error initializing " + codec.name().getString() + " decoder"); - } + public Decoder(CodecInfo inFormatInfo, CodecInfo outFormatInfo, CodecOptions options) { + super(inFormatInfo, outFormatInfo, AVPacket.class, AVFrame.class, options); } - // Get compressed packet, send to decoder @Override - protected void sendInPacket() { - inPacket = inPackets.poll(); - //logger.debug("decode send:"); - //logger.debug(" data[0]: {}", inPacket.data()); - //logger.debug("Sent frame to encoder"); - avcodec_send_packet(codec_ctx, av_packet_clone(inPacket)); - //av_packet_free(inPacket); - } - - // Receive uncompressed frame from decoder - @Override - protected void receiveOutPacket() { - synchronized (outPackets) { - while (avcodec_receive_frame(codec_ctx, outPacket) >= 0) { - //av_packet_free(inPacket); - outPackets.add(av_frame_clone(outPacket)); - //av_frame_free(outPacket); - //logger.debug("Decode Packet added"); - } + protected void initContext() { + synchronized (contextLock) { + codec = avcodec_find_decoder(inputFormat.getCodec().ffmpegId);; + codec_ctx = avcodec_alloc_context3(codec); + codec_ctx.pix_fmt(outputFormat.getPixelFmt().ffmpegId); } } @Override protected void deallocateInputPacket(AVPacket packet) { - av_packet_free(packet); - packet = null; + if (packet != null) { + av_packet_free(packet); + } } @Override protected void deallocateOutputPacket(AVFrame packet) { - av_frame_free(packet); - packet = null; - } - - @Override - protected void deallocateOutQueue() { - outPackets.clear(); + if (packet != null) { + av_frame_free(packet); + } } @Override - protected void allocatePackets() { - inPacket = new AVPacket(); - inPacket = av_packet_alloc(); - av_init_packet(inPacket); - outPacket = new AVFrame(); - outPacket = av_frame_alloc(); + protected AVFrame cloneOutput(AVFrame packet) { + if (packet != null) { + return av_frame_clone(packet); + } else { + return null; + } } @Override - protected void deallocatePackets() { - if (inPacket != null) { - av_packet_free(inPacket); - } - if (outPacket != null) { - av_frame_free(outPacket); - } - if (outPackets != null) { - for (AVFrame frame : outPackets) { - av_frame_free(frame); - } - outPackets.clear(); - } - if (inPackets != null) { - for (AVPacket packet : inPackets) { - av_packet_free(packet); + protected void processInputPacket(AVPacket inputPacket) { + if (inputPacket != null) { + AVFrame outputPacket = av_frame_alloc(); + avcodec_send_packet(codec_ctx, inputPacket); + while (avcodec_receive_frame(codec_ctx, outputPacket) >= 0) { + outQueue.add(av_frame_clone(outputPacket)); } - inPackets.clear(); } } } diff --git a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/coders/Encoder.java b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/coders/Encoder.java index da47fcdfc..1cacaedd5 100644 --- a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/coders/Encoder.java +++ b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/coders/Encoder.java @@ -5,107 +5,68 @@ import org.bytedeco.ffmpeg.avutil.AVFrame; import org.bytedeco.javacpp.BytePointer; import org.bytedeco.javacpp.PointerPointer; +import org.sensorhub.impl.process.video.transcoder.helpers.CodecInfo; +import org.sensorhub.impl.process.video.transcoder.helpers.CodecOptions; +import org.sensorhub.impl.process.video.transcoder.helpers.FullCodecEnum; +import org.sensorhub.impl.process.video.transcoder.helpers.FullPixelEnum; +import java.util.ArrayDeque; import java.util.HashMap; +import java.util.Queue; import static org.bytedeco.ffmpeg.global.avcodec.*; import static org.bytedeco.ffmpeg.global.avutil.*; +import static org.bytedeco.ffmpeg.global.swscale.sws_freeContext; public class Encoder extends Coder { - long pts = 0; - public Encoder(int codecId, HashMap options) { - super(codecId, AVFrame.class, AVPacket.class, options); - } - - @Override - protected void deallocateOutQueue() { - outPackets.clear(); + public Encoder(CodecInfo inFormatInfo, CodecInfo outFormatInfo, CodecOptions options) { + super(inFormatInfo, outFormatInfo, AVFrame.class, AVPacket.class, options); } @Override protected void initContext() { - pts = 0; - AVCodec codec = avcodec_find_encoder(codecId); - codec_ctx = avcodec_alloc_context3(codec); - - initOptions(codec_ctx); - int ret = avcodec_open2(codec_ctx, codec, (PointerPointer)null); - if (ret < 0) { - BytePointer errorBuffer = new BytePointer(AV_ERROR_MAX_STRING_SIZE); - - av_strerror(ret, errorBuffer, AV_ERROR_MAX_STRING_SIZE); - logger.debug("Receive Error: {}", errorBuffer.getString()); - throw new IllegalStateException("Error initializing " + codec.name().getString() + " encoder"); - } - } - - @Override - protected void sendInPacket() { - inPacket = inPackets.poll(); - //pts++; - //inPacket.pts(pts); - - avcodec_send_frame(codec_ctx, av_frame_clone(inPacket)); - //av_frame_free(inPacket); - - } + synchronized (contextLock) { + AVCodec codec = avcodec_find_encoder(outputFormat.getCodec().ffmpegId);; + codec_ctx = avcodec_alloc_context3(codec); + //initOptions(codec_ctx); - @Override - protected void receiveOutPacket() { - /* - synchronized (outPackets) { - - } - - */ - int ret = 0; - while (( ret = avcodec_receive_packet(codec_ctx, outPacket) ) >= 0) { - outPackets.add(av_packet_clone(outPacket)); + codec_ctx.pix_fmt(inputFormat.getPixelFmt().ffmpegId); } - //BytePointer errorBuffer = new BytePointer(AV_ERROR_MAX_STRING_SIZE); - //av_strerror(ret, errorBuffer, AV_ERROR_MAX_STRING_SIZE); - //logger.debug("Receive Error: {}", errorBuffer.getString()); } @Override protected void deallocateInputPacket(AVFrame packet) { - av_frame_free(packet); - packet = null; + if (packet != null) { + av_frame_free(packet); + } } @Override protected void deallocateOutputPacket(AVPacket packet) { - av_packet_free(packet); - packet = null; + if (packet != null) { + av_packet_free(packet); + } } @Override - protected void allocatePackets() { - inPacket = av_frame_alloc(); - outPacket = av_packet_alloc(); - av_init_packet(outPacket); + protected AVPacket cloneOutput(AVPacket packet) { + if (packet != null) { + return av_packet_clone(packet); + } else { + return null; + } } @Override - protected void deallocatePackets() { - if (inPacket != null) { - av_frame_free(inPacket); - } - if (outPacket != null) { - av_packet_free(outPacket); - } - if (outPackets != null) { - for (AVPacket packet : outPackets) { - av_packet_free(packet); - } - outPackets.clear(); - } - if (inPackets != null) { - for (AVFrame frame : inPackets) { - av_frame_free(frame); + protected void processInputPacket(AVFrame inputPacket) { + if (inputPacket != null) { + AVPacket outputPacket = av_packet_alloc(); + avcodec_send_frame(codec_ctx, inputPacket); + + while (avcodec_receive_packet(codec_ctx, outputPacket) >= 0) { + outQueue.add(av_packet_clone(outputPacket)); } - inPackets.clear(); } } } diff --git a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/coders/SwScaler.java b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/coders/SwScaler.java index 3bf44c047..343172662 100644 --- a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/coders/SwScaler.java +++ b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/coders/SwScaler.java @@ -7,8 +7,11 @@ import org.bytedeco.javacpp.BytePointer; import org.bytedeco.javacpp.DoublePointer; import org.bytedeco.javacpp.PointerPointer; +import org.sensorhub.impl.process.video.transcoder.helpers.CodecInfo; +import java.util.ArrayDeque; import java.util.HashMap; +import java.util.Queue; import static org.bytedeco.ffmpeg.global.avcodec.*; import static org.bytedeco.ffmpeg.global.avutil.*; @@ -17,97 +20,63 @@ public class SwScaler extends Coder { long pts = 0; SwsContext swsContext; - final int inPixFmt, outPixFmt, inWidth, inHeight, outWidth, outHeight; + final int inWidth, inHeight, outWidth, outHeight; - public SwScaler(int inPixFmt, int outPixFmt, int inWidth, int inHeight, int outWidth, int outHeight) { - super(0, AVFrame.class, AVFrame.class, new HashMap()); - this.inPixFmt = inPixFmt; - this.outPixFmt = outPixFmt; + public SwScaler(CodecInfo inputFormat, CodecInfo outputFormat, int inWidth, int inHeight, int outWidth, int outHeight) { + super(inputFormat, outputFormat, AVFrame.class, AVFrame.class, null); this.inWidth = inWidth; this.inHeight = inHeight; this.outWidth = outWidth; this.outHeight = outHeight; } - @Override - protected void deallocateOutQueue() { - outPackets.clear(); - } - @Override protected void initContext() { - pts = 0; - swsContext = sws_getContext(inWidth, inHeight, inPixFmt, - outWidth, outHeight, outPixFmt, + swsContext = sws_getContext(inWidth, inHeight, inputFormat.getPixelFmt().ffmpegId, + outWidth, outHeight, outputFormat.getPixelFmt().ffmpegId, SWS_BICUBIC, null, null, (DoublePointer) null); - - } - - @Override - protected void sendInPacket() { - inPacket = inPackets.poll(); - //pts++; - //inPacket.pts(pts); - sws_scale_frame(swsContext, outPacket, inPacket); - //avcodec_send_frame(codec_ctx, av_frame_clone(inPacket)); - //av_frame_free(inPacket); - } @Override - protected void receiveOutPacket() { - // We already have outPacket at this point (one method needed for scaling rather than two) - // Just use this to add the output to the out queue and make the buffers writable - outPackets.add(av_frame_clone(outPacket)); - av_frame_make_writable(outPacket); - } + protected void initOptions() {} // no options for swscaler @Override protected void deallocateInputPacket(AVFrame packet) { - av_frame_free(packet); - packet = null; + if (packet != null) { + av_frame_free(packet); + } } @Override protected void deallocateOutputPacket(AVFrame packet) { - av_frame_free(packet); - packet = null; + if (packet != null) { + av_frame_free(packet); + } } @Override - protected void allocatePackets() { - inPacket = av_frame_alloc(); + protected AVFrame cloneOutput(AVFrame packet) { + if (packet != null) { + return av_frame_clone(packet); + } else { + return null; + } + } - outPacket = av_frame_alloc(); - outPacket.format(outPixFmt); - outPacket.width(outWidth); - outPacket.height(outHeight); - av_image_alloc(outPacket.data(), outPacket.linesize(), - outWidth, outHeight, outPixFmt, 1); + @Override + protected void processInputPacket(AVFrame inputPacket) { + if (inputPacket != null) { + AVFrame outputPacket = av_frame_alloc(); + sws_scale_frame(swsContext, outPacket, inPacket); + outQueue.add(outputPacket); + } } @Override - protected void deallocatePackets() { - if (swsContext != null) { + public void close() { + synchronized (contextLock) { + super.close(); sws_freeContext(swsContext); } - if (inPacket != null) { - av_frame_free(inPacket); - } - if (outPacket != null) { - av_frame_free(outPacket); - } - if (outPackets != null) { - for (AVFrame frame : outPackets) { - av_frame_free(frame); - } - outPackets.clear(); - } - if (inPackets != null) { - for (AVFrame frame : inPackets) { - av_frame_free(frame); - } - inPackets.clear(); - } } } diff --git a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/formatters/AVByteFormatter.java b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/formatters/AVByteFormatter.java index 60227176d..c9b71f651 100644 --- a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/formatters/AVByteFormatter.java +++ b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/formatters/AVByteFormatter.java @@ -5,7 +5,7 @@ import org.bytedeco.javacpp.Pointer; import org.sensorhub.impl.process.video.transcoder.CodecEnum; -public abstract class AVByteFormatter { +public interface AVByteFormatter { public abstract T convertInput(byte[] inputData); diff --git a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/formatters/FrameFormatter.java b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/formatters/FrameFormatter.java new file mode 100644 index 000000000..04d6c301a --- /dev/null +++ b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/formatters/FrameFormatter.java @@ -0,0 +1,187 @@ +package org.sensorhub.impl.process.video.transcoder.formatters; + +import org.bytedeco.ffmpeg.avutil.*; +import org.bytedeco.ffmpeg.global.avutil; +import org.bytedeco.javacpp.BytePointer; + +import java.util.HashSet; +import java.util.Set; + +import static org.bytedeco.ffmpeg.global.avutil.*; + +// This class is intended to replace both RgbFormatter and YuvFormatter as a generic solution +public class FrameFormatter implements AVByteFormatter { + + private final int width; + private final int height; + private final int pixFmt; + private final AVPixFmtDescriptor desc; + private final int planeCount; + private final int[] planeWidths; + private final int[] planeHeights; + private final int[] planeSizes; + private final int totalSize; + + public FrameFormatter(int width, int height, int pixFmt) { + this.width = width; + this.height = height; + this.pixFmt = pixFmt; + this.desc = avutil.av_pix_fmt_desc_get(pixFmt); + boolean isPlanar = (desc.flags() & AV_PIX_FMT_FLAG_PLANAR) == 0; + + if (isPlanar) { + this.planeCount = countPlanes(desc); + + planeWidths = new int[planeCount]; + planeHeights = new int[planeCount]; + planeSizes = new int[planeCount]; + + calculatePlaneSizes(desc, width, height, planeWidths, planeHeights, planeSizes); + } else { + planeCount = 1; + planeWidths = new int[1]; + planeHeights = new int[1]; + planeSizes = new int[1]; + planeWidths[0] = width; + planeHeights[0] = height; + planeSizes[0] = width * height; + } + + totalSize = calcByteSize(planeSizes); + } + + public int getPixFmt() { + return pixFmt; + } + + public int getTotalSize() { + return totalSize; + } + + private static int calcByteSize(int[] planeSizes) { + int sum = 0; + for (int s : planeSizes) sum += s; + return sum; + } + + @Override + public AVFrame convertInput(byte[] inputData) { + AVFrame newFrame = generateFrame(); + + if ((desc.flags() & AV_PIX_FMT_FLAG_PLANAR) == 0) + setFrameDataPlanar(newFrame, inputData); + else + setFrameDataPacked(newFrame, inputData); + + return newFrame; + } + + private void setFrameDataPlanar(AVFrame newFrame, byte[] inputData) { + int offset = 0; + + for (int p = 0; p < planeCount; p++) { + int w = planeWidths[p]; + int h = planeHeights[p]; + int stride = newFrame.linesize(p); + + BytePointer dst = newFrame.data(p).position(0); + + int rowBytes = w; + + for (int y = 0; y < h; y++) { + dst.position(y * stride); + dst.put(inputData, offset + y * rowBytes, rowBytes); + } + + offset += planeSizes[p]; + } + } + + private void setFrameDataPacked(AVFrame newFrame, byte[] inputData) { + BytePointer dst = newFrame.data(0); + int linesize = newFrame.linesize(0); + + int bytesPerPixel = av_get_bits_per_pixel(av_pix_fmt_desc_get(pixFmt)) / 8; + + int rowBytes = width * bytesPerPixel; + + int offset = 0; + + for (int y = 0; y < height; y++) { + dst.position((long) y * linesize) + .put(inputData, offset, rowBytes); + + offset += rowBytes; + } + } + + private AVFrame generateFrame() { + AVFrame newFrame = av_frame_alloc(); + newFrame.format(pixFmt); + newFrame.width(width); + newFrame.height(height); + av_frame_get_buffer(newFrame, 32); + return newFrame; + } + + @Override + public byte[] convertOutput(AVFrame outputFrame) { + byte[] out = new byte[totalSize]; + int offset = 0; + + for (int plane = 0; plane < 3; plane++) { + int w = planeWidth(plane); + int h = planeHeight(plane); + int srcStride = outputFrame.linesize(plane); + + BytePointer src = outputFrame.data(plane).position(0); + + for (int y = 0; y < h; y++) { + src.position(y * srcStride); + src.get(out, offset, w); + offset += w; + } + } + return out; + } + + private int planeWidth(int plane) { + if (plane == 0) + return width; + else + return width >> desc.log2_chroma_w(); + } + + private int planeHeight(int plane) { + if (plane == 0) + return height; + else + return height >> desc.log2_chroma_h(); + } + + // Helper functions for PLANAR formats. Untested with packed formats. + private static int countPlanes(AVPixFmtDescriptor desc) { + Set planes = new HashSet<>(); + for (int i = 0; i < desc.nb_components(); i++) { + planes.add(desc.comp(i).plane()); + } + return planes.size(); + } + + private static void calculatePlaneSizes(AVPixFmtDescriptor desc, int width, int height, int[] planeWidths, int[] planeHeights, int[] planeSizes) { + for (int c = 0; c < desc.nb_components(); c++) { + AVComponentDescriptor comp = desc.comp(c); + int p = comp.plane(); + + int shiftW = (p == 0) ? 0 : desc.log2_chroma_w(); + int shiftH = (p == 0) ? 0 : desc.log2_chroma_h(); + + planeWidths[p] = width >> shiftW; + planeHeights[p] = height >> shiftH; + } + + for (int p = 0; p < planeSizes.length; p++) { + planeSizes[p] = planeWidths[p] * planeHeights[p]; + } + } +} diff --git a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/formatters/PacketFormatter.java b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/formatters/PacketFormatter.java index b3b09149b..e88de05fc 100644 --- a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/formatters/PacketFormatter.java +++ b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/formatters/PacketFormatter.java @@ -10,7 +10,7 @@ import static org.bytedeco.ffmpeg.global.avutil.*; import static org.bytedeco.ffmpeg.global.swscale.*; -public class PacketFormatter extends AVByteFormatter { +public class PacketFormatter implements AVByteFormatter { /** * Converts an array of bytes from compressed video into an {@link AVPacket}. diff --git a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/formatters/RgbFormatter.java b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/formatters/RgbFormatter.java index 2e0eb7fd8..5d5766c83 100644 --- a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/formatters/RgbFormatter.java +++ b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/formatters/RgbFormatter.java @@ -7,7 +7,7 @@ import static org.bytedeco.ffmpeg.global.avutil.*; import static org.bytedeco.ffmpeg.global.swscale.*; -public class RgbFormatter extends AVByteFormatter { +public class RgbFormatter implements AVByteFormatter { protected final int width, height, size; protected final int pixFormat; diff --git a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/helpers/CodecInfo.java b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/helpers/CodecInfo.java new file mode 100644 index 000000000..0e0315bf3 --- /dev/null +++ b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/helpers/CodecInfo.java @@ -0,0 +1,34 @@ +package org.sensorhub.impl.process.video.transcoder.helpers; + +public class CodecInfo { + private FullCodecEnum codec; + private FullPixelEnum pixelFmt; + + public CodecInfo(String name) { + try { + this.codec = Enum.valueOf(FullCodecEnum.class, name); + // YUVJ works best here + if (this.codec == FullCodecEnum.MJPEG) + this.pixelFmt = FullPixelEnum.YUVJ420P; + else + this.pixelFmt = FullPixelEnum.YUV420P; + + } catch (Exception e) { + this.codec = FullCodecEnum.RAWVIDEO; + this.pixelFmt = Enum.valueOf(FullPixelEnum.class, name); + } + } + + public CodecInfo(FullCodecEnum codec, FullPixelEnum pixelFmt) { + this.codec = codec; + this.pixelFmt = pixelFmt; + } + + public FullCodecEnum getCodec() { + return codec; + } + + public FullPixelEnum getPixelFmt() { + return pixelFmt; + } +} diff --git a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/helpers/CodecOptions.java b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/helpers/CodecOptions.java new file mode 100644 index 000000000..78e8ff375 --- /dev/null +++ b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/helpers/CodecOptions.java @@ -0,0 +1,109 @@ +package org.sensorhub.impl.process.video.transcoder.helpers; +import static org.bytedeco.ffmpeg.global.avcodec.*; + +public class CodecOptions { + private int fps; + private int bitRate; + private int width; + private int height; + private int compliance; + private String preset; + private String tune; + + public CodecOptions(int fps, int bitRate, int width, int height, int compliance, String preset, String tune) { + this.fps = Math.max(fps, 1); + this.bitRate = bitRate; + this.width = Math.max(width, 1); + this.height = Math.max(height, 1); + this.compliance = compliance; + this.preset = preset; + this.tune = tune; + } + + public int getFps() { + return fps; + } + + public int getBitRate() { + return bitRate; + } + + public int getWidth() { + return width; + } + + public int getHeight() { + return height; + } + + public int getCompliance() { + return compliance; + } + + public String getPreset() { + return preset; + } + + public String getTune() { + return tune; + } + + public static class Builder { + private int fps = 30; + private int bitRate = -1; + private int width = 1920; + private int height = 1080; + private int compliance = FF_COMPLIANCE_UNOFFICIAL; + private String preset = null; + private String tune = null; + + public CodecOptions build() { + return new CodecOptions(fps, bitRate, width, height, compliance, preset, tune); + } + + public Builder setFps(int fps) { + this.fps = fps; + return this; + } + + public Builder setBitRate(int bitRate) { + this.bitRate = bitRate; + return this; + } + + public Builder setWidth(int width) { + this.width = width; + return this; + } + + public Builder setHeight(int height) { + this.height = height; + return this; + } + + public Builder setCompliance(int compliance) { + this.compliance = compliance; + return this; + } + + public Builder setPreset(String preset) { + this.preset = preset; + return this; + } + + public Builder presetUltraFast() { + this.preset = "ultrafast"; + return this; + } + + public Builder setTune(String tune) { + this.tune = tune; + return this; + } + + public Builder tuneZeroLatency() { + this.tune = "zerolatency"; + return this; + } + } +} diff --git a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/helpers/FullCodecEnum.java b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/helpers/FullCodecEnum.java new file mode 100644 index 000000000..cc2827e9a --- /dev/null +++ b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/helpers/FullCodecEnum.java @@ -0,0 +1,85 @@ +package org.sensorhub.impl.process.video.transcoder.helpers; + +import org.bytedeco.ffmpeg.global.avcodec; + +public enum FullCodecEnum { + H264(avcodec.AV_CODEC_ID_H264), + H265(avcodec.AV_CODEC_ID_H265), + HEVC(avcodec.AV_CODEC_ID_HEVC), + MJPEG(avcodec.AV_CODEC_ID_MJPEG), + VP8(avcodec.AV_CODEC_ID_VP8), + VP9(avcodec.AV_CODEC_ID_VP9), + MPEG2(avcodec.AV_CODEC_ID_MPEG2TS), + MPEG4(avcodec.AV_CODEC_ID_MPEG4), + AV1(avcodec.AV_CODEC_ID_AV1), + THEORA(avcodec.AV_CODEC_ID_THEORA), + MPEG1VIDEO(avcodec.AV_CODEC_ID_MPEG1VIDEO), + WMV1(avcodec.AV_CODEC_ID_WMV1), + WMV2(avcodec.AV_CODEC_ID_WMV2), + WMV3(avcodec.AV_CODEC_ID_WMV3), + VC1(avcodec.AV_CODEC_ID_VC1), + FLV1(avcodec.AV_CODEC_ID_FLV1), + FLASHSV(avcodec.AV_CODEC_ID_FLASHSV), + FLASHSV2(avcodec.AV_CODEC_ID_FLASHSV2), + RV10(avcodec.AV_CODEC_ID_RV10), + RV20(avcodec.AV_CODEC_ID_RV20), + RV30(avcodec.AV_CODEC_ID_RV30), + RV40(avcodec.AV_CODEC_ID_RV40), + CINEPAK(avcodec.AV_CODEC_ID_CINEPAK), + INDEO2(avcodec.AV_CODEC_ID_INDEO2), + INDEO3(avcodec.AV_CODEC_ID_INDEO3), + INDEO4(avcodec.AV_CODEC_ID_INDEO4), + INDEO5(avcodec.AV_CODEC_ID_INDEO5), + MSMPEG4V1(avcodec.AV_CODEC_ID_MSMPEG4V1), + MSMPEG4V2(avcodec.AV_CODEC_ID_MSMPEG4V2), + MSMPEG4V3(avcodec.AV_CODEC_ID_MSMPEG4V3), + H261(avcodec.AV_CODEC_ID_H261), + H263(avcodec.AV_CODEC_ID_H263), + H263I(avcodec.AV_CODEC_ID_H263I), + H263P(avcodec.AV_CODEC_ID_H263P), + SNOW(avcodec.AV_CODEC_ID_SNOW), + SVQ1(avcodec.AV_CODEC_ID_SVQ1), + SVQ3(avcodec.AV_CODEC_ID_SVQ3), + DVVIDEO(avcodec.AV_CODEC_ID_DVVIDEO), + HUFFYUV(avcodec.AV_CODEC_ID_HUFFYUV), + FFVHUFF(avcodec.AV_CODEC_ID_FFVHUFF), + FFV1(avcodec.AV_CODEC_ID_FFV1), + ASV1(avcodec.AV_CODEC_ID_ASV1), + ASV2(avcodec.AV_CODEC_ID_ASV2), + VCR1(avcodec.AV_CODEC_ID_VCR1), + CLJR(avcodec.AV_CODEC_ID_CLJR), + MDEC(avcodec.AV_CODEC_ID_MDEC), + ROQ(avcodec.AV_CODEC_ID_ROQ), + INTERPLAY_VIDEO(avcodec.AV_CODEC_ID_INTERPLAY_VIDEO), + XAN_WC3(avcodec.AV_CODEC_ID_XAN_WC3), + XAN_WC4(avcodec.AV_CODEC_ID_XAN_WC4), + RPZA(avcodec.AV_CODEC_ID_RPZA), + SMC(avcodec.AV_CODEC_ID_SMC), + GIF(avcodec.AV_CODEC_ID_GIF), + RAWVIDEO(avcodec.AV_CODEC_ID_RAWVIDEO), + PNG(avcodec.AV_CODEC_ID_PNG), + PPM(avcodec.AV_CODEC_ID_PPM), + PBM(avcodec.AV_CODEC_ID_PBM), + PGM(avcodec.AV_CODEC_ID_PGM), + PAM(avcodec.AV_CODEC_ID_PAM), + BMP(avcodec.AV_CODEC_ID_BMP), + TIFF(avcodec.AV_CODEC_ID_TIFF), + SGI(avcodec.AV_CODEC_ID_SGI), + ALIAS_PIX(avcodec.AV_CODEC_ID_ALIAS_PIX), + DPX(avcodec.AV_CODEC_ID_DPX), + EXR(avcodec.AV_CODEC_ID_EXR), + WEBP(avcodec.AV_CODEC_ID_WEBP), + DIRAC(avcodec.AV_CODEC_ID_DIRAC), + DNXHD(avcodec.AV_CODEC_ID_DNXHD), + PRORES(avcodec.AV_CODEC_ID_PRORES), + JPEG2000(avcodec.AV_CODEC_ID_JPEG2000), + JPEGLS(avcodec.AV_CODEC_ID_JPEGLS), + HAP(avcodec.AV_CODEC_ID_HAP); + + public int ffmpegId; + + FullCodecEnum(int ffmpegId) + { + this.ffmpegId = ffmpegId; + } +} diff --git a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/helpers/FullPixelEnum.java b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/helpers/FullPixelEnum.java new file mode 100644 index 000000000..69effa6f7 --- /dev/null +++ b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/helpers/FullPixelEnum.java @@ -0,0 +1,189 @@ +package org.sensorhub.impl.process.video.transcoder.helpers; + +import org.bytedeco.ffmpeg.global.avutil; + +public enum FullPixelEnum { + NONE(avutil.AV_PIX_FMT_NONE), + + // --- Planar YUV --- + YUV420P(avutil.AV_PIX_FMT_YUV420P), + YUV422P(avutil.AV_PIX_FMT_YUV422P), + YUV444P(avutil.AV_PIX_FMT_YUV444P), + YUV410P(avutil.AV_PIX_FMT_YUV410P), + YUV411P(avutil.AV_PIX_FMT_YUV411P), + YUV440P(avutil.AV_PIX_FMT_YUV440P), + YUVJ420P(avutil.AV_PIX_FMT_YUVJ420P), + YUVJ422P(avutil.AV_PIX_FMT_YUVJ422P), + YUVJ444P(avutil.AV_PIX_FMT_YUVJ444P), + YUVJ440P(avutil.AV_PIX_FMT_YUVJ440P), + YUV420P9BE(avutil.AV_PIX_FMT_YUV420P9BE), + YUV420P9LE(avutil.AV_PIX_FMT_YUV420P9LE), + YUV420P10BE(avutil.AV_PIX_FMT_YUV420P10BE), + YUV420P10LE(avutil.AV_PIX_FMT_YUV420P10LE), + YUV420P12BE(avutil.AV_PIX_FMT_YUV420P12BE), + YUV420P12LE(avutil.AV_PIX_FMT_YUV420P12LE), + YUV420P14BE(avutil.AV_PIX_FMT_YUV420P14BE), + YUV420P14LE(avutil.AV_PIX_FMT_YUV420P14LE), + YUV420P16BE(avutil.AV_PIX_FMT_YUV420P16BE), + YUV420P16LE(avutil.AV_PIX_FMT_YUV420P16LE), + YUV422P9BE(avutil.AV_PIX_FMT_YUV422P9BE), + YUV422P9LE(avutil.AV_PIX_FMT_YUV422P9LE), + YUV422P10BE(avutil.AV_PIX_FMT_YUV422P10BE), + YUV422P10LE(avutil.AV_PIX_FMT_YUV422P10LE), + YUV422P12BE(avutil.AV_PIX_FMT_YUV422P12BE), + YUV422P12LE(avutil.AV_PIX_FMT_YUV422P12LE), + YUV422P14BE(avutil.AV_PIX_FMT_YUV422P14BE), + YUV422P14LE(avutil.AV_PIX_FMT_YUV422P14LE), + YUV422P16BE(avutil.AV_PIX_FMT_YUV422P16BE), + YUV422P16LE(avutil.AV_PIX_FMT_YUV422P16LE), + YUV444P9BE(avutil.AV_PIX_FMT_YUV444P9BE), + YUV444P9LE(avutil.AV_PIX_FMT_YUV444P9LE), + YUV444P10BE(avutil.AV_PIX_FMT_YUV444P10BE), + YUV444P10LE(avutil.AV_PIX_FMT_YUV444P10LE), + YUV444P12BE(avutil.AV_PIX_FMT_YUV444P12BE), + YUV444P12LE(avutil.AV_PIX_FMT_YUV444P12LE), + YUV444P14BE(avutil.AV_PIX_FMT_YUV444P14BE), + YUV444P14LE(avutil.AV_PIX_FMT_YUV444P14LE), + YUV444P16BE(avutil.AV_PIX_FMT_YUV444P16BE), + YUV444P16LE(avutil.AV_PIX_FMT_YUV444P16LE), + + // --- Packed YUV --- + YUYV422(avutil.AV_PIX_FMT_YUYV422), + UYVY422(avutil.AV_PIX_FMT_UYVY422), + YVYU422(avutil.AV_PIX_FMT_YVYU422), + UYYVYY411(avutil.AV_PIX_FMT_UYYVYY411), + NV12(avutil.AV_PIX_FMT_NV12), + NV21(avutil.AV_PIX_FMT_NV21), + NV16(avutil.AV_PIX_FMT_NV16), + NV20LE(avutil.AV_PIX_FMT_NV20LE), + NV20BE(avutil.AV_PIX_FMT_NV20BE), + NV24(avutil.AV_PIX_FMT_NV24), + NV42(avutil.AV_PIX_FMT_NV42), + + // --- RGB --- + RGB24(avutil.AV_PIX_FMT_RGB24), + BGR24(avutil.AV_PIX_FMT_BGR24), + ARGB(avutil.AV_PIX_FMT_ARGB), + RGBA(avutil.AV_PIX_FMT_RGBA), + ABGR(avutil.AV_PIX_FMT_ABGR), + BGRA(avutil.AV_PIX_FMT_BGRA), + RGB0(avutil.AV_PIX_FMT_RGB0), + BGR0(avutil.AV_PIX_FMT_BGR0), + RGB8(avutil.AV_PIX_FMT_RGB8), + BGR8(avutil.AV_PIX_FMT_BGR8), + RGB4(avutil.AV_PIX_FMT_RGB4), + BGR4(avutil.AV_PIX_FMT_BGR4), + RGB4_BYTE(avutil.AV_PIX_FMT_RGB4_BYTE), + BGR4_BYTE(avutil.AV_PIX_FMT_BGR4_BYTE), + RGB48BE(avutil.AV_PIX_FMT_RGB48BE), + RGB48LE(avutil.AV_PIX_FMT_RGB48LE), + RGBA64BE(avutil.AV_PIX_FMT_RGBA64BE), + RGBA64LE(avutil.AV_PIX_FMT_RGBA64LE), + BGR48BE(avutil.AV_PIX_FMT_BGR48BE), + BGR48LE(avutil.AV_PIX_FMT_BGR48LE), + BGRA64BE(avutil.AV_PIX_FMT_BGRA64BE), + BGRA64LE(avutil.AV_PIX_FMT_BGRA64LE), + RGB565BE(avutil.AV_PIX_FMT_RGB565BE), + RGB565LE(avutil.AV_PIX_FMT_RGB565LE), + RGB555BE(avutil.AV_PIX_FMT_RGB555BE), + RGB555LE(avutil.AV_PIX_FMT_RGB555LE), + RGB444BE(avutil.AV_PIX_FMT_RGB444BE), + RGB444LE(avutil.AV_PIX_FMT_RGB444LE), + BGR565BE(avutil.AV_PIX_FMT_BGR565BE), + BGR565LE(avutil.AV_PIX_FMT_BGR565LE), + BGR555BE(avutil.AV_PIX_FMT_BGR555BE), + BGR555LE(avutil.AV_PIX_FMT_BGR555LE), + BGR444BE(avutil.AV_PIX_FMT_BGR444BE), + BGR444LE(avutil.AV_PIX_FMT_BGR444LE), + + // --- Grayscale --- + GRAY8(avutil.AV_PIX_FMT_GRAY8), + GRAY8A(avutil.AV_PIX_FMT_GRAY8A), + GRAY9BE(avutil.AV_PIX_FMT_GRAY9BE), + GRAY9LE(avutil.AV_PIX_FMT_GRAY9LE), + GRAY10BE(avutil.AV_PIX_FMT_GRAY10BE), + GRAY10LE(avutil.AV_PIX_FMT_GRAY10LE), + GRAY12BE(avutil.AV_PIX_FMT_GRAY12BE), + GRAY12LE(avutil.AV_PIX_FMT_GRAY12LE), + GRAY14BE(avutil.AV_PIX_FMT_GRAY14BE), + GRAY14LE(avutil.AV_PIX_FMT_GRAY14LE), + GRAY16BE(avutil.AV_PIX_FMT_GRAY16BE), + GRAY16LE(avutil.AV_PIX_FMT_GRAY16LE), + MONOWHITE(avutil.AV_PIX_FMT_MONOWHITE), + MONOBLACK(avutil.AV_PIX_FMT_MONOBLACK), + + // --- YA --- + YA8(avutil.AV_PIX_FMT_YA8), + YA16BE(avutil.AV_PIX_FMT_YA16BE), + YA16LE(avutil.AV_PIX_FMT_YA16LE), + + // --- YUVA --- + YUVA420P(avutil.AV_PIX_FMT_YUVA420P), + YUVA422P(avutil.AV_PIX_FMT_YUVA422P), + YUVA444P(avutil.AV_PIX_FMT_YUVA444P), + YUVA420P9BE(avutil.AV_PIX_FMT_YUVA420P9BE), + YUVA420P9LE(avutil.AV_PIX_FMT_YUVA420P9LE), + YUVA422P9BE(avutil.AV_PIX_FMT_YUVA422P9BE), + YUVA422P9LE(avutil.AV_PIX_FMT_YUVA422P9LE), + YUVA444P9BE(avutil.AV_PIX_FMT_YUVA444P9BE), + YUVA444P9LE(avutil.AV_PIX_FMT_YUVA444P9LE), + YUVA420P10BE(avutil.AV_PIX_FMT_YUVA420P10BE), + YUVA420P10LE(avutil.AV_PIX_FMT_YUVA420P10LE), + YUVA422P10BE(avutil.AV_PIX_FMT_YUVA422P10BE), + YUVA422P10LE(avutil.AV_PIX_FMT_YUVA422P10LE), + YUVA444P10BE(avutil.AV_PIX_FMT_YUVA444P10BE), + YUVA444P10LE(avutil.AV_PIX_FMT_YUVA444P10LE), + YUVA420P16BE(avutil.AV_PIX_FMT_YUVA420P16BE), + YUVA420P16LE(avutil.AV_PIX_FMT_YUVA420P16LE), + YUVA422P16BE(avutil.AV_PIX_FMT_YUVA422P16BE), + YUVA422P16LE(avutil.AV_PIX_FMT_YUVA422P16LE), + YUVA444P16BE(avutil.AV_PIX_FMT_YUVA444P16BE), + YUVA444P16LE(avutil.AV_PIX_FMT_YUVA444P16LE), + + // --- Bayer --- + BAYER_BGGR8(avutil.AV_PIX_FMT_BAYER_BGGR8), + BAYER_RGGB8(avutil.AV_PIX_FMT_BAYER_RGGB8), + BAYER_GBRG8(avutil.AV_PIX_FMT_BAYER_GBRG8), + BAYER_GRBG8(avutil.AV_PIX_FMT_BAYER_GRBG8), + BAYER_BGGR16LE(avutil.AV_PIX_FMT_BAYER_BGGR16LE), + BAYER_BGGR16BE(avutil.AV_PIX_FMT_BAYER_BGGR16BE), + BAYER_RGGB16LE(avutil.AV_PIX_FMT_BAYER_RGGB16LE), + BAYER_RGGB16BE(avutil.AV_PIX_FMT_BAYER_RGGB16BE), + BAYER_GBRG16LE(avutil.AV_PIX_FMT_BAYER_GBRG16LE), + BAYER_GBRG16BE(avutil.AV_PIX_FMT_BAYER_GBRG16BE), + BAYER_GRBG16LE(avutil.AV_PIX_FMT_BAYER_GRBG16LE), + BAYER_GRBG16BE(avutil.AV_PIX_FMT_BAYER_GRBG16BE), + + // --- Floating Point --- + GRAYF32BE(avutil.AV_PIX_FMT_GRAYF32BE), + GRAYF32LE(avutil.AV_PIX_FMT_GRAYF32LE), + + // --- Palette --- + PAL8(avutil.AV_PIX_FMT_PAL8), + + // --- XYZ --- + XYZ12LE(avutil.AV_PIX_FMT_XYZ12LE), + XYZ12BE(avutil.AV_PIX_FMT_XYZ12BE), + + // --- Miscellaneous --- + XTOP(avutil.AV_PIX_FMT_X2RGB10LE), + P010LE(avutil.AV_PIX_FMT_P010LE), + P010BE(avutil.AV_PIX_FMT_P010BE), + P016LE(avutil.AV_PIX_FMT_P016LE), + P016BE(avutil.AV_PIX_FMT_P016BE), + P210BE(avutil.AV_PIX_FMT_P210BE), + P210LE(avutil.AV_PIX_FMT_P210LE), + P410BE(avutil.AV_PIX_FMT_P410BE), + P410LE(avutil.AV_PIX_FMT_P410LE), + P216BE(avutil.AV_PIX_FMT_P216BE), + P216LE(avutil.AV_PIX_FMT_P216LE), + P416BE(avutil.AV_PIX_FMT_P416BE), + P416LE(avutil.AV_PIX_FMT_P416LE); + + public int ffmpegId; + + FullPixelEnum(int ffmpegId) + { + this.ffmpegId = ffmpegId; + } +} From 800a640d78116bdf0afc8e6f0215168cb6868279 Mon Sep 17 00:00:00 2001 From: kyle-fitzp Date: Thu, 23 Apr 2026 13:48:35 -0500 Subject: [PATCH 02/10] Change CodecInfo, CodecOptions to records --- .../video/transcoder/FFMpegTranscoder.java | 14 +++--- .../video/transcoder/coders/Coder.java | 17 +++---- .../video/transcoder/coders/Decoder.java | 4 +- .../video/transcoder/coders/Encoder.java | 4 +- .../video/transcoder/coders/SwScaler.java | 4 +- .../video/transcoder/helpers/CodecInfo.java | 36 +++++---------- .../transcoder/helpers/CodecOptions.java | 44 +++---------------- 7 files changed, 41 insertions(+), 82 deletions(-) diff --git a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/FFMpegTranscoder.java b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/FFMpegTranscoder.java index 0e02c8a73..4df94fd0b 100644 --- a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/FFMpegTranscoder.java +++ b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/FFMpegTranscoder.java @@ -323,8 +323,8 @@ public void init() throws ProcessException { //inCodec = CodecEnum.valueOf(inCodecParam.getData().getStringValue().toUpperCase()); //outCodec = CodecEnum.valueOf(outCodecParam.getData().getStringValue().toUpperCase()); - inCodec = new CodecInfo(inCodecParam.getData().getStringValue()); - outCodec = new CodecInfo(outCodecParam.getData().getStringValue()); + inCodec = CodecInfo.newCodecInfoFromName(inCodecParam.getData().getStringValue()); + outCodec = CodecInfo.newCodecInfoFromName(outCodecParam.getData().getStringValue()); setImgEncoding(); initCodecOptions(); @@ -376,9 +376,11 @@ private void initCodecOptions() { } decOptions = decOptionBuilder.setFps(fps).setBitRate(bitrate) - .setWidth(width).setHeight(height).presetUltraFast().tuneZeroLatency().build(); + .setWidth(width).setHeight(height).presetUltraFast().tuneZeroLatency() + .setComplianceUnofficial().build(); encOptions = encOptionBuilder.setFps(fps).setBitRate(bitrate) - .setWidth(outWidth).setHeight(outHeight).presetUltraFast().tuneZeroLatency().build(); + .setWidth(outWidth).setHeight(outHeight).presetUltraFast().tuneZeroLatency() + .setComplianceUnofficial().build(); } /** @@ -393,7 +395,7 @@ private void initCodecOptions() { private AVByteFormatter getFormatter(CodecInfo codec, @Nullable Integer width, @Nullable Integer height) throws ProcessException { try { int pixFmt; - if ((pixFmt = codec.getPixelFmt().ffmpegId) != AV_PIX_FMT_NONE) + if ((pixFmt = codec.pixelFmt().ffmpegId) != AV_PIX_FMT_NONE) return new FrameFormatter(width, height, pixFmt); else return new PacketFormatter(); @@ -415,7 +417,7 @@ private AVByteFormatter getFormatter(CodecInfo codec, @Nullable Integer width, @ * @return Is the codec {@link CodecEnum#RGB} or {@link CodecEnum#YUV}? */ private boolean isUncompressed(CodecInfo codec) { - return codec.getCodec() == FullCodecEnum.RAWVIDEO; + return codec.codec() == FullCodecEnum.RAWVIDEO; } /** diff --git a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/coders/Coder.java b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/coders/Coder.java index e2009fef8..363fd9910 100644 --- a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/coders/Coder.java +++ b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/coders/Coder.java @@ -62,6 +62,7 @@ public Coder(CodecInfo inFormatInfo, CodecInfo outFormatInfo, Class inputClas this.inputFormat = inFormatInfo; this.inputClass = inputClass; + this.outputFormat = outFormatInfo; this.outputClass = outputClass; this.options = options; @@ -94,16 +95,16 @@ protected void openContext() { */ protected void initOptions() { - codec_ctx.time_base(av_make_q(1, options.getFps())); + codec_ctx.time_base(av_make_q(1, options.fps())); - if (options.getBitRate() > 0) { - codec_ctx.bit_rate(options.getBitRate() * 1000); + if (options.bitRate() > 0) { + codec_ctx.bit_rate(options.bitRate() * 1000); } else { //codec_ctx.bit_rate(150*1000); } - codec_ctx.width(options.getWidth()); - codec_ctx.height(options.getHeight()); + codec_ctx.width(options.width()); + codec_ctx.height(options.height()); /* if (options.containsKey("pix_fmt")) { @@ -118,9 +119,9 @@ protected void initOptions() { */ - av_opt_set(codec_ctx.priv_data(), "preset", "ultrafast", 0); - av_opt_set(codec_ctx.priv_data(), "tune", "zerolatency", 0); - codec_ctx.strict_std_compliance(FF_COMPLIANCE_UNOFFICIAL); // Needed so that yuvj420p works (used for mjpeg) + av_opt_set(codec_ctx.priv_data(), "preset", options.preset(), 0); + av_opt_set(codec_ctx.priv_data(), "tune", options.tune(), 0); + codec_ctx.strict_std_compliance(options.compliance()); // Needed so that yuvj420p works (used for mjpeg) } protected abstract void deallocateInputPacket(I packet); diff --git a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/coders/Decoder.java b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/coders/Decoder.java index 9adc2a778..13de8f7cc 100644 --- a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/coders/Decoder.java +++ b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/coders/Decoder.java @@ -25,9 +25,9 @@ public Decoder(CodecInfo inFormatInfo, CodecInfo outFormatInfo, CodecOptions opt @Override protected void initContext() { synchronized (contextLock) { - codec = avcodec_find_decoder(inputFormat.getCodec().ffmpegId);; + codec = avcodec_find_decoder(inputFormat.codec().ffmpegId);; codec_ctx = avcodec_alloc_context3(codec); - codec_ctx.pix_fmt(outputFormat.getPixelFmt().ffmpegId); + codec_ctx.pix_fmt(outputFormat.pixelFmt().ffmpegId); } } diff --git a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/coders/Encoder.java b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/coders/Encoder.java index 1cacaedd5..1e514577a 100644 --- a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/coders/Encoder.java +++ b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/coders/Encoder.java @@ -27,11 +27,11 @@ public Encoder(CodecInfo inFormatInfo, CodecInfo outFormatInfo, CodecOptions opt @Override protected void initContext() { synchronized (contextLock) { - AVCodec codec = avcodec_find_encoder(outputFormat.getCodec().ffmpegId);; + AVCodec codec = avcodec_find_encoder(outputFormat.codec().ffmpegId);; codec_ctx = avcodec_alloc_context3(codec); //initOptions(codec_ctx); - codec_ctx.pix_fmt(inputFormat.getPixelFmt().ffmpegId); + codec_ctx.pix_fmt(inputFormat.pixelFmt().ffmpegId); } } diff --git a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/coders/SwScaler.java b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/coders/SwScaler.java index 343172662..8a8731d73 100644 --- a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/coders/SwScaler.java +++ b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/coders/SwScaler.java @@ -32,8 +32,8 @@ public SwScaler(CodecInfo inputFormat, CodecInfo outputFormat, int inWidth, int @Override protected void initContext() { - swsContext = sws_getContext(inWidth, inHeight, inputFormat.getPixelFmt().ffmpegId, - outWidth, outHeight, outputFormat.getPixelFmt().ffmpegId, + swsContext = sws_getContext(inWidth, inHeight, inputFormat.pixelFmt().ffmpegId, + outWidth, outHeight, outputFormat.pixelFmt().ffmpegId, SWS_BICUBIC, null, null, (DoublePointer) null); } diff --git a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/helpers/CodecInfo.java b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/helpers/CodecInfo.java index 0e0315bf3..f27102bf3 100644 --- a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/helpers/CodecInfo.java +++ b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/helpers/CodecInfo.java @@ -1,34 +1,20 @@ package org.sensorhub.impl.process.video.transcoder.helpers; -public class CodecInfo { - private FullCodecEnum codec; - private FullPixelEnum pixelFmt; - - public CodecInfo(String name) { +public record CodecInfo(FullCodecEnum codec, FullPixelEnum pixelFmt) { + public static CodecInfo newCodecInfoFromName(String name) { + FullCodecEnum codec; + FullPixelEnum pixel; try { - this.codec = Enum.valueOf(FullCodecEnum.class, name); - // YUVJ works best here - if (this.codec == FullCodecEnum.MJPEG) - this.pixelFmt = FullPixelEnum.YUVJ420P; + codec = Enum.valueOf(FullCodecEnum.class, name); + if (codec == FullCodecEnum.MJPEG) + pixel = FullPixelEnum.YUVJ420P; else - this.pixelFmt = FullPixelEnum.YUV420P; - + pixel = FullPixelEnum.YUV420P; } catch (Exception e) { - this.codec = FullCodecEnum.RAWVIDEO; - this.pixelFmt = Enum.valueOf(FullPixelEnum.class, name); + codec = FullCodecEnum.RAWVIDEO; + pixel = Enum.valueOf(FullPixelEnum.class, name); } - } - - public CodecInfo(FullCodecEnum codec, FullPixelEnum pixelFmt) { - this.codec = codec; - this.pixelFmt = pixelFmt; - } - - public FullCodecEnum getCodec() { - return codec; - } - public FullPixelEnum getPixelFmt() { - return pixelFmt; + return new CodecInfo(codec, pixel); } } diff --git a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/helpers/CodecOptions.java b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/helpers/CodecOptions.java index 78e8ff375..64e0771b3 100644 --- a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/helpers/CodecOptions.java +++ b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/helpers/CodecOptions.java @@ -1,15 +1,7 @@ package org.sensorhub.impl.process.video.transcoder.helpers; import static org.bytedeco.ffmpeg.global.avcodec.*; -public class CodecOptions { - private int fps; - private int bitRate; - private int width; - private int height; - private int compliance; - private String preset; - private String tune; - +public record CodecOptions(int fps, int bitRate, int width, int height, int compliance, String preset, String tune) { public CodecOptions(int fps, int bitRate, int width, int height, int compliance, String preset, String tune) { this.fps = Math.max(fps, 1); this.bitRate = bitRate; @@ -20,34 +12,6 @@ public CodecOptions(int fps, int bitRate, int width, int height, int compliance, this.tune = tune; } - public int getFps() { - return fps; - } - - public int getBitRate() { - return bitRate; - } - - public int getWidth() { - return width; - } - - public int getHeight() { - return height; - } - - public int getCompliance() { - return compliance; - } - - public String getPreset() { - return preset; - } - - public String getTune() { - return tune; - } - public static class Builder { private int fps = 30; private int bitRate = -1; @@ -86,6 +50,12 @@ public Builder setCompliance(int compliance) { return this; } + // Convenience, unofficial provides widest range of pixel formats + public Builder setComplianceUnofficial() { + this.compliance = FF_COMPLIANCE_UNOFFICIAL; + return this; + } + public Builder setPreset(String preset) { this.preset = preset; return this; From 9a92a7ad33a13124dea5be011d7584982c98c144 Mon Sep 17 00:00:00 2001 From: kyle-fitzp Date: Thu, 23 Apr 2026 13:53:40 -0500 Subject: [PATCH 03/10] Replace asserts with proper checks and exceptions --- .../impl/process/video/transcoder/coders/Coder.java | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/coders/Coder.java b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/coders/Coder.java index 363fd9910..7c5201780 100644 --- a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/coders/Coder.java +++ b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/coders/Coder.java @@ -46,7 +46,7 @@ public interface CoderCallback { protected I inPacket; protected O outPacket; protected final Queue outQueue = new ArrayDeque<>(10); - private AtomicBoolean isProcessing = new AtomicBoolean(true); // Set false to indicate packets should no longer be accepted + private final AtomicBoolean isProcessing = new AtomicBoolean(true); // Set false to indicate packets should no longer be accepted final Object contextLock = new Object(); Class inputClass; Class outputClass; @@ -56,9 +56,13 @@ public interface CoderCallback { public Coder(CodecInfo inFormatInfo, CodecInfo outFormatInfo, Class inputClass, Class outputClass, CodecOptions options) { super(); - assert inputClass == AVPacket.class || inputClass == AVFrame.class; - assert outputClass == AVPacket.class || outputClass == AVFrame.class; - assert options != null; + if ((inputClass != AVPacket.class && inputClass != AVFrame.class) + || (outputClass != AVPacket.class && outputClass != AVFrame.class)) { + throw new IllegalArgumentException("Input and output classes must be either AVPacket or AVFrame"); + } + + if (options == null) + throw new IllegalArgumentException("Options cannot be null"); this.inputFormat = inFormatInfo; this.inputClass = inputClass; From 4932a7ce8393832bed3298d3e1480adc91246fd1 Mon Sep 17 00:00:00 2001 From: kyle-fitzp Date: Wed, 29 Apr 2026 13:01:53 -0500 Subject: [PATCH 04/10] Minor fixes --- .../impl/process/video/FFmpegProcess.java | 3 +- .../video/transcoder/FFMpegTranscoder.java | 94 ++++++++------ .../video/transcoder/coders/Coder.java | 122 +++++++++++++----- .../video/transcoder/coders/Decoder.java | 19 ++- .../video/transcoder/coders/Encoder.java | 51 +++++++- .../video/transcoder/coders/SwScaler.java | 3 + .../video/transcoder/helpers/CodecInfo.java | 5 +- .../transcoder/helpers/FullCodecEnum.java | 3 +- 8 files changed, 206 insertions(+), 94 deletions(-) diff --git a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/FFmpegProcess.java b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/FFmpegProcess.java index e6fe0a6be..77193644e 100644 --- a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/FFmpegProcess.java +++ b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/FFmpegProcess.java @@ -246,6 +246,7 @@ public void doStart() throws SensorException, SensorHubException { // Reinit executable. Not always necessary, but doesn't hurt. executable.init(); process.start(this::onError); + //process.start(this::onError); } catch (ProcessException e) { logger.error("Could not initialize process.", e); return; @@ -459,7 +460,7 @@ public void publishData() { } - // Listens for event from dara + // Listens for event from data protected class DataQueuePusher implements IEventListener { DataQueue dataQueue; diff --git a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/FFMpegTranscoder.java b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/FFMpegTranscoder.java index 4df94fd0b..6123923f3 100644 --- a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/FFMpegTranscoder.java +++ b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/FFMpegTranscoder.java @@ -59,8 +59,7 @@ public class FFMpegTranscoder extends ExecutableProcessImpl public static final OSHProcessInfo INFO = new OSHProcessInfo("video:FFMpegTranscoder", "FFMPEG Video Transcoder", null, FFMpegTranscoder.class); - AtomicBoolean doRun = new AtomicBoolean(true); - AtomicBoolean isRunning = new AtomicBoolean(false); + AtomicBoolean isInit = new AtomicBoolean(false); Time inputTimeStamp; Count inputWidth, inputHeight; DataArray imgIn; @@ -159,11 +158,6 @@ private void initFormatters() throws ProcessException { * queue in the order they should process the incoming data. At most, the flow will be Decoder -> SWScale -> Encoder. */ private void initCoders() { - try { - stopProcessing(); - } catch (Exception e){ - logger.error("Transcoder could not stop process threads during re-init.", e); - } if (videoProcs != null) { videoProcs.clear(); } @@ -197,33 +191,18 @@ private void initCoders() { } - /** - * Invoked during the first call to {@link FFMpegTranscoder#execute()} (when {@link FFMpegTranscoder#isRunning} is false). - * Start all {@link Coder} thread objects stored in {@link FFMpegTranscoder#videoProcs}. - */ - private void startProcessThreads() { - doRun.set(true); - if (videoProcs == null || videoProcs.isEmpty()) { - initCoders(); - } - - isRunning.set(true); - } - /** * Invoked on process stop and init. * Stop all {@link Coder} thread objects stored in {@link FFMpegTranscoder#videoProcs}. */ private void stopProcessing() throws InterruptedException { - doRun.set(false); //TODO These atomic booleans may be entirely unnecessary, remove + isInit.set(false); if (videoProcs != null) { for (Coder codec : videoProcs) { codec.close(); } videoProcs.clear(); } - - isRunning.set(false); } @@ -259,12 +238,10 @@ public void notifyParamChange() @Override public void stop() { - if (isRunning.get()) { - try { - stopProcessing(); - } catch (InterruptedException e) { - logger.warn("Interrupted while stopping process threads"); - } + try { + stopProcessing(); + } catch (InterruptedException e) { + logger.warn("Interrupted while stopping process threads"); } super.stop(); } @@ -314,8 +291,9 @@ private void setImgEncoding() { @Override public void init() throws ProcessException { - doRun.set(true); - isRunning.set(false); + if (!isInit.compareAndSet(false, true)) { + return; + } // init decoder according to configured codec // TODO: Automatically detect input codec from compression in data struct? @@ -335,7 +313,7 @@ public void init() throws ProcessException } catch (IllegalArgumentException e) { - reportError("Unsupported codec" + inCodecParam.getData().getStringValue() + ". Must be one of " + Arrays.toString(CodecEnum.values())); + reportError("Unsupported codec " + inCodecParam.getData().getStringValue() + ". Must be one of " + Arrays.toString(CodecEnum.values())); } super.init(); @@ -352,16 +330,33 @@ private void initCodecOptions() { int fps = safeGetCountVal(inputFps); int bitrate = safeGetCountVal(inputBitrate); width = safeGetCountVal(inputWidth); + if (width <= 0) { + try { + width = imgIn.getComponent("row").getComponentCount(); + } catch (Exception e) { + width = 1920; + logger.warn("Input width not specified, using default: 1920", e); + } + } + height = safeGetCountVal(inputHeight); + if (height <= 0) { + try { + height = imgIn.getComponentCount(); + } catch (Exception e) { + height = 1080; + logger.warn("Input height not specified, using default: 1080", e); + } + } outHeight = safeGetCountVal(outputHeight); if (outHeight <= 0) { try { outHeight = imgOut.getComponentCount(); - } catch (Exception ignored) { + } catch (Exception e) { outHeight = height; - logger.warn("Output height not specified, using input height"); + logger.warn("Output height not specified, using input height", e); } } @@ -369,9 +364,9 @@ private void initCodecOptions() { if (outWidth <= 0) { try { outWidth = imgIn.getComponent("row").getComponentCount(); - } catch (Exception ignored) { + } catch (Exception e) { outWidth = width; - logger.warn("Output width not specified, using input width"); + logger.warn("Output width not specified, using input width", e); } } @@ -394,9 +389,8 @@ private void initCodecOptions() { */ private AVByteFormatter getFormatter(CodecInfo codec, @Nullable Integer width, @Nullable Integer height) throws ProcessException { try { - int pixFmt; - if ((pixFmt = codec.pixelFmt().ffmpegId) != AV_PIX_FMT_NONE) - return new FrameFormatter(width, height, pixFmt); + if (isUncompressed(codec)) + return new FrameFormatter(width, height, codec.pixelFmt().ffmpegId); else return new PacketFormatter(); /* @@ -407,7 +401,7 @@ private AVByteFormatter getFormatter(CodecInfo codec, @Nullable Integer width, @ }; */ } catch (NullPointerException e) { - reportError("Raw formatter for " + codec + " requires non-null width and height.", e); + reportError("Formatter for " + codec + " requires non-null width and height.", e); } return null; } @@ -478,8 +472,13 @@ public void execute() throws ProcessException } // Start the threads if not already started - if (!isRunning.get()) { - startProcessThreads(); + if (!isInit.get()) { + init(); + } + + if (!isVideoProcChainReady()) { + logger.warn("Video processor not ready"); + return; } videoProcs.get(0).submitInputPacket( @@ -487,6 +486,15 @@ public void execute() throws ProcessException ); } + private boolean isVideoProcChainReady() { + for (Coder proc : videoProcs) { + if (!proc.isReady()) { + return false; + } + } + return true; + } + @Override protected void publishData() throws InterruptedException { @@ -499,7 +507,7 @@ public void dispose() { super.dispose(); - doRun.set(false); + isInit.set(false); if (videoProcs != null) { for (Coder proc : videoProcs) { diff --git a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/coders/Coder.java b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/coders/Coder.java index 7c5201780..81347fb3e 100644 --- a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/coders/Coder.java +++ b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/coders/Coder.java @@ -6,8 +6,10 @@ import java.util.HashMap; import java.util.Map; import java.util.Queue; +import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import static org.bytedeco.ffmpeg.global.avcodec.*; @@ -16,6 +18,7 @@ import org.bytedeco.ffmpeg.avcodec.AVCodecContext; import org.bytedeco.ffmpeg.avcodec.AVPacket; import org.bytedeco.ffmpeg.avutil.AVFrame; +import org.bytedeco.javacpp.BytePointer; import org.bytedeco.javacpp.Pointer; import org.bytedeco.javacpp.PointerPointer; import org.sensorhub.impl.process.video.transcoder.helpers.CodecInfo; @@ -35,7 +38,8 @@ public interface CoderCallback { private static int coderCount = 0; private final int coderNum = coderCount++; - private final ExecutorService submitExecutor = Executors.newSingleThreadExecutor(r -> new Thread(r, "ffmpeg-codec-thread-" + coderNum)); + private final ExecutorService executor = Executors.newSingleThreadExecutor( + r -> new Thread(r, "ffmpeg-codec-thread-" + coderNum)); private final ExecutorService outputExecutor = Executors.newSingleThreadExecutor(r -> new Thread(r, "ffmpeg-codec-output-thread-" + coderNum)); private final Map, ExecutorService> callbackMap = new HashMap<>(); @@ -46,7 +50,7 @@ public interface CoderCallback { protected I inPacket; protected O outPacket; protected final Queue outQueue = new ArrayDeque<>(10); - private final AtomicBoolean isProcessing = new AtomicBoolean(true); // Set false to indicate packets should no longer be accepted + private final AtomicBoolean isProcessing = new AtomicBoolean(false); // Set false to indicate packets should no longer be accepted final Object contextLock = new Object(); Class inputClass; Class outputClass; @@ -70,11 +74,39 @@ public Coder(CodecInfo inFormatInfo, CodecInfo outFormatInfo, Class inputClas this.outputClass = outputClass; this.options = options; - submitExecutor.submit(() -> { - initContext(); - initOptions(); - openContext(); - }); + // All codec operations must happen in a separate thread + try { + executor.submit(() -> { + initContext(); + initOptions(); + openContext(); + isProcessing.set(true); + }).get(); // blocks constructor until init is complete + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException("Interrupted during codec initialization", e); + } catch (ExecutionException e) { + throw new RuntimeException("Error initializing codec context", e.getCause()); + } + } + + private void submitTask(Runnable task) { + if (!executor.isShutdown()) { + executor.submit(() -> { + try { + task.run(); + } catch (Throwable t) { + // Prevent silent thread replacement — log and propagate + // only after codec state is consistent + logger.error("Fatal error in codec runner thread", t); + //throw t; // will kill this task but not spawn a new thread silently + } + }); + } + } + + public boolean isReady() { + return isProcessing.get(); } public Class getOutputClass() { @@ -88,14 +120,22 @@ public Class getInputClass() { protected abstract void initContext(); protected void openContext() { - if (avcodec_open2(codec_ctx, codec, (PointerPointer) null) < 0) { + int ret; + if ((ret = avcodec_open2(codec_ctx, codec, (PointerPointer) null)) < 0) { + logFFmpeg(ret); throw new IllegalStateException("Error opening codec " + codec.name().getString()); } } + protected static void logFFmpeg(int retCode) { + BytePointer buf = new BytePointer(AV_ERROR_MAX_STRING_SIZE); + av_strerror(retCode, buf, buf.capacity()); + logger.warn("FFmpeg returned error code {}: {}", retCode, buf.getString()); + } + /** * Set certain options in the codec context. - * @param codec_ctx Codec context. Context must be allocated first. + * Context must be allocated first using {@link #initContext()}. */ protected void initOptions() { @@ -110,22 +150,23 @@ protected void initOptions() { codec_ctx.width(options.width()); codec_ctx.height(options.height()); - /* - if (options.containsKey("pix_fmt")) { - codec_ctx.pix_fmt(options.get("pix_fmt")); - } else { - if (inputFormat == AV_CODEC_ID_MJPEG) { - codec_ctx.pix_fmt(AV_PIX_FMT_YUVJ420P); - } else { - codec_ctx.pix_fmt(AV_PIX_FMT_YUV420P); - } - } + codec_ctx.framerate(av_make_q(options.fps(), 1)); - */ + if (inputFormat.codec().ffmpegId == AV_CODEC_ID_H264) { + // OpenH264 only supports Baseline (66) and Main (77) + codec_ctx.profile(AV_PROFILE_H264_MAIN); - av_opt_set(codec_ctx.priv_data(), "preset", options.preset(), 0); - av_opt_set(codec_ctx.priv_data(), "tune", options.tune(), 0); - codec_ctx.strict_std_compliance(options.compliance()); // Needed so that yuvj420p works (used for mjpeg) + // Enable frame skip so bitrate control works correctly, + // or it falls back to quality mode and ignores the bitrate setting + av_opt_set(codec_ctx.priv_data(), "skip_frames", "1", 0); + + // OpenH264 uses slice_mode instead of preset + av_opt_set(codec_ctx.priv_data(), "slice_mode", "auto", 0); + } + + //av_opt_set(codec_ctx.priv_data(), "preset", options.preset(), 0); + //av_opt_set(codec_ctx.priv_data(), "tune", options.tune(), 0); + codec_ctx.strict_std_compliance(options.compliance()); // Needed so that yuvj420p works (used for mjpeg, must be set to unofficial) } protected abstract void deallocateInputPacket(I packet); @@ -139,7 +180,7 @@ public void submitInputPacket(I inputPacket) { if (inputPacket == null || !isProcessing.get()) { return; } - submitExecutor.submit(() -> { + submitTask(() -> { // Process the input processInputPacket(inputPacket); deallocateInputPacket(inputPacket); @@ -186,21 +227,32 @@ public void unregisterAllCallbacks() { public void close() { synchronized (contextLock) { if (isProcessing.compareAndSet(true, false)) { - submitExecutor.shutdownNow(); - outputExecutor.shutdownNow(); - if (codec_ctx != null) { - avcodec_free_context(codec_ctx); + // Submit cleanup *before* shutdown so it is the last task to run + executor.submit(this::cleanup); + executor.shutdown(); + try { + executor.awaitTermination(Long.MAX_VALUE, TimeUnit.NANOSECONDS); + } catch (InterruptedException ignored) { + logger.warn("Interrupted while waiting for ffmpeg thread to finish"); + Thread.currentThread().interrupt(); } - codec_ctx = null; - codec = null; - unregisterAllCallbacks(); - - for (var packet : outQueue) { - deallocateOutputPacket(packet); - } } } } + + private void cleanup() { + if (codec_ctx != null) { + avcodec_free_context(codec_ctx); + } + codec_ctx = null; + codec = null; + + unregisterAllCallbacks(); + + for (var packet : outQueue) { + deallocateOutputPacket(packet); + } + } } diff --git a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/coders/Decoder.java b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/coders/Decoder.java index 13de8f7cc..c08c86c3c 100644 --- a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/coders/Decoder.java +++ b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/coders/Decoder.java @@ -25,8 +25,9 @@ public Decoder(CodecInfo inFormatInfo, CodecInfo outFormatInfo, CodecOptions opt @Override protected void initContext() { synchronized (contextLock) { - codec = avcodec_find_decoder(inputFormat.codec().ffmpegId);; + codec = avcodec_find_decoder(inputFormat.codec().ffmpegId); codec_ctx = avcodec_alloc_context3(codec); + codec_ctx.codec_id(inputFormat.codec().ffmpegId); codec_ctx.pix_fmt(outputFormat.pixelFmt().ffmpegId); } } @@ -56,13 +57,23 @@ protected AVFrame cloneOutput(AVFrame packet) { @Override protected void processInputPacket(AVPacket inputPacket) { - if (inputPacket != null) { + if (inputPacket != null && !inputPacket.isNull()) { + if (avcodec_send_packet(codec_ctx, inputPacket) < 0) { + logger.warn("Error sending packet to decoder"); + //avcodec_flush_buffers(codec_ctx); + return; + } + AVFrame outputPacket = av_frame_alloc(); - avcodec_send_packet(codec_ctx, inputPacket); while (avcodec_receive_frame(codec_ctx, outputPacket) >= 0) { - outQueue.add(av_frame_clone(outputPacket)); + if (!outputPacket.isNull()) { + outQueue.add(outputPacket); + } + outputPacket = av_frame_alloc(); } + + av_frame_free(outputPacket); } } } diff --git a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/coders/Encoder.java b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/coders/Encoder.java index 1e514577a..83bbf4b27 100644 --- a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/coders/Encoder.java +++ b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/coders/Encoder.java @@ -27,11 +27,29 @@ public Encoder(CodecInfo inFormatInfo, CodecInfo outFormatInfo, CodecOptions opt @Override protected void initContext() { synchronized (contextLock) { - AVCodec codec = avcodec_find_encoder(outputFormat.codec().ffmpegId);; + // For H264, prefer x264 over OpenH264 — better option compatibility + if (outputFormat.codec() == FullCodecEnum.H264) { + codec = avcodec_find_encoder_by_name("libx264"); + } + + // Fall back to the default encoder for this codec ID + if (codec == null || codec.isNull()) { + codec = avcodec_find_encoder(outputFormat.codec().ffmpegId); + } + + if (codec == null || codec.isNull()) { + throw new IllegalStateException("Could not find encoder for: " + outputFormat.codec()); + } + codec_ctx = avcodec_alloc_context3(codec); - //initOptions(codec_ctx); + + if (codec_ctx == null || codec_ctx.isNull()) { + throw new IllegalStateException("Could not allocate encoder context for: " + codec.name().getString()); + } codec_ctx.pix_fmt(inputFormat.pixelFmt().ffmpegId); + + logger.debug("Using encoder: {}", codec.name().getString()); } } @@ -60,13 +78,36 @@ protected AVPacket cloneOutput(AVPacket packet) { @Override protected void processInputPacket(AVFrame inputPacket) { - if (inputPacket != null) { + if (inputPacket != null && !inputPacket.isNull()) { + int ret; + + logger.debug("Sending frame to encoder: format={} width={} height={} pts={}", + inputPacket.format(), + inputPacket.width(), + inputPacket.height(), + inputPacket.pts()); + logger.debug("Encoder expects: format={} width={} height={}", + codec_ctx.pix_fmt(), + codec_ctx.width(), + codec_ctx.height()); + + if ((ret = avcodec_send_frame(codec_ctx, inputPacket)) < 0) { + logger.warn("Error sending packet to encoder"); + logFFmpeg(ret); + //avcodec_flush_buffers(codec_ctx); + return; + } + AVPacket outputPacket = av_packet_alloc(); - avcodec_send_frame(codec_ctx, inputPacket); while (avcodec_receive_packet(codec_ctx, outputPacket) >= 0) { - outQueue.add(av_packet_clone(outputPacket)); + if (!outputPacket.isNull()) { + outQueue.add(outputPacket); + } + outputPacket = av_packet_alloc(); } + + av_packet_free(outputPacket); } } } diff --git a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/coders/SwScaler.java b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/coders/SwScaler.java index 8a8731d73..80b13f6a4 100644 --- a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/coders/SwScaler.java +++ b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/coders/SwScaler.java @@ -40,6 +40,9 @@ protected void initContext() { @Override protected void initOptions() {} // no options for swscaler + @Override + protected void openContext() {} // no codec to open + @Override protected void deallocateInputPacket(AVFrame packet) { if (packet != null) { diff --git a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/helpers/CodecInfo.java b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/helpers/CodecInfo.java index f27102bf3..e5c9b45ed 100644 --- a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/helpers/CodecInfo.java +++ b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/helpers/CodecInfo.java @@ -6,10 +6,7 @@ public static CodecInfo newCodecInfoFromName(String name) { FullPixelEnum pixel; try { codec = Enum.valueOf(FullCodecEnum.class, name); - if (codec == FullCodecEnum.MJPEG) - pixel = FullPixelEnum.YUVJ420P; - else - pixel = FullPixelEnum.YUV420P; + pixel = FullPixelEnum.YUVJ420P; } catch (Exception e) { codec = FullCodecEnum.RAWVIDEO; pixel = Enum.valueOf(FullPixelEnum.class, name); diff --git a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/helpers/FullCodecEnum.java b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/helpers/FullCodecEnum.java index cc2827e9a..c488614d6 100644 --- a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/helpers/FullCodecEnum.java +++ b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/helpers/FullCodecEnum.java @@ -4,12 +4,11 @@ public enum FullCodecEnum { H264(avcodec.AV_CODEC_ID_H264), - H265(avcodec.AV_CODEC_ID_H265), HEVC(avcodec.AV_CODEC_ID_HEVC), MJPEG(avcodec.AV_CODEC_ID_MJPEG), VP8(avcodec.AV_CODEC_ID_VP8), VP9(avcodec.AV_CODEC_ID_VP9), - MPEG2(avcodec.AV_CODEC_ID_MPEG2TS), + MPEG2(avcodec.AV_CODEC_ID_MPEG2VIDEO), MPEG4(avcodec.AV_CODEC_ID_MPEG4), AV1(avcodec.AV_CODEC_ID_AV1), THEORA(avcodec.AV_CODEC_ID_THEORA), From 39a39615423561846dac251f10de117ce5c25f55 Mon Sep 17 00:00:00 2001 From: kyle-fitzp Date: Tue, 12 May 2026 15:12:44 -0500 Subject: [PATCH 05/10] Fixes to encoder, decoder, scaler --- .../process/video/transcoder/CodecEnum.java | 4 +- .../video/transcoder/FFMpegTranscoder.java | 82 +++++++---- .../coders/{Coder.java => Codec.java} | 138 ++++++++++++------ .../video/transcoder/coders/Decoder.java | 29 ++-- .../video/transcoder/coders/Encoder.java | 41 ++---- .../video/transcoder/coders/SwScaler.java | 32 ++-- .../transcoder/formatters/FrameFormatter.java | 45 ++++-- .../formatters/PacketFormatter.java | 5 +- .../video/transcoder/helpers/CodecInfo.java | 22 ++- 9 files changed, 243 insertions(+), 155 deletions(-) rename processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/coders/{Coder.java => Codec.java} (65%) diff --git a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/CodecEnum.java b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/CodecEnum.java index 701c49996..42474af40 100644 --- a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/CodecEnum.java +++ b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/CodecEnum.java @@ -18,8 +18,8 @@ public enum CodecEnum { VP9(AV_CODEC_ID_VP9), MPEG2(AV_CODEC_ID_MPEG2TS), MPEG4(AV_CODEC_ID_MPEG4), - RGB(AV_PIX_FMT_RGB24), - YUV(AV_PIX_FMT_YUV420P); + RGB24(AV_PIX_FMT_RGB24), + YUV420P(AV_PIX_FMT_YUV420P); int ffmpegId; diff --git a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/FFMpegTranscoder.java b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/FFMpegTranscoder.java index 6123923f3..09a23116f 100644 --- a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/FFMpegTranscoder.java +++ b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/FFMpegTranscoder.java @@ -17,19 +17,14 @@ import net.opengis.swe.v20.*; import org.bytedeco.ffmpeg.avcodec.AVPacket; import org.bytedeco.ffmpeg.avutil.AVFrame; -import org.bytedeco.ffmpeg.global.avcodec; -import org.bytedeco.ffmpeg.global.avutil; import org.bytedeco.javacpp.BytePointer; -import org.bytedeco.javacpp.Pointer; import org.sensorhub.api.processing.OSHProcessInfo; -import org.sensorhub.impl.process.video.transcoder.coders.Coder; -import org.sensorhub.impl.process.video.transcoder.coders.Decoder; -import org.sensorhub.impl.process.video.transcoder.coders.Encoder; -import org.sensorhub.impl.process.video.transcoder.coders.SwScaler; +import org.sensorhub.impl.process.video.transcoder.coders.*; import org.sensorhub.impl.process.video.transcoder.formatters.*; import org.sensorhub.impl.process.video.transcoder.helpers.CodecInfo; import org.sensorhub.impl.process.video.transcoder.helpers.CodecOptions; import org.sensorhub.impl.process.video.transcoder.helpers.FullCodecEnum; +import org.sensorhub.impl.process.video.transcoder.helpers.FullPixelEnum; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.vast.data.DataBlockCompressed; @@ -69,7 +64,7 @@ public class FFMpegTranscoder extends ExecutableProcessImpl Text inCodecParam; Text outCodecParam; - List videoProcs; + List videoProcs; AVByteFormatter inputFormatter, outputFormatter; CodecOptions decOptions, encOptions; @@ -157,25 +152,49 @@ private void initFormatters() throws ProcessException { * Initializes all encoder/decoder/swscaler objects. These objects are added to the {@link FFMpegTranscoder#videoProcs} * queue in the order they should process the incoming data. At most, the flow will be Decoder -> SWScale -> Encoder. */ - private void initCoders() { + private void initCodecs() { if (videoProcs != null) { videoProcs.clear(); } videoProcs = new ArrayList<>(); + Decoder decoder = null; + Encoder encoder = null; + SwScaler swScaler = null; + //FullPixelEnum decOutPixFmt = outCodec.pixelFmt; + //FullPixelEnum encInPixFmt = outCodec.pixelFmt; + if (!isUncompressed(inCodec)) { - videoProcs.add(new Decoder(inCodec, outCodec, decOptions)); - } - if (width != outWidth || height != outHeight || (isUncompressed(inCodec) && isUncompressed(outCodec))) { - //int inFmt = isUncompressed(inCodec) ? inCodec.getCodec().ffmpegId : AV_PIX_FMT_YUV420P; - //int outFmt = isUncompressed(outCodec) ? outCodec.getCodec().ffmpegId : AV_PIX_FMT_YUV420P; - videoProcs.add(new SwScaler(inCodec, outCodec, width, height, outWidth, outHeight)); + decoder = new Decoder(inCodec, outCodec, decOptions); + outCodec.pixelFmt = decoder.init(); + //videoProcs.add(decoder); } if (!isUncompressed(outCodec)) { - Encoder encoder = new Encoder(inCodec, outCodec, decOptions); - videoProcs.add(encoder); + encoder = new Encoder(inCodec, outCodec, encOptions); + inCodec.pixelFmt = encoder.init(); + //videoProcs.add(encoder); + } + + logger.info("Input pixel format: {}, Output pixel format: {}", inCodec.pixelFmt, outCodec.pixelFmt); + + if (inCodec.pixelFmt == null || outCodec.pixelFmt == null) { logger.warn("Pixel format is null"); } + + if (width != outWidth || height != outHeight + || (isUncompressed(inCodec) && isUncompressed(outCodec)) + || (inCodec.pixelFmt != outCodec.pixelFmt)) { + + swScaler = new SwScaler(inCodec, outCodec, width, height, outWidth, outHeight); + swScaler.init(); + //videoProcs.add(swScaler); } + if (decoder != null) { videoProcs.add(decoder); } + if (swScaler != null) { videoProcs.add(swScaler); } + if (encoder != null) { videoProcs.add(encoder); } + + } + + private void initPipeline() { // Frame pipe between decoder, swscaler, encoder for (int i = 0; i < videoProcs.size() - 1; i++) { var nextProc = videoProcs.get(i + 1); @@ -185,20 +204,24 @@ private void initCoders() { } // Output - videoProcs.get(videoProcs.size() - 1).registerCallback(packet -> { - publishFrameData(outputFormatter.convertOutput(packet)); + var finalProc = videoProcs.get(videoProcs.size() - 1); + finalProc.registerCallback(packet -> { + try { + publishFrameData(outputFormatter.convertOutput(packet)); + } finally { + finalProc.deallocateOutputPacket(packet); + } }); - } /** * Invoked on process stop and init. - * Stop all {@link Coder} thread objects stored in {@link FFMpegTranscoder#videoProcs}. + * Stop all {@link Codec} thread objects stored in {@link FFMpegTranscoder#videoProcs}. */ private void stopProcessing() throws InterruptedException { isInit.set(false); if (videoProcs != null) { - for (Coder codec : videoProcs) { + for (Codec codec : videoProcs) { codec.close(); } videoProcs.clear(); @@ -307,13 +330,14 @@ public void init() throws ProcessException setImgEncoding(); initCodecOptions(); initFormatters(); - initCoders(); + initCodecs(); + initPipeline(); imgOut.setData(new DataBlockCompressed()); } catch (IllegalArgumentException e) { - reportError("Unsupported codec " + inCodecParam.getData().getStringValue() + ". Must be one of " + Arrays.toString(CodecEnum.values())); + reportError("Unsupported codec " + inCodecParam.getData().getStringValue() + ". Must be one of " + Arrays.toString(CodecEnum.values()), e); } super.init(); @@ -390,7 +414,7 @@ private void initCodecOptions() { private AVByteFormatter getFormatter(CodecInfo codec, @Nullable Integer width, @Nullable Integer height) throws ProcessException { try { if (isUncompressed(codec)) - return new FrameFormatter(width, height, codec.pixelFmt().ffmpegId); + return new FrameFormatter(width, height, codec.pixelFmt.ffmpegId); else return new PacketFormatter(); /* @@ -408,10 +432,10 @@ private AVByteFormatter getFormatter(CodecInfo codec, @Nullable Integer width, @ /** * @param codec Video codec or uncompressed format. - * @return Is the codec {@link CodecEnum#RGB} or {@link CodecEnum#YUV}? + * @return Is the codec {@link FullCodecEnum#RAWVIDEO}? */ private boolean isUncompressed(CodecInfo codec) { - return codec.codec() == FullCodecEnum.RAWVIDEO; + return codec.codec == FullCodecEnum.RAWVIDEO; } /** @@ -487,7 +511,7 @@ public void execute() throws ProcessException } private boolean isVideoProcChainReady() { - for (Coder proc : videoProcs) { + for (Codec proc : videoProcs) { if (!proc.isReady()) { return false; } @@ -510,7 +534,7 @@ public void dispose() isInit.set(false); if (videoProcs != null) { - for (Coder proc : videoProcs) { + for (Codec proc : videoProcs) { proc.close(); } } diff --git a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/coders/Coder.java b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/coders/Codec.java similarity index 65% rename from processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/coders/Coder.java rename to processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/coders/Codec.java index 81347fb3e..20101af6a 100644 --- a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/coders/Coder.java +++ b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/coders/Codec.java @@ -19,14 +19,17 @@ import org.bytedeco.ffmpeg.avcodec.AVPacket; import org.bytedeco.ffmpeg.avutil.AVFrame; import org.bytedeco.javacpp.BytePointer; +import org.bytedeco.javacpp.IntPointer; import org.bytedeco.javacpp.Pointer; import org.bytedeco.javacpp.PointerPointer; import org.sensorhub.impl.process.video.transcoder.helpers.CodecInfo; import org.sensorhub.impl.process.video.transcoder.helpers.CodecOptions; +import org.sensorhub.impl.process.video.transcoder.helpers.FullCodecEnum; +import org.sensorhub.impl.process.video.transcoder.helpers.FullPixelEnum; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public abstract class Coder implements AutoCloseable { +public abstract class Codec implements AutoCloseable { public interface CoderCallback { @@ -34,21 +37,19 @@ public interface CoderCallback { public abstract void onPacket(O packet); } - protected static final Logger logger = LoggerFactory.getLogger(Coder.class); + protected static final Logger logger = LoggerFactory.getLogger(Codec.class); - private static int coderCount = 0; - private final int coderNum = coderCount++; + private static int codecCount = 0; + private final int codecNum = codecCount++; private final ExecutorService executor = Executors.newSingleThreadExecutor( - r -> new Thread(r, "ffmpeg-codec-thread-" + coderNum)); - private final ExecutorService outputExecutor = Executors.newSingleThreadExecutor(r -> new Thread(r, "ffmpeg-codec-output-thread-" + coderNum)); + r -> new Thread(r, "ffmpeg-codec-thread-" + codecNum)); + private final ExecutorService outputExecutor = Executors.newSingleThreadExecutor(r -> new Thread(r, "ffmpeg-codec-output-thread-" + codecNum)); private final Map, ExecutorService> callbackMap = new HashMap<>(); CodecInfo inputFormat; CodecInfo outputFormat; protected AVCodecContext codec_ctx; protected AVCodec codec; - protected I inPacket; - protected O outPacket; protected final Queue outQueue = new ArrayDeque<>(10); private final AtomicBoolean isProcessing = new AtomicBoolean(false); // Set false to indicate packets should no longer be accepted final Object contextLock = new Object(); @@ -57,7 +58,7 @@ public interface CoderCallback { CodecOptions options; AtomicBoolean isNotifying = new AtomicBoolean(false); - public Coder(CodecInfo inFormatInfo, CodecInfo outFormatInfo, Class inputClass, Class outputClass, CodecOptions options) { + public Codec(CodecInfo inFormatInfo, CodecInfo outFormatInfo, Class inputClass, Class outputClass, CodecOptions options) { super(); if ((inputClass != AVPacket.class && inputClass != AVFrame.class) @@ -65,29 +66,19 @@ public Coder(CodecInfo inFormatInfo, CodecInfo outFormatInfo, Class inputClas throw new IllegalArgumentException("Input and output classes must be either AVPacket or AVFrame"); } - if (options == null) - throw new IllegalArgumentException("Options cannot be null"); - - this.inputFormat = inFormatInfo; + this.inputFormat = inFormatInfo.clone(); this.inputClass = inputClass; - this.outputFormat = outFormatInfo; + this.outputFormat = outFormatInfo.clone(); this.outputClass = outputClass; this.options = options; + } - // All codec operations must happen in a separate thread - try { - executor.submit(() -> { - initContext(); - initOptions(); - openContext(); - isProcessing.set(true); - }).get(); // blocks constructor until init is complete - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw new RuntimeException("Interrupted during codec initialization", e); - } catch (ExecutionException e) { - throw new RuntimeException("Error initializing codec context", e.getCause()); - } + public FullPixelEnum init() { + initContext(); + initOptions(); + var pixFmt = openContext(); + isProcessing.set(true); + return pixFmt; } private void submitTask(Runnable task) { @@ -119,12 +110,24 @@ public Class getInputClass() { protected abstract void initContext(); - protected void openContext() { + protected FullPixelEnum openContext() { int ret; + FullPixelEnum pixelFmt; if ((ret = avcodec_open2(codec_ctx, codec, (PointerPointer) null)) < 0) { logFFmpeg(ret); throw new IllegalStateException("Error opening codec " + codec.name().getString()); } + try { + var desc = av_pix_fmt_desc_get(codec_ctx.pix_fmt()); + pixelFmt = FullPixelEnum.valueOf(desc.name().getString().toUpperCase()); + inputFormat.pixelFmt = pixelFmt; + outputFormat.pixelFmt = pixelFmt; + desc.deallocate(); + } catch (Exception e) { + pixelFmt = null; + logger.warn("Could not determine codec info for " + codec.name().getString(), e); + } + return pixelFmt; } protected static void logFFmpeg(int retCode) { @@ -133,26 +136,65 @@ protected static void logFFmpeg(int retCode) { logger.warn("FFmpeg returned error code {}: {}", retCode, buf.getString()); } + protected static void setCodecPixFmt(AVCodecContext codec_ctx, FullPixelEnum desiredFmt) { + String codecString = codec_ctx.codec().name().getString(); + PointerPointer pixelFmts = new PointerPointer<>(1); + + avcodec_get_supported_config(codec_ctx, null, AV_CODEC_CONFIG_PIX_FORMAT, 0, pixelFmts, (IntPointer) null); + + IntPointer fmts = pixelFmts.get(IntPointer.class, 0); + // If null, all formats are supported + if (fmts == null || fmts.isNull()) { + if (desiredFmt == FullPixelEnum.NONE) { + codec_ctx.pix_fmt(AV_PIX_FMT_YUV420P); + } else { + codec_ctx.pix_fmt(desiredFmt.ffmpegId); + } + } + else { + boolean found = false; + for (int i = 0; fmts.get(i) != AV_PIX_FMT_NONE; i++) { + if (fmts.get(i) == desiredFmt.ffmpegId) { + found = true; + codec_ctx.pix_fmt(fmts.get(i)); + break; + } + } + if (!found) { + logger.warn("Preferred pixel format for codec {} could not be found", codecString); + codec_ctx.pix_fmt(fmts.get(0)); + } + } + + if (fmts != null) + fmts.deallocate(); + pixelFmts.deallocate(); + } + /** * Set certain options in the codec context. * Context must be allocated first using {@link #initContext()}. */ protected void initOptions() { - codec_ctx.time_base(av_make_q(1, options.fps())); - if (options.bitRate() > 0) { codec_ctx.bit_rate(options.bitRate() * 1000); } else { - //codec_ctx.bit_rate(150*1000); + codec_ctx.bit_rate(0); } codec_ctx.width(options.width()); codec_ctx.height(options.height()); - codec_ctx.framerate(av_make_q(options.fps(), 1)); + if (options.fps() > 0) { + codec_ctx.framerate(av_make_q(options.fps(), 1)); + codec_ctx.time_base(av_make_q(1, options.fps())); + } else { + codec_ctx.framerate(av_make_q(25, 1)); + codec_ctx.time_base(av_make_q(1, 25)); + } - if (inputFormat.codec().ffmpegId == AV_CODEC_ID_H264) { + if (inputFormat.codec.ffmpegId == AV_CODEC_ID_H264) { // OpenH264 only supports Baseline (66) and Main (77) codec_ctx.profile(AV_PROFILE_H264_MAIN); @@ -169,8 +211,8 @@ protected void initOptions() { codec_ctx.strict_std_compliance(options.compliance()); // Needed so that yuvj420p works (used for mjpeg, must be set to unofficial) } - protected abstract void deallocateInputPacket(I packet); - protected abstract void deallocateOutputPacket(O packet); + public abstract void deallocateInputPacket(I packet); + public abstract void deallocateOutputPacket(O packet); protected abstract O cloneOutput(O packet); protected abstract void processInputPacket(I inputPacket); @@ -187,11 +229,12 @@ public void submitInputPacket(I inputPacket) { if (!outQueue.isEmpty() && isNotifying.compareAndSet(false, true)) { outputExecutor.submit(() -> { - for (var outputPacket : outQueue) { + while (!outQueue.isEmpty()) { + var outputPacket = outQueue.poll(); notifyCallbacks(outputPacket); } + isNotifying.set(false); }); - isNotifying.set(false); } }); } @@ -199,19 +242,19 @@ public void submitInputPacket(I inputPacket) { private void notifyCallbacks(O outputPacket) { for (var entry: callbackMap.entrySet()) { + var clonedOutputPacket = cloneOutput(outputPacket); entry.getValue().submit(() -> { - var clonedOutputPacket = cloneOutput(outputPacket); entry.getKey().onPacket(clonedOutputPacket); - deallocateOutputPacket(clonedOutputPacket); }); } + deallocateOutputPacket(outputPacket); } public void registerCallback(CoderCallback callback) { if (!callbackMap.containsKey(callback)) { - callbackMap.put(callback, Executors.newSingleThreadExecutor(r -> new Thread(r, "ffmpeg-codec-" + coderNum + "-callback-thread"))); + callbackMap.put(callback, Executors.newSingleThreadExecutor(r -> new Thread(r, "ffmpeg-codec-" + codecNum + "-callback-thread"))); } else { - logger.warn("This callback was already registered for codec " + coderNum); + logger.warn("This callback was already registered for codec " + codecNum); } } @@ -230,13 +273,22 @@ public void close() { // Submit cleanup *before* shutdown so it is the last task to run executor.submit(this::cleanup); - executor.shutdown(); + executor.shutdownNow(); try { executor.awaitTermination(Long.MAX_VALUE, TimeUnit.NANOSECONDS); } catch (InterruptedException ignored) { logger.warn("Interrupted while waiting for ffmpeg thread to finish"); Thread.currentThread().interrupt(); } + + outputExecutor.shutdownNow(); + try { + outputExecutor.awaitTermination(Long.MAX_VALUE, TimeUnit.NANOSECONDS); + } catch (InterruptedException ignored) { + logger.warn("Interrupted while waiting for ffmpeg thread to finish"); + Thread.currentThread().interrupt(); + } + unregisterAllCallbacks(); } } diff --git a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/coders/Decoder.java b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/coders/Decoder.java index c08c86c3c..6030ee755 100644 --- a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/coders/Decoder.java +++ b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/coders/Decoder.java @@ -1,22 +1,14 @@ package org.sensorhub.impl.process.video.transcoder.coders; -import org.bytedeco.ffmpeg.avcodec.AVCodec; import org.bytedeco.ffmpeg.avcodec.AVPacket; import org.bytedeco.ffmpeg.avutil.AVFrame; -import org.bytedeco.javacpp.PointerPointer; import org.sensorhub.impl.process.video.transcoder.helpers.CodecInfo; import org.sensorhub.impl.process.video.transcoder.helpers.CodecOptions; -import org.sensorhub.impl.process.video.transcoder.helpers.FullCodecEnum; -import org.sensorhub.impl.process.video.transcoder.helpers.FullPixelEnum; - -import java.util.ArrayDeque; -import java.util.HashMap; -import java.util.Queue; import static org.bytedeco.ffmpeg.global.avcodec.*; import static org.bytedeco.ffmpeg.global.avutil.*; -public class Decoder extends Coder { +public class Decoder extends Codec { public Decoder(CodecInfo inFormatInfo, CodecInfo outFormatInfo, CodecOptions options) { super(inFormatInfo, outFormatInfo, AVPacket.class, AVFrame.class, options); @@ -25,22 +17,23 @@ public Decoder(CodecInfo inFormatInfo, CodecInfo outFormatInfo, CodecOptions opt @Override protected void initContext() { synchronized (contextLock) { - codec = avcodec_find_decoder(inputFormat.codec().ffmpegId); + codec = avcodec_find_decoder(inputFormat.codec.ffmpegId); codec_ctx = avcodec_alloc_context3(codec); - codec_ctx.codec_id(inputFormat.codec().ffmpegId); - codec_ctx.pix_fmt(outputFormat.pixelFmt().ffmpegId); + codec_ctx.codec_id(inputFormat.codec.ffmpegId); + setCodecPixFmt(codec_ctx, outputFormat.pixelFmt); + //codec_ctx.pix_fmt(outputFormat.pixelFmt().ffmpegId); } } @Override - protected void deallocateInputPacket(AVPacket packet) { + public void deallocateInputPacket(AVPacket packet) { if (packet != null) { av_packet_free(packet); } } @Override - protected void deallocateOutputPacket(AVFrame packet) { + public void deallocateOutputPacket(AVFrame packet) { if (packet != null) { av_frame_free(packet); } @@ -56,10 +49,12 @@ protected AVFrame cloneOutput(AVFrame packet) { } @Override - protected void processInputPacket(AVPacket inputPacket) { + protected synchronized void processInputPacket(AVPacket inputPacket) { if (inputPacket != null && !inputPacket.isNull()) { - if (avcodec_send_packet(codec_ctx, inputPacket) < 0) { - logger.warn("Error sending packet to decoder"); + int ret; + if ((ret = avcodec_send_packet(codec_ctx, inputPacket)) < 0) { + //logger.warn("Error sending packet to decoder"); + //logFFmpeg(ret); //avcodec_flush_buffers(codec_ctx); return; } diff --git a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/coders/Encoder.java b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/coders/Encoder.java index 83bbf4b27..c0310429c 100644 --- a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/coders/Encoder.java +++ b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/coders/Encoder.java @@ -1,24 +1,19 @@ package org.sensorhub.impl.process.video.transcoder.coders; -import org.bytedeco.ffmpeg.avcodec.AVCodec; import org.bytedeco.ffmpeg.avcodec.AVPacket; import org.bytedeco.ffmpeg.avutil.AVFrame; -import org.bytedeco.javacpp.BytePointer; +import org.bytedeco.ffmpeg.avutil.AVPixFmtDescriptor; +import org.bytedeco.javacpp.IntPointer; +import org.bytedeco.javacpp.Pointer; import org.bytedeco.javacpp.PointerPointer; import org.sensorhub.impl.process.video.transcoder.helpers.CodecInfo; import org.sensorhub.impl.process.video.transcoder.helpers.CodecOptions; import org.sensorhub.impl.process.video.transcoder.helpers.FullCodecEnum; -import org.sensorhub.impl.process.video.transcoder.helpers.FullPixelEnum; - -import java.util.ArrayDeque; -import java.util.HashMap; -import java.util.Queue; import static org.bytedeco.ffmpeg.global.avcodec.*; import static org.bytedeco.ffmpeg.global.avutil.*; -import static org.bytedeco.ffmpeg.global.swscale.sws_freeContext; -public class Encoder extends Coder { +public class Encoder extends Codec { public Encoder(CodecInfo inFormatInfo, CodecInfo outFormatInfo, CodecOptions options) { super(inFormatInfo, outFormatInfo, AVFrame.class, AVPacket.class, options); @@ -28,17 +23,17 @@ public Encoder(CodecInfo inFormatInfo, CodecInfo outFormatInfo, CodecOptions opt protected void initContext() { synchronized (contextLock) { // For H264, prefer x264 over OpenH264 — better option compatibility - if (outputFormat.codec() == FullCodecEnum.H264) { + if (outputFormat.codec == FullCodecEnum.H264) { codec = avcodec_find_encoder_by_name("libx264"); } // Fall back to the default encoder for this codec ID if (codec == null || codec.isNull()) { - codec = avcodec_find_encoder(outputFormat.codec().ffmpegId); + codec = avcodec_find_encoder(outputFormat.codec.ffmpegId); } if (codec == null || codec.isNull()) { - throw new IllegalStateException("Could not find encoder for: " + outputFormat.codec()); + throw new IllegalStateException("Could not find encoder for: " + outputFormat.codec); } codec_ctx = avcodec_alloc_context3(codec); @@ -47,21 +42,21 @@ protected void initContext() { throw new IllegalStateException("Could not allocate encoder context for: " + codec.name().getString()); } - codec_ctx.pix_fmt(inputFormat.pixelFmt().ffmpegId); + setCodecPixFmt(codec_ctx, inputFormat.pixelFmt); logger.debug("Using encoder: {}", codec.name().getString()); } } @Override - protected void deallocateInputPacket(AVFrame packet) { + public void deallocateInputPacket(AVFrame packet) { if (packet != null) { av_frame_free(packet); } } @Override - protected void deallocateOutputPacket(AVPacket packet) { + public void deallocateOutputPacket(AVPacket packet) { if (packet != null) { av_packet_free(packet); } @@ -77,23 +72,13 @@ protected AVPacket cloneOutput(AVPacket packet) { } @Override - protected void processInputPacket(AVFrame inputPacket) { + protected synchronized void processInputPacket(AVFrame inputPacket) { if (inputPacket != null && !inputPacket.isNull()) { int ret; - logger.debug("Sending frame to encoder: format={} width={} height={} pts={}", - inputPacket.format(), - inputPacket.width(), - inputPacket.height(), - inputPacket.pts()); - logger.debug("Encoder expects: format={} width={} height={}", - codec_ctx.pix_fmt(), - codec_ctx.width(), - codec_ctx.height()); - if ((ret = avcodec_send_frame(codec_ctx, inputPacket)) < 0) { - logger.warn("Error sending packet to encoder"); - logFFmpeg(ret); + //logger.warn("Error sending packet to encoder"); + //logFFmpeg(ret); //avcodec_flush_buffers(codec_ctx); return; } diff --git a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/coders/SwScaler.java b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/coders/SwScaler.java index 80b13f6a4..a8d1233cb 100644 --- a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/coders/SwScaler.java +++ b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/coders/SwScaler.java @@ -1,23 +1,15 @@ package org.sensorhub.impl.process.video.transcoder.coders; -import org.bytedeco.ffmpeg.avcodec.AVCodec; -import org.bytedeco.ffmpeg.avcodec.AVPacket; import org.bytedeco.ffmpeg.avutil.AVFrame; import org.bytedeco.ffmpeg.swscale.SwsContext; -import org.bytedeco.javacpp.BytePointer; import org.bytedeco.javacpp.DoublePointer; -import org.bytedeco.javacpp.PointerPointer; import org.sensorhub.impl.process.video.transcoder.helpers.CodecInfo; +import org.sensorhub.impl.process.video.transcoder.helpers.FullPixelEnum; -import java.util.ArrayDeque; -import java.util.HashMap; -import java.util.Queue; - -import static org.bytedeco.ffmpeg.global.avcodec.*; import static org.bytedeco.ffmpeg.global.avutil.*; import static org.bytedeco.ffmpeg.global.swscale.*; -public class SwScaler extends Coder { +public class SwScaler extends Codec { long pts = 0; SwsContext swsContext; final int inWidth, inHeight, outWidth, outHeight; @@ -32,8 +24,8 @@ public SwScaler(CodecInfo inputFormat, CodecInfo outputFormat, int inWidth, int @Override protected void initContext() { - swsContext = sws_getContext(inWidth, inHeight, inputFormat.pixelFmt().ffmpegId, - outWidth, outHeight, outputFormat.pixelFmt().ffmpegId, + swsContext = sws_getContext(inWidth, inHeight, inputFormat.pixelFmt.ffmpegId, + outWidth, outHeight, outputFormat.pixelFmt.ffmpegId, SWS_BICUBIC, null, null, (DoublePointer) null); } @@ -41,17 +33,19 @@ protected void initContext() { protected void initOptions() {} // no options for swscaler @Override - protected void openContext() {} // no codec to open + protected FullPixelEnum openContext() { + return outputFormat.pixelFmt; + } // no codec to open @Override - protected void deallocateInputPacket(AVFrame packet) { + public void deallocateInputPacket(AVFrame packet) { if (packet != null) { av_frame_free(packet); } } @Override - protected void deallocateOutputPacket(AVFrame packet) { + public void deallocateOutputPacket(AVFrame packet) { if (packet != null) { av_frame_free(packet); } @@ -67,10 +61,14 @@ protected AVFrame cloneOutput(AVFrame packet) { } @Override - protected void processInputPacket(AVFrame inputPacket) { + protected synchronized void processInputPacket(AVFrame inputPacket) { if (inputPacket != null) { AVFrame outputPacket = av_frame_alloc(); - sws_scale_frame(swsContext, outPacket, inPacket); + outputPacket.format(outputFormat.pixelFmt.ffmpegId); + outputPacket.width(outWidth); + outputPacket.height(outHeight); + av_frame_get_buffer(outputPacket, 0); + sws_scale_frame(swsContext, outputPacket, inputPacket); outQueue.add(outputPacket); } } diff --git a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/formatters/FrameFormatter.java b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/formatters/FrameFormatter.java index 04d6c301a..71e61d73f 100644 --- a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/formatters/FrameFormatter.java +++ b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/formatters/FrameFormatter.java @@ -21,13 +21,14 @@ public class FrameFormatter implements AVByteFormatter { private final int[] planeHeights; private final int[] planeSizes; private final int totalSize; + boolean isPlanar; public FrameFormatter(int width, int height, int pixFmt) { this.width = width; this.height = height; this.pixFmt = pixFmt; this.desc = avutil.av_pix_fmt_desc_get(pixFmt); - boolean isPlanar = (desc.flags() & AV_PIX_FMT_FLAG_PLANAR) == 0; + isPlanar = (desc.flags() & AV_PIX_FMT_FLAG_PLANAR) != 0; if (isPlanar) { this.planeCount = countPlanes(desc); @@ -44,7 +45,7 @@ public FrameFormatter(int width, int height, int pixFmt) { planeSizes = new int[1]; planeWidths[0] = width; planeHeights[0] = height; - planeSizes[0] = width * height; + planeSizes[0] = width * height * ((av_get_bits_per_pixel(desc) + 7) / 8); } totalSize = calcByteSize(planeSizes); @@ -68,7 +69,7 @@ private static int calcByteSize(int[] planeSizes) { public AVFrame convertInput(byte[] inputData) { AVFrame newFrame = generateFrame(); - if ((desc.flags() & AV_PIX_FMT_FLAG_PLANAR) == 0) + if (isPlanar) setFrameDataPlanar(newFrame, inputData); else setFrameDataPacked(newFrame, inputData); @@ -120,27 +121,41 @@ private AVFrame generateFrame() { newFrame.format(pixFmt); newFrame.width(width); newFrame.height(height); - av_frame_get_buffer(newFrame, 32); + int ret = av_frame_get_buffer(newFrame, 32); + if (ret < 0) { + av_frame_free(newFrame); + throw new IllegalStateException("Could not allocate AVFrame buffer, ffmpeg error: " + ret); + } return newFrame; } @Override public byte[] convertOutput(AVFrame outputFrame) { + if (outputFrame.format() != pixFmt) { + throw new IllegalArgumentException( + "Unexpected frame pixel format: " + outputFrame.format() + ", expected " + pixFmt); + } + byte[] out = new byte[totalSize]; int offset = 0; - for (int plane = 0; plane < 3; plane++) { - int w = planeWidth(plane); - int h = planeHeight(plane); - int srcStride = outputFrame.linesize(plane); - - BytePointer src = outputFrame.data(plane).position(0); - - for (int y = 0; y < h; y++) { - src.position(y * srcStride); - src.get(out, offset, w); - offset += w; + if (isPlanar) { + for (int plane = 0; plane < planeCount; plane++) { + int w = planeWidth(plane); + int h = planeHeight(plane); + int srcStride = outputFrame.linesize(plane); + + BytePointer src = outputFrame.data(plane).position(0); + + for (int y = 0; y < h; y++) { + src.position(y * srcStride); + src.get(out, offset, w); + offset += w; + } } + } else { + BytePointer src = outputFrame.data(0); + src.get(out, 0, totalSize); } return out; } diff --git a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/formatters/PacketFormatter.java b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/formatters/PacketFormatter.java index e88de05fc..394e92570 100644 --- a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/formatters/PacketFormatter.java +++ b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/formatters/PacketFormatter.java @@ -20,9 +20,8 @@ public class PacketFormatter implements AVByteFormatter { @Override public AVPacket convertInput(byte[] inputData) { AVPacket newPacket = av_packet_alloc(); - av_new_packet(newPacket, inputData.length); - newPacket.data().position(0); - newPacket.data().put(inputData.clone(), 0, inputData.length); + //av_new_packet(newPacket, inputData.length); + av_packet_from_data(newPacket, inputData, inputData.length); return newPacket; } diff --git a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/helpers/CodecInfo.java b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/helpers/CodecInfo.java index e5c9b45ed..f4cc69bf1 100644 --- a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/helpers/CodecInfo.java +++ b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/helpers/CodecInfo.java @@ -1,6 +1,9 @@ package org.sensorhub.impl.process.video.transcoder.helpers; -public record CodecInfo(FullCodecEnum codec, FullPixelEnum pixelFmt) { +public class CodecInfo implements Cloneable { + public FullCodecEnum codec; + public FullPixelEnum pixelFmt; + public static CodecInfo newCodecInfoFromName(String name) { FullCodecEnum codec; FullPixelEnum pixel; @@ -14,4 +17,21 @@ public static CodecInfo newCodecInfoFromName(String name) { return new CodecInfo(codec, pixel); } + + public CodecInfo(FullCodecEnum codec, FullPixelEnum pixelFmt) { + this.codec = codec; + this.pixelFmt = pixelFmt; + } + + @Override + public CodecInfo clone() { + try { + CodecInfo clone = (CodecInfo) super.clone(); + clone.codec = codec; + clone.pixelFmt = pixelFmt; + return clone; + } catch (CloneNotSupportedException e) { + throw new AssertionError(); + } + } } From b789c86419bce0a520947b30fcb12bd3644fcff7 Mon Sep 17 00:00:00 2001 From: kyle-fitzp Date: Tue, 12 May 2026 16:14:11 -0500 Subject: [PATCH 06/10] Add max outqueue size, other fixes --- .../impl/process/video/transcoder/coders/Codec.java | 11 +++++++++++ .../impl/process/video/transcoder/coders/Decoder.java | 2 +- .../impl/process/video/transcoder/coders/Encoder.java | 2 +- .../process/video/transcoder/coders/SwScaler.java | 2 +- .../video/transcoder/formatters/FrameFormatter.java | 4 ++-- .../video/transcoder/formatters/PacketFormatter.java | 6 ++++-- 6 files changed, 20 insertions(+), 7 deletions(-) diff --git a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/coders/Codec.java b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/coders/Codec.java index 20101af6a..b47597be7 100644 --- a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/coders/Codec.java +++ b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/coders/Codec.java @@ -51,6 +51,7 @@ public interface CoderCallback { protected AVCodecContext codec_ctx; protected AVCodec codec; protected final Queue outQueue = new ArrayDeque<>(10); + protected final int maxOutQueue = 10; private final AtomicBoolean isProcessing = new AtomicBoolean(false); // Set false to indicate packets should no longer be accepted final Object contextLock = new Object(); Class inputClass; @@ -96,6 +97,16 @@ private void submitTask(Runnable task) { } } + protected boolean addOutFrame(O outFrame) { + if (outQueue.size() < maxOutQueue) { + outQueue.add(outFrame); + return true; + } else { + deallocateOutputPacket(outFrame); + return false; + } + } + public boolean isReady() { return isProcessing.get(); } diff --git a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/coders/Decoder.java b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/coders/Decoder.java index 6030ee755..4f144bf45 100644 --- a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/coders/Decoder.java +++ b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/coders/Decoder.java @@ -63,7 +63,7 @@ protected synchronized void processInputPacket(AVPacket inputPacket) { while (avcodec_receive_frame(codec_ctx, outputPacket) >= 0) { if (!outputPacket.isNull()) { - outQueue.add(outputPacket); + addOutFrame(outputPacket); } outputPacket = av_frame_alloc(); } diff --git a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/coders/Encoder.java b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/coders/Encoder.java index c0310429c..cff1401a2 100644 --- a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/coders/Encoder.java +++ b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/coders/Encoder.java @@ -87,7 +87,7 @@ protected synchronized void processInputPacket(AVFrame inputPacket) { while (avcodec_receive_packet(codec_ctx, outputPacket) >= 0) { if (!outputPacket.isNull()) { - outQueue.add(outputPacket); + addOutFrame(outputPacket); } outputPacket = av_packet_alloc(); } diff --git a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/coders/SwScaler.java b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/coders/SwScaler.java index a8d1233cb..6347d51a5 100644 --- a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/coders/SwScaler.java +++ b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/coders/SwScaler.java @@ -69,7 +69,7 @@ protected synchronized void processInputPacket(AVFrame inputPacket) { outputPacket.height(outHeight); av_frame_get_buffer(outputPacket, 0); sws_scale_frame(swsContext, outputPacket, inputPacket); - outQueue.add(outputPacket); + addOutFrame(outputPacket); } } diff --git a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/formatters/FrameFormatter.java b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/formatters/FrameFormatter.java index 71e61d73f..a0f1341dc 100644 --- a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/formatters/FrameFormatter.java +++ b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/formatters/FrameFormatter.java @@ -102,7 +102,7 @@ private void setFrameDataPacked(AVFrame newFrame, byte[] inputData) { BytePointer dst = newFrame.data(0); int linesize = newFrame.linesize(0); - int bytesPerPixel = av_get_bits_per_pixel(av_pix_fmt_desc_get(pixFmt)) / 8; + int bytesPerPixel = (av_get_bits_per_pixel(av_pix_fmt_desc_get(pixFmt)) + 7) / 8; int rowBytes = width * bytesPerPixel; @@ -121,7 +121,7 @@ private AVFrame generateFrame() { newFrame.format(pixFmt); newFrame.width(width); newFrame.height(height); - int ret = av_frame_get_buffer(newFrame, 32); + int ret = av_frame_get_buffer(newFrame, 0); if (ret < 0) { av_frame_free(newFrame); throw new IllegalStateException("Could not allocate AVFrame buffer, ffmpeg error: " + ret); diff --git a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/formatters/PacketFormatter.java b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/formatters/PacketFormatter.java index 394e92570..46699ff2c 100644 --- a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/formatters/PacketFormatter.java +++ b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/formatters/PacketFormatter.java @@ -20,8 +20,10 @@ public class PacketFormatter implements AVByteFormatter { @Override public AVPacket convertInput(byte[] inputData) { AVPacket newPacket = av_packet_alloc(); - //av_new_packet(newPacket, inputData.length); - av_packet_from_data(newPacket, inputData, inputData.length); + av_new_packet(newPacket, inputData.length); + newPacket.data().position(0); + newPacket.data().limit(inputData.length); + newPacket.data().put(inputData); return newPacket; } From b22c7adb0744848dfc92da8de11a654b5f01f81f Mon Sep 17 00:00:00 2001 From: kyle-fitzp Date: Wed, 13 May 2026 17:10:32 -0500 Subject: [PATCH 07/10] Fixes to pixel formats, inject headers in encoder --- .../video/transcoder/FFMpegTranscoder.java | 36 +++--- .../transcoder/FFmpegTranscoderConfig.java | 8 +- .../transcoder/FFmpegTranscoderProcess.java | 12 +- .../video/transcoder/coders/Codec.java | 109 +++++++++++------- .../video/transcoder/coders/Decoder.java | 4 +- .../video/transcoder/coders/Encoder.java | 56 ++++++++- .../video/transcoder/coders/SwScaler.java | 23 ++-- .../transcoder/formatters/FrameFormatter.java | 75 ++++++++---- .../transcoder/helpers/FullCodecEnum.java | 16 +++ .../transcoder/helpers/FullPixelEnum.java | 16 +++ .../impl/service/sos/video/MP4Serializer.java | 2 +- 11 files changed, 258 insertions(+), 99 deletions(-) diff --git a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/FFMpegTranscoder.java b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/FFMpegTranscoder.java index 09a23116f..1a8d7af87 100644 --- a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/FFMpegTranscoder.java +++ b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/FFMpegTranscoder.java @@ -161,37 +161,34 @@ private void initCodecs() { Decoder decoder = null; Encoder encoder = null; SwScaler swScaler = null; - //FullPixelEnum decOutPixFmt = outCodec.pixelFmt; - //FullPixelEnum encInPixFmt = outCodec.pixelFmt; if (!isUncompressed(inCodec)) { decoder = new Decoder(inCodec, outCodec, decOptions); - outCodec.pixelFmt = decoder.init(); + inCodec.pixelFmt = decoder.init(); + if (inCodec.pixelFmt == null) + inCodec.pixelFmt = FullPixelEnum.YUV420P; //videoProcs.add(decoder); } + if (!isUncompressed(outCodec)) { encoder = new Encoder(inCodec, outCodec, encOptions); - inCodec.pixelFmt = encoder.init(); + outCodec.pixelFmt = encoder.init(); //videoProcs.add(encoder); } - logger.info("Input pixel format: {}, Output pixel format: {}", inCodec.pixelFmt, outCodec.pixelFmt); - - if (inCodec.pixelFmt == null || outCodec.pixelFmt == null) { logger.warn("Pixel format is null"); } - - if (width != outWidth || height != outHeight - || (isUncompressed(inCodec) && isUncompressed(outCodec)) - || (inCodec.pixelFmt != outCodec.pixelFmt)) { - - swScaler = new SwScaler(inCodec, outCodec, width, height, outWidth, outHeight); - swScaler.init(); - //videoProcs.add(swScaler); - } + // Always want swScaler. Decoder can output frames in a format different from what was set. + swScaler = new SwScaler(inCodec, outCodec, width, height, outWidth, outHeight); + swScaler.init(); + //videoProcs.add(swScaler); if (decoder != null) { videoProcs.add(decoder); } if (swScaler != null) { videoProcs.add(swScaler); } if (encoder != null) { videoProcs.add(encoder); } + logger.info("Input pixel format: {}, Output pixel format: {}", inCodec.pixelFmt, outCodec.pixelFmt); + + if (inCodec.pixelFmt == null || outCodec.pixelFmt == null) { logger.warn("Pixel format is null"); } + } private void initPipeline() { @@ -329,8 +326,8 @@ public void init() throws ProcessException setImgEncoding(); initCodecOptions(); - initFormatters(); initCodecs(); + initFormatters(); initPipeline(); imgOut.setData(new DataBlockCompressed()); @@ -471,12 +468,15 @@ private void publishFrameData(byte[] frameData) { ((DataBlockCompressed) imgOut.getData()).setUnderlyingObject(frameData.clone()); // also copy frame timestamp - double ts; + double ts = System.currentTimeMillis() / 1000d; + /* if (inputTimeStamp != null && inputTimeStamp.getData() != null) { ts = inputTimeStamp.getData().getDoubleValue(); } else { ts = System.currentTimeMillis(); } + + */ outputTimeStamp.getData().setDoubleValue(ts); try { //logger.debug("Publishing"); diff --git a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/FFmpegTranscoderConfig.java b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/FFmpegTranscoderConfig.java index b14c3bc5c..8f59b221a 100644 --- a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/FFmpegTranscoderConfig.java +++ b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/FFmpegTranscoderConfig.java @@ -6,9 +6,12 @@ public class FFmpegTranscoderConfig extends FFmpegProcessConfig { @DisplayInfo.Required - @DisplayInfo(label="Input Codec") + @DisplayInfo(label="Input Format") public CodecEnum inCodec = CodecEnum.H264; + @DisplayInfo(label="Input Format Override") + public String inCodecOverride = ""; + @DisplayInfo(label="Automatically Detect Input Codec", desc="If enabled, process will attempt to determine the input codec." + " If a codec could not be determined from the input data, it will fall back to the selected Input Codec.") public boolean detectInput = false; @@ -17,6 +20,9 @@ public class FFmpegTranscoderConfig extends FFmpegProcessConfig { @DisplayInfo(label="Output Codec") public CodecEnum outCodec = CodecEnum.H264; + @DisplayInfo(label="Output Format Override") + public String outCodecOverride = ""; + @DisplayInfo(label="Output Width") public Integer outputWidth = null; diff --git a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/FFmpegTranscoderProcess.java b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/FFmpegTranscoderProcess.java index a6fe55895..c6d0073a1 100644 --- a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/FFmpegTranscoderProcess.java +++ b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/FFmpegTranscoderProcess.java @@ -19,10 +19,18 @@ public FFmpegTranscoderProcess() { @Override public void initExcProcess(IProcessExec executable) { DataBlock inCodec = new DataBlockString(1); - inCodec.setStringValue(((FFmpegTranscoderConfig)config).inCodec.toString()); + if (!((FFmpegTranscoderConfig)config).inCodecOverride.isBlank()) { + inCodec.setStringValue(((FFmpegTranscoderConfig)config).inCodecOverride); + } else { + inCodec.setStringValue(((FFmpegTranscoderConfig) config).inCodec.toString()); + } DataBlock outCodec = new DataBlockString(1); - outCodec.setStringValue(((FFmpegTranscoderConfig)config).outCodec.toString()); + if (!((FFmpegTranscoderConfig)config).outCodecOverride.isBlank()) { + outCodec.setStringValue(((FFmpegTranscoderConfig)config).outCodecOverride); + } else { + outCodec.setStringValue(((FFmpegTranscoderConfig) config).outCodec.toString()); + } DataBlockInt outWidthBlock = new DataBlockInt(1); if (((FFmpegTranscoderConfig) config).outputWidth != null) { diff --git a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/coders/Codec.java b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/coders/Codec.java index b47597be7..a2fc7c48f 100644 --- a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/coders/Codec.java +++ b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/coders/Codec.java @@ -6,7 +6,7 @@ import java.util.HashMap; import java.util.Map; import java.util.Queue; -import java.util.concurrent.ExecutionException; +import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; @@ -24,7 +24,6 @@ import org.bytedeco.javacpp.PointerPointer; import org.sensorhub.impl.process.video.transcoder.helpers.CodecInfo; import org.sensorhub.impl.process.video.transcoder.helpers.CodecOptions; -import org.sensorhub.impl.process.video.transcoder.helpers.FullCodecEnum; import org.sensorhub.impl.process.video.transcoder.helpers.FullPixelEnum; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -50,14 +49,17 @@ public interface CoderCallback { CodecInfo outputFormat; protected AVCodecContext codec_ctx; protected AVCodec codec; - protected final Queue outQueue = new ArrayDeque<>(10); - protected final int maxOutQueue = 10; - private final AtomicBoolean isProcessing = new AtomicBoolean(false); // Set false to indicate packets should no longer be accepted + protected final Queue inQueue = new ConcurrentLinkedQueue<>(); + protected final Queue outQueue = new ConcurrentLinkedQueue<>(); + protected final int maxQueue = 10; + private final AtomicBoolean isProcessing = new AtomicBoolean(false); + private final AtomicBoolean isAlive = new AtomicBoolean(false); // Set false to indicate packets should no longer be accepted final Object contextLock = new Object(); Class inputClass; Class outputClass; CodecOptions options; AtomicBoolean isNotifying = new AtomicBoolean(false); + byte[] headers; public Codec(CodecInfo inFormatInfo, CodecInfo outFormatInfo, Class inputClass, Class outputClass, CodecOptions options) { super(); @@ -78,7 +80,7 @@ public FullPixelEnum init() { initContext(); initOptions(); var pixFmt = openContext(); - isProcessing.set(true); + isAlive.set(true); return pixFmt; } @@ -97,18 +99,24 @@ private void submitTask(Runnable task) { } } - protected boolean addOutFrame(O outFrame) { - if (outQueue.size() < maxOutQueue) { - outQueue.add(outFrame); - return true; - } else { - deallocateOutputPacket(outFrame); - return false; + protected void addOutPacket(O outPacket) { + while (outQueue.size() >= maxQueue) { + var packet = outQueue.poll(); + deallocateOutputPacket(packet); + } + outQueue.add(outPacket); + } + + protected void addInPacket(I inPacket) { + while (inQueue.size() >= maxQueue) { + var packet = inQueue.poll(); + deallocateInputPacket(packet); } + inQueue.add(inPacket); } public boolean isReady() { - return isProcessing.get(); + return isAlive.get(); } public Class getOutputClass() { @@ -124,10 +132,16 @@ public Class getInputClass() { protected FullPixelEnum openContext() { int ret; FullPixelEnum pixelFmt; + + codec_ctx.flags(codec_ctx.flags() | AV_CODEC_FLAG_GLOBAL_HEADER); + if ((ret = avcodec_open2(codec_ctx, codec, (PointerPointer) null)) < 0) { logFFmpeg(ret); throw new IllegalStateException("Error opening codec " + codec.name().getString()); } + + headers = getExtradata(codec_ctx); + try { var desc = av_pix_fmt_desc_get(codec_ctx.pix_fmt()); pixelFmt = FullPixelEnum.valueOf(desc.name().getString().toUpperCase()); @@ -141,6 +155,15 @@ protected FullPixelEnum openContext() { return pixelFmt; } + protected static byte[] getExtradata(AVCodecContext codecCtx) { + if (codecCtx.extradata() == null || codecCtx.extradata_size() == 0) + return null; + + byte[] data = new byte[codecCtx.extradata_size()]; + codecCtx.extradata().get(data); + return data; + } + protected static void logFFmpeg(int retCode) { BytePointer buf = new BytePointer(AV_ERROR_MAX_STRING_SIZE); av_strerror(retCode, buf, buf.capacity()); @@ -152,7 +175,6 @@ protected static void setCodecPixFmt(AVCodecContext codec_ctx, FullPixelEnum des PointerPointer pixelFmts = new PointerPointer<>(1); avcodec_get_supported_config(codec_ctx, null, AV_CODEC_CONFIG_PIX_FORMAT, 0, pixelFmts, (IntPointer) null); - IntPointer fmts = pixelFmts.get(IntPointer.class, 0); // If null, all formats are supported if (fmts == null || fmts.isNull()) { @@ -173,12 +195,11 @@ protected static void setCodecPixFmt(AVCodecContext codec_ctx, FullPixelEnum des } if (!found) { logger.warn("Preferred pixel format for codec {} could not be found", codecString); - codec_ctx.pix_fmt(fmts.get(0)); + IntPointer loss = new IntPointer(1); + int fmt = avcodec_find_best_pix_fmt_of_list(fmts, desiredFmt.ffmpegId, 0, loss); + codec_ctx.pix_fmt(fmt); } } - - if (fmts != null) - fmts.deallocate(); pixelFmts.deallocate(); } @@ -197,12 +218,12 @@ protected void initOptions() { codec_ctx.width(options.width()); codec_ctx.height(options.height()); + codec_ctx.time_base(av_make_q(1, 90000)); + if (options.fps() > 0) { codec_ctx.framerate(av_make_q(options.fps(), 1)); - codec_ctx.time_base(av_make_q(1, options.fps())); } else { codec_ctx.framerate(av_make_q(25, 1)); - codec_ctx.time_base(av_make_q(1, 25)); } if (inputFormat.codec.ffmpegId == AV_CODEC_ID_H264) { @@ -227,27 +248,37 @@ protected void initOptions() { protected abstract O cloneOutput(O packet); protected abstract void processInputPacket(I inputPacket); - // Take data from input queue and send to encoder/decoder + public void submitInputPacket(I inputPacket) { synchronized (contextLock) { - if (inputPacket == null || !isProcessing.get()) { + if (!isAlive.get()) { return; } - submitTask(() -> { - // Process the input - processInputPacket(inputPacket); - deallocateInputPacket(inputPacket); - - if (!outQueue.isEmpty() && isNotifying.compareAndSet(false, true)) { - outputExecutor.submit(() -> { - while (!outQueue.isEmpty()) { - var outputPacket = outQueue.poll(); - notifyCallbacks(outputPacket); - } - isNotifying.set(false); - }); - } - }); + if (inputPacket != null) { + addInPacket(inputPacket); + } + if (isProcessing.compareAndSet(false, true)) { + submitTask(() -> { + // Process the input + while (!inQueue.isEmpty()) { + I inPacket = inQueue.poll(); + processInputPacket(inPacket); + deallocateInputPacket(inPacket); + } + + if (!outQueue.isEmpty() && isNotifying.compareAndSet(false, true)) { + outputExecutor.submit(() -> { + while (!outQueue.isEmpty()) { + O outputPacket = outQueue.poll(); + notifyCallbacks(outputPacket); + } + isNotifying.set(false); + }); + } + + isProcessing.set(false); + }); + } } } @@ -280,7 +311,7 @@ public void unregisterAllCallbacks() { @Override public void close() { synchronized (contextLock) { - if (isProcessing.compareAndSet(true, false)) { + if (isAlive.compareAndSet(true, false)) { // Submit cleanup *before* shutdown so it is the last task to run executor.submit(this::cleanup); diff --git a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/coders/Decoder.java b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/coders/Decoder.java index 4f144bf45..f8340644c 100644 --- a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/coders/Decoder.java +++ b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/coders/Decoder.java @@ -20,7 +20,7 @@ protected void initContext() { codec = avcodec_find_decoder(inputFormat.codec.ffmpegId); codec_ctx = avcodec_alloc_context3(codec); codec_ctx.codec_id(inputFormat.codec.ffmpegId); - setCodecPixFmt(codec_ctx, outputFormat.pixelFmt); + //setCodecPixFmt(codec_ctx, outputFormat.pixelFmt); //codec_ctx.pix_fmt(outputFormat.pixelFmt().ffmpegId); } } @@ -63,7 +63,7 @@ protected synchronized void processInputPacket(AVPacket inputPacket) { while (avcodec_receive_frame(codec_ctx, outputPacket) >= 0) { if (!outputPacket.isNull()) { - addOutFrame(outputPacket); + addOutPacket(outputPacket); } outputPacket = av_frame_alloc(); } diff --git a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/coders/Encoder.java b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/coders/Encoder.java index cff1401a2..179f57663 100644 --- a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/coders/Encoder.java +++ b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/coders/Encoder.java @@ -2,26 +2,44 @@ import org.bytedeco.ffmpeg.avcodec.AVPacket; import org.bytedeco.ffmpeg.avutil.AVFrame; -import org.bytedeco.ffmpeg.avutil.AVPixFmtDescriptor; -import org.bytedeco.javacpp.IntPointer; -import org.bytedeco.javacpp.Pointer; -import org.bytedeco.javacpp.PointerPointer; +import org.bytedeco.javacpp.BytePointer; import org.sensorhub.impl.process.video.transcoder.helpers.CodecInfo; import org.sensorhub.impl.process.video.transcoder.helpers.CodecOptions; import org.sensorhub.impl.process.video.transcoder.helpers.FullCodecEnum; +import org.sensorhub.impl.process.video.transcoder.helpers.FullPixelEnum; import static org.bytedeco.ffmpeg.global.avcodec.*; import static org.bytedeco.ffmpeg.global.avutil.*; public class Encoder extends Codec { + volatile long pts = 0; + long timebaseNum, timebaseDen, framerateNum, framerateDen; + final int GOP_SIZE = 10; + volatile long frameSinceGop = 0; + final Object timestampLock = new Object(); + public Encoder(CodecInfo inFormatInfo, CodecInfo outFormatInfo, CodecOptions options) { super(inFormatInfo, outFormatInfo, AVFrame.class, AVPacket.class, options); } + // Override so we can get the timebase + @Override + public FullPixelEnum init() { + var pixfmt = super.init(); + timebaseNum = codec_ctx.time_base().num(); + timebaseDen = codec_ctx.time_base().den(); + framerateNum = codec_ctx.framerate().num(); + framerateDen = codec_ctx.framerate().den(); + //GOP_SIZE = codec_ctx.gop_size(); + return pixfmt; + } + @Override protected void initContext() { synchronized (contextLock) { + pts = 0; + // For H264, prefer x264 over OpenH264 — better option compatibility if (outputFormat.codec == FullCodecEnum.H264) { codec = avcodec_find_encoder_by_name("libx264"); @@ -42,6 +60,9 @@ protected void initContext() { throw new IllegalStateException("Could not allocate encoder context for: " + codec.name().getString()); } + codec_ctx.gop_size(GOP_SIZE); + codec_ctx.max_b_frames(0); + setCodecPixFmt(codec_ctx, inputFormat.pixelFmt); logger.debug("Using encoder: {}", codec.name().getString()); @@ -75,6 +96,21 @@ protected AVPacket cloneOutput(AVPacket packet) { protected synchronized void processInputPacket(AVFrame inputPacket) { if (inputPacket != null && !inputPacket.isNull()) { int ret; + /* + long timeStamp = pts; + // pts += 1/fps * 1/timebase + // fps must be <= 1/timebase + pts += (framerateDen * timebaseDen) / (framerateNum * timebaseNum); + inputPacket.pts(timeStamp); + inputPacket.pkt_dts(timeStamp); + + */ + + if (frameSinceGop++ >= GOP_SIZE) { + frameSinceGop = 0; + inputPacket.pict_type(AV_PICTURE_TYPE_I); + inputPacket.flags(inputPacket.flags() | AVFrame.AV_FRAME_FLAG_KEY); + } if ((ret = avcodec_send_frame(codec_ctx, inputPacket)) < 0) { //logger.warn("Error sending packet to encoder"); @@ -87,7 +123,17 @@ protected synchronized void processInputPacket(AVFrame inputPacket) { while (avcodec_receive_packet(codec_ctx, outputPacket) >= 0) { if (!outputPacket.isNull()) { - addOutFrame(outputPacket); + // Add headers if necessary/available + if (((outputPacket.flags() & AV_PKT_FLAG_KEY) != 0) && headers != null) { + int originalSize = outputPacket.size(); + byte[] original = new byte[originalSize]; + outputPacket.data().capacity(originalSize).position(0).get(original); + av_grow_packet(outputPacket, headers.length); + BytePointer dst = outputPacket.data().capacity((long) headers.length + originalSize); + dst.position(0).put(headers); + dst.position(headers.length).put(original); + } + addOutPacket(outputPacket); } outputPacket = av_packet_alloc(); } diff --git a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/coders/SwScaler.java b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/coders/SwScaler.java index 6347d51a5..a87beac4c 100644 --- a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/coders/SwScaler.java +++ b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/coders/SwScaler.java @@ -62,15 +62,22 @@ protected AVFrame cloneOutput(AVFrame packet) { @Override protected synchronized void processInputPacket(AVFrame inputPacket) { - if (inputPacket != null) { - AVFrame outputPacket = av_frame_alloc(); - outputPacket.format(outputFormat.pixelFmt.ffmpegId); - outputPacket.width(outWidth); - outputPacket.height(outHeight); - av_frame_get_buffer(outputPacket, 0); - sws_scale_frame(swsContext, outputPacket, inputPacket); - addOutFrame(outputPacket); + if (inputPacket == null) { return; } + + // FFmpeg can be weird when it comes to decoder pixfmt. May need to change + // on the fly (usually first frame). + if (inputPacket.format() != inputFormat.pixelFmt.ffmpegId) { + inputFormat.pixelFmt = FullPixelEnum.fromId(inputPacket.format()); + initContext(); } + + AVFrame outputPacket = av_frame_alloc(); + outputPacket.format(outputFormat.pixelFmt.ffmpegId); + outputPacket.width(outWidth); + outputPacket.height(outHeight); + av_frame_get_buffer(outputPacket, 0); + sws_scale_frame(swsContext, outputPacket, inputPacket); + addOutPacket(outputPacket); } @Override diff --git a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/formatters/FrameFormatter.java b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/formatters/FrameFormatter.java index a0f1341dc..dd2e30b31 100644 --- a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/formatters/FrameFormatter.java +++ b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/formatters/FrameFormatter.java @@ -47,7 +47,6 @@ public FrameFormatter(int width, int height, int pixFmt) { planeHeights[0] = height; planeSizes[0] = width * height * ((av_get_bits_per_pixel(desc) + 7) / 8); } - totalSize = calcByteSize(planeSizes); } @@ -85,12 +84,15 @@ private void setFrameDataPlanar(AVFrame newFrame, byte[] inputData) { int h = planeHeights[p]; int stride = newFrame.linesize(p); - BytePointer dst = newFrame.data(p).position(0); + BytePointer dst = newFrame.data(p) + .capacity((long) stride * h) + .position(0); - int rowBytes = w; + int bitsPerComponent = desc.comp(p < 1 ? 0 : 1).depth(); + int rowBytes = w * ((bitsPerComponent + 7) / 8); for (int y = 0; y < h; y++) { - dst.position(y * stride); + dst.position((long) y * stride); dst.put(inputData, offset + y * rowBytes, rowBytes); } @@ -99,10 +101,12 @@ private void setFrameDataPlanar(AVFrame newFrame, byte[] inputData) { } private void setFrameDataPacked(AVFrame newFrame, byte[] inputData) { - BytePointer dst = newFrame.data(0); int linesize = newFrame.linesize(0); + BytePointer dst = newFrame.data(0) + .capacity((long) linesize * height) + .position(0); - int bytesPerPixel = (av_get_bits_per_pixel(av_pix_fmt_desc_get(pixFmt)) + 7) / 8; + int bytesPerPixel = (av_get_bits_per_pixel(desc) + 7) / 8; int rowBytes = width * bytesPerPixel; @@ -137,29 +141,53 @@ public byte[] convertOutput(AVFrame outputFrame) { } byte[] out = new byte[totalSize]; - int offset = 0; if (isPlanar) { - for (int plane = 0; plane < planeCount; plane++) { - int w = planeWidth(plane); - int h = planeHeight(plane); - int srcStride = outputFrame.linesize(plane); - - BytePointer src = outputFrame.data(plane).position(0); - - for (int y = 0; y < h; y++) { - src.position(y * srcStride); - src.get(out, offset, w); - offset += w; - } - } + getFrameDataPlanar(outputFrame, out); } else { - BytePointer src = outputFrame.data(0); - src.get(out, 0, totalSize); + getFrameDataPacked(outputFrame, out); } return out; } + private void getFrameDataPlanar(AVFrame outputFrame, byte[] out) { + int offset = 0; + for (int plane = 0; plane < planeCount; plane++) { + int w = planeWidth(plane); + int h = planeHeight(plane); + int srcStride = outputFrame.linesize(plane); + + BytePointer src = outputFrame.data(plane) + .capacity((long) srcStride * h) + .position(0); + + int bitsPerComponent = desc.comp(plane < 1 ? 0 : 1).depth(); + int rowBytes = w * ((bitsPerComponent + 7) / 8); + + for (int y = 0; y < h; y++) { + src.position((long) y * srcStride); + src.get(out, offset, rowBytes); + offset += rowBytes; + } + } + } + + private void getFrameDataPacked(AVFrame outputFrame, byte[] out) { + int bytesPerPixel = (av_get_bits_per_pixel(desc) + 7) / 8; + int rowBytes = width * bytesPerPixel; + int linesize = outputFrame.linesize(0); + + BytePointer src = outputFrame.data(0) + .capacity((long) linesize * height); + + int offset = 0; + for (int y = 0; y < height; y++) { + src.position((long) y * linesize) + .get(out, offset, rowBytes); + offset += rowBytes; + } + } + private int planeWidth(int plane) { if (plane == 0) return width; @@ -196,7 +224,8 @@ private static void calculatePlaneSizes(AVPixFmtDescriptor desc, int width, int } for (int p = 0; p < planeSizes.length; p++) { - planeSizes[p] = planeWidths[p] * planeHeights[p]; + int bytesPerSample = (desc.comp(p == 0 ? 0 : 1).depth() + 7) / 8; + planeSizes[p] = planeWidths[p] * planeHeights[p] * bytesPerSample; } } } diff --git a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/helpers/FullCodecEnum.java b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/helpers/FullCodecEnum.java index c488614d6..1db8625f6 100644 --- a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/helpers/FullCodecEnum.java +++ b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/helpers/FullCodecEnum.java @@ -2,6 +2,9 @@ import org.bytedeco.ffmpeg.global.avcodec; +import java.util.HashMap; +import java.util.Map; + public enum FullCodecEnum { H264(avcodec.AV_CODEC_ID_H264), HEVC(avcodec.AV_CODEC_ID_HEVC), @@ -77,6 +80,19 @@ public enum FullCodecEnum { public int ffmpegId; + private static final Map BY_ID; + + static { + BY_ID = new HashMap<>(); + for (FullCodecEnum fmt : values()) { + BY_ID.put(fmt.ffmpegId, fmt); + } + } + + public static FullCodecEnum fromId(int ffmpegId) { + return BY_ID.getOrDefault(ffmpegId, RAWVIDEO); + } + FullCodecEnum(int ffmpegId) { this.ffmpegId = ffmpegId; diff --git a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/helpers/FullPixelEnum.java b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/helpers/FullPixelEnum.java index 69effa6f7..be7002944 100644 --- a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/helpers/FullPixelEnum.java +++ b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/helpers/FullPixelEnum.java @@ -2,7 +2,11 @@ import org.bytedeco.ffmpeg.global.avutil; +import java.util.HashMap; +import java.util.Map; + public enum FullPixelEnum { + NONE(avutil.AV_PIX_FMT_NONE), // --- Planar YUV --- @@ -181,6 +185,18 @@ public enum FullPixelEnum { P416LE(avutil.AV_PIX_FMT_P416LE); public int ffmpegId; + private static final Map BY_ID; + + static { + BY_ID = new HashMap<>(); + for (FullPixelEnum fmt : values()) { + BY_ID.put(fmt.ffmpegId, fmt); + } + } + + public static FullPixelEnum fromId(int ffmpegId) { + return BY_ID.getOrDefault(ffmpegId, NONE); + } FullPixelEnum(int ffmpegId) { diff --git a/services/sensorhub-service-video/src/main/java/org/sensorhub/impl/service/sos/video/MP4Serializer.java b/services/sensorhub-service-video/src/main/java/org/sensorhub/impl/service/sos/video/MP4Serializer.java index 1825381dd..11359cc00 100644 --- a/services/sensorhub-service-video/src/main/java/org/sensorhub/impl/service/sos/video/MP4Serializer.java +++ b/services/sensorhub-service-video/src/main/java/org/sensorhub/impl/service/sos/video/MP4Serializer.java @@ -72,7 +72,7 @@ public void sendNextFrame(DataBlock nextFrame, OutputStream os) throws IOExcepti ByteBuffer nals = ByteBuffer.wrap(frameData); // debug - //os.write(frameData); + os.write(frameData); //os.flush(); // look for next nal unit From e7fd22433bef97f209e0f780c055622de9769b7e Mon Sep 17 00:00:00 2001 From: kyle-fitzp Date: Thu, 14 May 2026 14:57:08 -0500 Subject: [PATCH 08/10] Reject frames with incorrect pixel format --- .../process/video/transcoder/coders/Encoder.java | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/coders/Encoder.java b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/coders/Encoder.java index 179f57663..df1666f9c 100644 --- a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/coders/Encoder.java +++ b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/coders/Encoder.java @@ -96,15 +96,10 @@ protected AVPacket cloneOutput(AVPacket packet) { protected synchronized void processInputPacket(AVFrame inputPacket) { if (inputPacket != null && !inputPacket.isNull()) { int ret; - /* - long timeStamp = pts; - // pts += 1/fps * 1/timebase - // fps must be <= 1/timebase - pts += (framerateDen * timebaseDen) / (framerateNum * timebaseNum); - inputPacket.pts(timeStamp); - inputPacket.pkt_dts(timeStamp); - - */ + if (inputPacket.format() != codec_ctx.pix_fmt()) { + throw new IllegalArgumentException("AVFrame pixel format: " + FullPixelEnum.fromId(inputPacket.format()) + + " incompatible with codec pixel format: " + FullPixelEnum.fromId(codec_ctx.pix_fmt())); + } if (frameSinceGop++ >= GOP_SIZE) { frameSinceGop = 0; From b775ebd18882eadc399187310e80ee314c02a083 Mon Sep 17 00:00:00 2001 From: kyle-fitzp Date: Thu, 14 May 2026 14:57:32 -0500 Subject: [PATCH 09/10] Update config and readme --- .../process/video/transcoder/CodecEnum.java | 3 +- .../transcoder/FFmpegTranscoderConfig.java | 6 +- .../impl/process/video/transcoder/README.md | 183 ++++++++++++++++- .../impl/process/video/transcoder/README.md | 185 +++++++++++++++++- 4 files changed, 357 insertions(+), 20 deletions(-) diff --git a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/CodecEnum.java b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/CodecEnum.java index 42474af40..d0cdd3db3 100644 --- a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/CodecEnum.java +++ b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/CodecEnum.java @@ -11,8 +11,7 @@ public enum CodecEnum { //AUTO("auto"), H264(AV_CODEC_ID_H264), - H265(AV_CODEC_ID_H265), - HEVC(AV_CODEC_ID_HEVC), // HEVC and H265 are the same. Having both in this enum helps with auto codec detection. + HEVC(AV_CODEC_ID_HEVC), MJPEG(AV_CODEC_ID_MJPEG), VP8(AV_CODEC_ID_VP8), VP9(AV_CODEC_ID_VP9), diff --git a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/FFmpegTranscoderConfig.java b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/FFmpegTranscoderConfig.java index 8f59b221a..365093129 100644 --- a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/FFmpegTranscoderConfig.java +++ b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/FFmpegTranscoderConfig.java @@ -12,12 +12,12 @@ public class FFmpegTranscoderConfig extends FFmpegProcessConfig { @DisplayInfo(label="Input Format Override") public String inCodecOverride = ""; - @DisplayInfo(label="Automatically Detect Input Codec", desc="If enabled, process will attempt to determine the input codec." - + " If a codec could not be determined from the input data, it will fall back to the selected Input Codec.") + @DisplayInfo(label="Automatically Detect Input Format", desc="If enabled, process will attempt to determine the input format." + + " If a codec could not be determined from the input data, it will fall back to the selected Input Format.") public boolean detectInput = false; @DisplayInfo.Required - @DisplayInfo(label="Output Codec") + @DisplayInfo(label="Output Format") public CodecEnum outCodec = CodecEnum.H264; @DisplayInfo(label="Output Format Override") diff --git a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/README.md b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/README.md index 7ccf4a2db..fd89f33ab 100644 --- a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/README.md +++ b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/README.md @@ -2,7 +2,8 @@ ## Overview -This module provides a process module that can decode and/or encode video. +FFmpeg video decode/encode/transcode module. +Input and output may be compressed or uncompressed video. ## Configuration @@ -24,17 +25,185 @@ When added to an OpenSensorHub node, the process has the following configuration - **Video Source:** A module with video output. Once the transcoder starts, video from this source module will be decoded/encoded and outputted from the transcoder. - - **Input Codec:** (Optional) - The codec used for decoding the incoming video data. If incoming video is uncompressed, + - **Input Format:** (Optional) + The format used for decoding the incoming video data. If incoming video is uncompressed, select either RGB or YUV. - - **Output Codec:** - The codec used for encoding the outgoing video data. If outgoing video should be uncompressed, + - **Input Format Override:** (Optional) + Manually specify any input codec or pixel format, allowing for any format not available in the short Input Codec list. + Only required if the format is not in the Input Format list. Otherwise, leave blank. + A full list of codecs and pixel formats is available at the end of this file. + - **Output Format:** + The format used for encoding the outgoing video data. If outgoing video should be uncompressed, select either RGB or YUV. + - **Output Format Override:** (Optional) + Manually specify any output codec or pixel format, allowing for any format not available in the short Output Codec list. + Only required if the format is not in the Output Format list. Otherwise, leave blank. + A full list of codecs and pixel formats is available at the end of this file. - **Output Width:** (Optional) The width of the output video frame. Leave this empty to avoid scaling the video frame size. - **Output Height:** (Optional) The height of the output video frame. Leave this empty to avoid scaling the video frame size. - **Auto Start:** If checked, automatically start this sensor when the OpenSensorHub node is launched. - - **Automatically Detect Input Codec:** - If checked, automatically determine the input video codec based on the input's encoding information. \ No newline at end of file + - **Automatically Detect Input Format:** + If checked, automatically determine the input video format based on the input's encoding information. + +## All Supported Formats +This list contains all the supported codecs and pixel formats. To use a format, copy it into one of the format +override config fields. + +| Codecs (Compressed Video) | Pixel Formats (Uncompressed Video) | +|---------------------------|------------------------------------| +| H264 | YUV420P | +| HEVC | YUV422P | +| MJPEG | YUV444P | +| VP8 | YUV410P | +| VP9 | YUV411P | +| MPEG2 | YUV440P | +| MPEG4 | YUVJ420P | +| AV1 | YUVJ422P | +| THEORA | YUVJ444P | +| MPEG1VIDEO | YUVJ440P | +| WMV1 | YUV420P9BE | +| WMV2 | YUV420P9LE | +| WMV3 | YUV420P10BE | +| VC1 | YUV420P10LE | +| FLV1 | YUV420P12BE | +| FLASHSV | YUV420P12LE | +| FLASHSV2 | YUV420P14BE | +| RV10 | YUV420P14LE | +| RV20 | YUV420P16BE | +| RV30 | YUV420P16LE | +| RV40 | YUV422P9BE | +| CINEPAK | YUV422P9LE | +| INDEO2 | YUV422P10BE | +| INDEO3 | YUV422P10LE | +| INDEO4 | YUV422P12BE | +| INDEO5 | YUV422P12LE | +| MSMPEG4V1 | YUV422P14BE | +| MSMPEG4V2 | YUV422P14LE | +| MSMPEG4V3 | YUV422P16BE | +| H261 | YUV422P16LE | +| H263 | YUV444P9BE | +| H263I | YUV444P9LE | +| H263P | YUV444P10BE | +| SNOW | YUV444P10LE | +| SVQ1 | YUV444P12BE | +| SVQ3 | YUV444P12LE | +| DVVIDEO | YUV444P14BE | +| HUFFYUV | YUV444P14LE | +| FFVHUFF | YUV444P16BE | +| FFV1 | YUV444P16LE | +| ASV1 | YUYV422 | +| ASV2 | UYVY422 | +| VCR1 | YVYU422 | +| CLJR | UYYVYY411 | +| MDEC | NV12 | +| ROQ | NV21 | +| INTERPLAY_VIDEO | NV16 | +| XAN_WC3 | NV20LE | +| XAN_WC4 | NV20BE | +| RPZA | NV24 | +| SMC | NV42 | +| GIF | RGB24 | +| PNG | BGR24 | +| PPM | ARGB | +| PBM | RGBA | +| PGM | ABGR | +| PAM | BGRA | +| BMP | RGB0 | +| TIFF | BGR0 | +| SGI | RGB8 | +| ALIAS_PIX | BGR8 | +| DPX | RGB4 | +| EXR | BGR4 | +| WEBP | RGB4_BYTE | +| DIRAC | BGR4_BYTE | +| DNXHD | RGB48BE | +| PRORES | RGB48LE | +| JPEG2000 | RGBA64BE | +| JPEGLS | RGBA64LE | +| HAP | BGR48BE | +| | BGR48LE | +| | BGRA64BE | +| | BGRA64LE | +| | RGB565BE | +| | RGB565LE | +| | RGB555BE | +| | RGB555LE | +| | RGB444BE | +| | RGB444LE | +| | BGR565BE | +| | BGR565LE | +| | BGR555BE | +| | BGR555LE | +| | BGR444BE | +| | BGR444LE | +| | GRAY8 | +| | GRAY8A | +| | GRAY9BE | +| | GRAY9LE | +| | GRAY10BE | +| | GRAY10LE | +| | GRAY12BE | +| | GRAY12LE | +| | GRAY14BE | +| | GRAY14LE | +| | GRAY16BE | +| | GRAY16LE | +| | MONOWHITE | +| | MONOBLACK | +| | YA8 | +| | YA16BE | +| | YA16LE | +| | YUVA420P | +| | YUVA422P | +| | YUVA444P | +| | YUVA420P9BE | +| | YUVA420P9LE | +| | YUVA422P9BE | +| | YUVA422P9LE | +| | YUVA444P9BE | +| | YUVA444P9LE | +| | YUVA420P10BE | +| | YUVA420P10LE | +| | YUVA422P10BE | +| | YUVA422P10LE | +| | YUVA444P10BE | +| | YUVA444P10LE | +| | YUVA420P16BE | +| | YUVA420P16LE | +| | YUVA422P16BE | +| | YUVA422P16LE | +| | YUVA444P16BE | +| | YUVA444P16LE | +| | BAYER_BGGR8 | +| | BAYER_RGGB8 | +| | BAYER_GBRG8 | +| | BAYER_GRBG8 | +| | BAYER_BGGR16LE | +| | BAYER_BGGR16BE | +| | BAYER_RGGB16LE | +| | BAYER_RGGB16BE | +| | BAYER_GBRG16LE | +| | BAYER_GBRG16BE | +| | BAYER_GRBG16LE | +| | BAYER_GRBG16BE | +| | GRAYF32BE | +| | GRAYF32LE | +| | PAL8 | +| | XYZ12LE | +| | XYZ12BE | +| | XTOP | +| | P010LE | +| | P010BE | +| | P016LE | +| | P016BE | +| | P210BE | +| | P210LE | +| | P410BE | +| | P410LE | +| | P216BE | +| | P216LE | +| | P416BE | +| | P416LE | \ No newline at end of file diff --git a/processing/sensorhub-process-ffmpeg/src/main/resources/org/sensorhub/impl/process/video/transcoder/README.md b/processing/sensorhub-process-ffmpeg/src/main/resources/org/sensorhub/impl/process/video/transcoder/README.md index 3ba1e0c72..fd89f33ab 100644 --- a/processing/sensorhub-process-ffmpeg/src/main/resources/org/sensorhub/impl/process/video/transcoder/README.md +++ b/processing/sensorhub-process-ffmpeg/src/main/resources/org/sensorhub/impl/process/video/transcoder/README.md @@ -2,11 +2,12 @@ ## Overview -This module provides a process module that will can decode and encode video. +FFmpeg video decode/encode/transcode module. +Input and output may be compressed or uncompressed video. ## Configuration -When added to an OpenSensorHub node, the process has the following configuration properties: +When added to an OpenSensorHub node, the process has the following configuration options: - **General:** - **Module ID:** *Not editable.* @@ -24,17 +25,185 @@ When added to an OpenSensorHub node, the process has the following configuration - **Video Source:** A module with video output. Once the transcoder starts, video from this source module will be decoded/encoded and outputted from the transcoder. - - **Input Codec:** (Optional) - The codec used for decoding the incoming video data. If incoming video is uncompressed, + - **Input Format:** (Optional) + The format used for decoding the incoming video data. If incoming video is uncompressed, select either RGB or YUV. - - **Output Codec:** - The codec used for encoding the outgoing video data. If outgoing video should be uncompressed, + - **Input Format Override:** (Optional) + Manually specify any input codec or pixel format, allowing for any format not available in the short Input Codec list. + Only required if the format is not in the Input Format list. Otherwise, leave blank. + A full list of codecs and pixel formats is available at the end of this file. + - **Output Format:** + The format used for encoding the outgoing video data. If outgoing video should be uncompressed, select either RGB or YUV. + - **Output Format Override:** (Optional) + Manually specify any output codec or pixel format, allowing for any format not available in the short Output Codec list. + Only required if the format is not in the Output Format list. Otherwise, leave blank. + A full list of codecs and pixel formats is available at the end of this file. - **Output Width:** (Optional) The width of the output video frame. Leave this empty to avoid scaling the video frame size. - **Output Height:** (Optional) The height of the output video frame. Leave this empty to avoid scaling the video frame size. - **Auto Start:** If checked, automatically start this sensor when the OpenSensorHub node is launched. - - **Automatically Detect Input Codec:** - If checked, automatically determine the input video codec based on the input's encoding information. \ No newline at end of file + - **Automatically Detect Input Format:** + If checked, automatically determine the input video format based on the input's encoding information. + +## All Supported Formats +This list contains all the supported codecs and pixel formats. To use a format, copy it into one of the format +override config fields. + +| Codecs (Compressed Video) | Pixel Formats (Uncompressed Video) | +|---------------------------|------------------------------------| +| H264 | YUV420P | +| HEVC | YUV422P | +| MJPEG | YUV444P | +| VP8 | YUV410P | +| VP9 | YUV411P | +| MPEG2 | YUV440P | +| MPEG4 | YUVJ420P | +| AV1 | YUVJ422P | +| THEORA | YUVJ444P | +| MPEG1VIDEO | YUVJ440P | +| WMV1 | YUV420P9BE | +| WMV2 | YUV420P9LE | +| WMV3 | YUV420P10BE | +| VC1 | YUV420P10LE | +| FLV1 | YUV420P12BE | +| FLASHSV | YUV420P12LE | +| FLASHSV2 | YUV420P14BE | +| RV10 | YUV420P14LE | +| RV20 | YUV420P16BE | +| RV30 | YUV420P16LE | +| RV40 | YUV422P9BE | +| CINEPAK | YUV422P9LE | +| INDEO2 | YUV422P10BE | +| INDEO3 | YUV422P10LE | +| INDEO4 | YUV422P12BE | +| INDEO5 | YUV422P12LE | +| MSMPEG4V1 | YUV422P14BE | +| MSMPEG4V2 | YUV422P14LE | +| MSMPEG4V3 | YUV422P16BE | +| H261 | YUV422P16LE | +| H263 | YUV444P9BE | +| H263I | YUV444P9LE | +| H263P | YUV444P10BE | +| SNOW | YUV444P10LE | +| SVQ1 | YUV444P12BE | +| SVQ3 | YUV444P12LE | +| DVVIDEO | YUV444P14BE | +| HUFFYUV | YUV444P14LE | +| FFVHUFF | YUV444P16BE | +| FFV1 | YUV444P16LE | +| ASV1 | YUYV422 | +| ASV2 | UYVY422 | +| VCR1 | YVYU422 | +| CLJR | UYYVYY411 | +| MDEC | NV12 | +| ROQ | NV21 | +| INTERPLAY_VIDEO | NV16 | +| XAN_WC3 | NV20LE | +| XAN_WC4 | NV20BE | +| RPZA | NV24 | +| SMC | NV42 | +| GIF | RGB24 | +| PNG | BGR24 | +| PPM | ARGB | +| PBM | RGBA | +| PGM | ABGR | +| PAM | BGRA | +| BMP | RGB0 | +| TIFF | BGR0 | +| SGI | RGB8 | +| ALIAS_PIX | BGR8 | +| DPX | RGB4 | +| EXR | BGR4 | +| WEBP | RGB4_BYTE | +| DIRAC | BGR4_BYTE | +| DNXHD | RGB48BE | +| PRORES | RGB48LE | +| JPEG2000 | RGBA64BE | +| JPEGLS | RGBA64LE | +| HAP | BGR48BE | +| | BGR48LE | +| | BGRA64BE | +| | BGRA64LE | +| | RGB565BE | +| | RGB565LE | +| | RGB555BE | +| | RGB555LE | +| | RGB444BE | +| | RGB444LE | +| | BGR565BE | +| | BGR565LE | +| | BGR555BE | +| | BGR555LE | +| | BGR444BE | +| | BGR444LE | +| | GRAY8 | +| | GRAY8A | +| | GRAY9BE | +| | GRAY9LE | +| | GRAY10BE | +| | GRAY10LE | +| | GRAY12BE | +| | GRAY12LE | +| | GRAY14BE | +| | GRAY14LE | +| | GRAY16BE | +| | GRAY16LE | +| | MONOWHITE | +| | MONOBLACK | +| | YA8 | +| | YA16BE | +| | YA16LE | +| | YUVA420P | +| | YUVA422P | +| | YUVA444P | +| | YUVA420P9BE | +| | YUVA420P9LE | +| | YUVA422P9BE | +| | YUVA422P9LE | +| | YUVA444P9BE | +| | YUVA444P9LE | +| | YUVA420P10BE | +| | YUVA420P10LE | +| | YUVA422P10BE | +| | YUVA422P10LE | +| | YUVA444P10BE | +| | YUVA444P10LE | +| | YUVA420P16BE | +| | YUVA420P16LE | +| | YUVA422P16BE | +| | YUVA422P16LE | +| | YUVA444P16BE | +| | YUVA444P16LE | +| | BAYER_BGGR8 | +| | BAYER_RGGB8 | +| | BAYER_GBRG8 | +| | BAYER_GRBG8 | +| | BAYER_BGGR16LE | +| | BAYER_BGGR16BE | +| | BAYER_RGGB16LE | +| | BAYER_RGGB16BE | +| | BAYER_GBRG16LE | +| | BAYER_GBRG16BE | +| | BAYER_GRBG16LE | +| | BAYER_GRBG16BE | +| | GRAYF32BE | +| | GRAYF32LE | +| | PAL8 | +| | XYZ12LE | +| | XYZ12BE | +| | XTOP | +| | P010LE | +| | P010BE | +| | P016LE | +| | P016BE | +| | P210BE | +| | P210LE | +| | P410BE | +| | P410LE | +| | P216BE | +| | P216LE | +| | P416BE | +| | P416LE | \ No newline at end of file From 21b7f1d12832246bf9997799536b7c6ffd57bde7 Mon Sep 17 00:00:00 2001 From: kyle-fitzp Date: Thu, 14 May 2026 15:21:38 -0500 Subject: [PATCH 10/10] Add documentation to Codec --- .../video/transcoder/coders/Codec.java | 45 ++++++++++++++++--- 1 file changed, 40 insertions(+), 5 deletions(-) diff --git a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/coders/Codec.java b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/coders/Codec.java index a2fc7c48f..f1e01f8a7 100644 --- a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/coders/Codec.java +++ b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/coders/Codec.java @@ -2,7 +2,6 @@ import static org.bytedeco.ffmpeg.global.avutil.*; -import java.util.ArrayDeque; import java.util.HashMap; import java.util.Map; import java.util.Queue; @@ -28,10 +27,46 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +/** + * Consumes AVPackets/AVFrames and produces AVPackets/AVFrames. Asynchronously submit packets to the codec with + * {@link #submitInputPacket(Pointer input)}. Listen for output by submitting a {@link CodecCallback} (typically a lambda) + * to {@link #registerCallback(CodecCallback callback)}. If transcoding, use callbacks to connect the output of one + * {@link #Codec} instance to the {@link #submitInputPacket(Pointer input)} method of the next. + * + *

AVPacket/AVFrame ownership is transferred to the consumer/listener. The consumer shall be responsible for + * deallocating AVPackets/AVFrames by passing them to the {@link #deallocateOutputPacket(Pointer packet)} method of the + * codec which most recently produced them. ONLY deallocate AVPackets/AVFrames that are not in use by a + * {@link #Codec}. {@link #submitInputPacket(Pointer input)} will automatically deallocate its input. + * + *

Codec/Callback Sample: + * 

+ * {@code
+ * // Packet pipe between decoder, swscaler, encoder
+ *         for (int i = 0; i < codecList.size() - 1; i++) {
+ *             var nextCodec = codecList.get(i + 1);
+ *             codecList.get(i).registerCallback(packet -> {
+ *                 nextCodec.submitInputPacket(packet);
+ *             });
+ *         }
+ *
+ *         // Output
+ *         var finalProc = codecList.get(codecList.size() - 1);
+ *         finalProc.registerCallback(packet -> {
+ *             try {
+ *                 publishFrameData(outputFormatter.convertOutput(packet));
+ *             } finally {
+ *                 finalProc.deallocateOutputPacket(packet);
+ *             }
+ *         });
+ * }
+ *
+ * @param Input class (either AVFrame or AVPacket) + * @param Output class (either AVFrame or AVPacket) + */ public abstract class Codec implements AutoCloseable { - public interface CoderCallback { + public interface CodecCallback { // The recipient does not need to deallocate the output; this is done automatically public abstract void onPacket(O packet); } @@ -43,7 +78,7 @@ public interface CoderCallback { private final ExecutorService executor = Executors.newSingleThreadExecutor( r -> new Thread(r, "ffmpeg-codec-thread-" + codecNum)); private final ExecutorService outputExecutor = Executors.newSingleThreadExecutor(r -> new Thread(r, "ffmpeg-codec-output-thread-" + codecNum)); - private final Map, ExecutorService> callbackMap = new HashMap<>(); + private final Map, ExecutorService> callbackMap = new HashMap<>(); CodecInfo inputFormat; CodecInfo outputFormat; @@ -292,7 +327,7 @@ private void notifyCallbacks(O outputPacket) { deallocateOutputPacket(outputPacket); } - public void registerCallback(CoderCallback callback) { + public void registerCallback(CodecCallback callback) { if (!callbackMap.containsKey(callback)) { callbackMap.put(callback, Executors.newSingleThreadExecutor(r -> new Thread(r, "ffmpeg-codec-" + codecNum + "-callback-thread"))); } else { @@ -300,7 +335,7 @@ public void registerCallback(CoderCallback callback) { } } - public void unregisterCallback(CoderCallback callback) { + public void unregisterCallback(CodecCallback callback) { callbackMap.remove(callback); }