performance optimization

app.py

@@ -39,8 +39,7 @@ from tqdm import tqdm
 import imageio
 import av
 import uuid
-import tempfile
-import shutil
+import io
 from pathlib import Path
 from typing import Dict, Any, List, Optional, Tuple, Union
 
@@ -171,7 +170,7 @@ if not APP_STATE["torch_compile_applied"] and ENABLE_TORCH_COMPILATION:
 
 def frames_to_mp4_base64(frames, fps = 15):
     """
-    Convert frames directly to base64 data URI using PyAV.
+    Convert frames directly to base64 data URI using PyAV with in-memory file.
     
     Args:
         frames: List of numpy arrays (HWC, RGB, uint8)
@@ -185,14 +184,12 @@ def frames_to_mp4_base64(frames, fps = 15):
     
     height, width = frames[0].shape[:2]
     
-    # Create temporary file for MP4 encoding
-    temp_file = tempfile.NamedTemporaryFile(suffix='.mp4', delete=False)
-    temp_filepath = temp_file.name
-    temp_file.close()
+    # Create BytesIO "in memory file"
+    output_memory_file = io.BytesIO()
     
     try:
-        # Create container for MP4 format
-        container = av.open(temp_filepath, mode='w', format='mp4')
+        # Create container for MP4 format using in-memory file
+        container = av.open(output_memory_file, mode='w', format='mp4')
         
         # Add video stream with fast settings
         stream = container.add_stream('h264', rate=fps)
@@ -222,18 +219,14 @@ def frames_to_mp4_base64(frames, fps = 15):
         finally:
             container.close()
         
-        # Read the MP4 file and encode to base64
-        with open(temp_filepath, 'rb') as f:
-            video_data = f.read()
-            base64_data = base64.b64encode(video_data).decode('utf-8')
-            return f"data:video/mp4;base64,{base64_data}"
+        # Get video data from in-memory file and encode to base64
+        video_data = output_memory_file.getbuffer()
+        base64_data = base64.b64encode(video_data).decode('utf-8')
+        return f"data:video/mp4;base64,{base64_data}"
             
-    finally:
-        # Clean up temporary file
-        if os.path.exists(temp_filepath):
-            os.unlink(temp_filepath)
-    
-    return "data:video/mp4;base64,"
+    except Exception as e:
+        print(f"Error encoding video: {e}")
+        return "data:video/mp4;base64,"
 
 # note: we set use_taehv to be able to use other resolutions
 # this might impact performance

example_pyav.md

@@ -0,0 +1,91 @@
+We may use [PyAV](https://github.com/PyAV-Org/PyAV) for encoding "in memory file".
+
+PyAV is a Pythonic binding for the [FFmpeg](https://ffmpeg.org/) libraries.  
+The interface is relatively low level, but it allows us to do things that are not possible using other FFmpeg bindings.
+
+Here are the main stages for creating MP4 in memory using PyAV:
+
+*   Create BytesIO "in memory file":
+    
+         output_memory_file = io.BytesIO()
+        
+    
+*   Use PyAV to open "in memory file" as MP4 video output file:
+    
+         output = av.open(output_memory_file, 'w', format="mp4")
+        
+    
+*   Add H.264 video stream to the MP4 container, and set codec parameters:
+    
+         stream = output.add_stream('h264', str(fps))
+         stream.width = width
+         stream.height = height
+         stream.pix_fmt = 'yuv444p'
+         stream.options = {'crf': '17'}
+        
+    
+*   Iterate the OpenCV images, convert image to PyAV `VideoFrame`, encode, and "Mux":
+    
+         for i in range(n_frmaes):
+             img = make_sample_image(i)  # Create OpenCV image for testing (resolution 192x108, pixel format BGR).
+             frame = av.VideoFrame.from_ndarray(img, format='bgr24')
+             packet = stream.encode(frame)
+             output.mux(packet)
+        
+    
+*   Flush the encoder and close the "in memory" file:
+    
+         packet = stream.encode(None)
+         output.mux(packet)
+         output.close()
+        
+    
+
+* * *
+
+The following code samples encode 100 synthetic images to "in memory" MP4 memory file.  
+Each synthetic image applies OpenCV image, with sequential blue frame number (used for testing).  
+At the end, the memory file is written to `output.mp4` file for testing.
+
+    import numpy as np
+    import cv2
+    import av
+    import io
+    
+    n_frmaes = 100  # Select number of frames (for testing).
+    width, height, fps = 192, 108, 23.976  # Select video resolution and framerate.
+    
+    output_memory_file = io.BytesIO()  # Create BytesIO "in memory file".
+    
+    output = av.open(output_memory_file, 'w', format="mp4")  # Open "in memory file" as MP4 video output
+    stream = output.add_stream('h264', str(fps))  # Add H.264 video stream to the MP4 container, with framerate = fps.
+    stream.width = width  # Set frame width
+    stream.height = height  # Set frame height
+    stream.pix_fmt = 'yuv444p'   # Select yuv444p pixel format (better quality than default yuv420p).
+    stream.options = {'crf': '17'}  # Select low crf for high quality (the price is larger file size).
+    
+    
+    def make_sample_image(i):
+        """ Build synthetic "raw BGR" image for testing """
+        p = width//60
+        img = np.full((height, width, 3), 60, np.uint8)
+        cv2.putText(img, str(i+1), (width//2-p*10*len(str(i+1)), height//2+p*10), cv2.FONT_HERSHEY_DUPLEX, p, (255, 30, 30), p*2)  # Blue number
+        return img
+    
+    
+    # Iterate the created images, encode and write to MP4 memory file.
+    for i in range(n_frmaes):
+        img = make_sample_image(i)  # Create OpenCV image for testing (resolution 192x108, pixel format BGR).
+        frame = av.VideoFrame.from_ndarray(img, format='bgr24')  # Convert image from NumPy Array to frame.
+        packet = stream.encode(frame)  # Encode video frame
+        output.mux(packet)  # "Mux" the encoded frame (add the encoded frame to MP4 file).
+    
+    # Flush the encoder
+    packet = stream.encode(None)
+    output.mux(packet)
+    
+    output.close()
+    
+    # Write BytesIO from RAM to file, for testing
+    with open("output.mp4", "wb") as f:
+        f.write(output_memory_file.getbuffer())