Initial video support without audio

CMakeLists.txt

@@ -58,6 +58,8 @@ add_executable(
         src/WallpaperEngine/Render/Objects/CImage.cpp
         src/WallpaperEngine/Render/Objects/CSound.h
         src/WallpaperEngine/Render/Objects/CSound.cpp
+        src/WallpaperEngine/Render/Objects/CVideo.h
+        src/WallpaperEngine/Render/Objects/CVideo.cpp
         src/WallpaperEngine/Render/Objects/CEffect.h
         src/WallpaperEngine/Render/Objects/CEffect.cpp
 
@@ -103,6 +105,8 @@ add_executable(
         src/WallpaperEngine/Core/Objects/CImage.h
         src/WallpaperEngine/Core/Objects/CSound.cpp
         src/WallpaperEngine/Core/Objects/CSound.h
+        src/WallpaperEngine/Core/Objects/CVideo.cpp
+        src/WallpaperEngine/Core/Objects/CVideo.h
         src/WallpaperEngine/Core/Objects/CEffect.cpp
         src/WallpaperEngine/Core/Objects/CEffect.h
         src/WallpaperEngine/Core/Objects/CParticle.cpp

CMakeModules/FindFFmpeg.cmake

@@ -1,4 +1,4 @@
-# - Try to find ffmpeg libraries (libavcodec, libavformat and libavutil)
+# - Try to find ffmpeg libraries (libavcodec, libavformat, libavutil and libswscale)
 # Once done this will define
 #
 # FFMPEG_FOUND - system has ffmpeg or libav
@@ -7,6 +7,7 @@
 # FFMPEG_LIBAVCODEC
 # FFMPEG_LIBAVFORMAT
 # FFMPEG_LIBAVUTIL
+# FFMPEG_LIBSWSCALE
 #
 # Copyright (c) 2008 Andreas Schneider <mail@cynapses.org>
 # Modified for other libraries by Lasse Kärkkäinen <tronic>
@@ -38,6 +39,7 @@ else()
     pkg_check_modules(_FFMPEG_AVCODEC libavcodec)
     pkg_check_modules(_FFMPEG_AVFORMAT libavformat)
     pkg_check_modules(_FFMPEG_AVUTIL libavutil)
+    pkg_check_modules(_FFMPEG_SWSCALE libswscale)
   endif()
 
   find_path(FFMPEG_AVCODEC_INCLUDE_DIR
@@ -73,6 +75,14 @@ else()
       /opt/local/lib
       /sw/lib)
 
+   find_library(FFMPEG_LIBSWSCALE
+    NAMES swscale
+    PATHS ${_FFMPEG_SWSCALE_LIBRARY_DIRS}
+      /usr/lib
+      /usr/local/lib
+      /opt/local/lib
+      /sw/lib)
+
   if(FFMPEG_LIBAVCODEC AND FFMPEG_LIBAVFORMAT)
     set(FFMPEG_FOUND TRUE)
   endif()
@@ -82,7 +92,8 @@ else()
     set(FFMPEG_LIBRARIES
       ${FFMPEG_LIBAVCODEC}
       ${FFMPEG_LIBAVFORMAT}
-      ${FFMPEG_LIBAVUTIL})
+      ${FFMPEG_LIBAVUTIL}
+      ${FFMPEG_LIBSWSCALE})
   endif()
 
   if(FFMPEG_FOUND)
@@ -93,7 +104,7 @@ else()
   else()
     if(FFMPEG_FIND_REQUIRED)
       message(FATAL_ERROR
-      "Could not find libavcodec or libavformat or libavutil")
+      "Could not find libavcodec or libavformat or libavutil or libswscale")
     endif()
   endif()
 endif()

src/WallpaperEngine/Core/CScene.cpp

@@ -1,10 +1,33 @@
 #include "CScene.h"
 #include "CProject.h"
 
+#include "WallpaperEngine/Core/Objects/CVideo.h"
 #include "WallpaperEngine/FileSystem/FileSystem.h"
 
 using namespace WallpaperEngine::Core;
 
+CScene::CScene () :
+    m_camera (new Core::Scenes::CCamera()),
+    m_ambientColor (irr::video::SColorf (0)),
+    m_bloom (false),
+    m_bloomStrength (0),
+    m_bloomThreshold (0),
+    m_cameraFade (false),
+    m_cameraParallax (false),
+    m_cameraParallaxAmount (0),
+    m_cameraParallaxDelay (0),
+    m_cameraParallaxMouseInfluence (0),
+    m_cameraPreview (false),
+    m_cameraShake (false),
+    m_cameraShakeAmplitude (0),
+    m_cameraShakeRoughness (0),
+    m_cameraShakeSpeed (0),
+    m_clearColor (irr::video::SColorf (0)),
+    m_orthogonalProjection (new Core::Scenes::CProjection()),
+    m_skylightColor (irr::video::SColorf (0))
+{
+}
+
 CScene::CScene (
         Scenes::CCamera* camera,
         irr::video::SColorf ambientColor,
@@ -121,7 +144,13 @@ CScene* CScene::loadScene (const irr::io::path& filename)
 
 CScene* CScene::loadVideo (const irr::io::path& filename)
 {
-    return nullptr;
+    CScene* scene = new CScene();
+    
+    scene->insertObject (
+            new Core::Objects::CVideo(filename)
+    );
+    
+    return scene;
 }
 
 const std::vector<CObject*>& CScene::getObjects () const

src/WallpaperEngine/Core/CScene.h

@@ -20,6 +20,8 @@ namespace WallpaperEngine::Core
     class CScene
     {
     public:
+        CScene ();
+
         static CScene* fromFile (const irr::io::path& filename, const char *type);
         static CScene* loadScene (const irr::io::path& filename);
         static CScene* loadVideo (const irr::io::path& filename);

src/WallpaperEngine/Core/Objects/CVideo.cpp

@@ -0,0 +1,148 @@
+#include "CVideo.h"
+
+using namespace WallpaperEngine::Core::Objects;
+
+CVideo::CVideo (
+        const irr::io::path& filename) :
+        CObject (true, 0, "video", Type, irr::core::vector3df(0), irr::core::vector3df(0), irr::core::vector3df(0))
+{
+    if (avformat_open_input (&m_formatCtx, filename.c_str(), NULL, NULL) < 0)
+        throw std::runtime_error ("Failed to open file");
+
+    if (avformat_find_stream_info (m_formatCtx, NULL) < 0)
+        throw std::runtime_error ("Failed to get stream info");
+
+    // Find first video stream
+    for (int i = 0; i < m_formatCtx->nb_streams; i++)
+    {
+        if (m_formatCtx->streams[i]->codecpar->codec_type == AVMEDIA_TYPE_VIDEO)
+        {
+            m_videoStream = i;
+            break;
+        }
+    }
+
+    // Find first audio stream
+    for (int i = 0; i < m_formatCtx->nb_streams; i++)
+    {
+        if (m_formatCtx->streams[i]->codecpar->codec_type == AVMEDIA_TYPE_AUDIO)
+        {
+            m_audioStream = i; 
+            break;
+        }
+    }
+
+    // Only video stream is required
+    if (m_videoStream == -1)
+        throw std::runtime_error ("Failed to find video stream");
+
+    AVCodec* codec = avcodec_find_decoder (m_formatCtx->streams[m_videoStream]->codecpar->codec_id);
+    if (codec == nullptr)
+        throw std::runtime_error ("Failed to find codec");
+
+    m_codecCtx = avcodec_alloc_context3 (codec);
+    if (avcodec_parameters_to_context (m_codecCtx, m_formatCtx->streams[m_videoStream]->codecpar))
+        throw std::runtime_error ("Failed to copy codec parameters");
+
+    if (avcodec_open2 (m_codecCtx, codec, NULL) < 0)
+        throw std::runtime_error ("Failed to open codec");
+}
+
+void CVideo::initFrames (int width, int height)
+{
+    m_videoFrame = av_frame_alloc ();
+    m_videoFrameRGB = av_frame_alloc ();
+    if (m_videoFrameRGB == nullptr)
+        throw std::runtime_error ("Failed to allocate video frame");
+
+    m_width = width;
+    m_height = height;
+
+    int numBytes = av_image_get_buffer_size (AV_PIX_FMT_RGB24, m_width, m_height, 1);
+    uint8_t* buffer = (uint8_t*)av_malloc (numBytes * sizeof (uint8_t));
+
+    av_image_fill_arrays (m_videoFrameRGB->data, m_videoFrameRGB->linesize, buffer, AV_PIX_FMT_RGB24, m_width, m_height, 1);
+
+    m_swsCtx = sws_getContext (m_codecCtx->width, m_codecCtx->height,
+                    m_codecCtx->pix_fmt,
+                    m_width, m_height,
+                    AV_PIX_FMT_RGB24,
+                    SWS_BILINEAR, NULL, NULL, NULL);
+
+    avcodec_flush_buffers (m_codecCtx);
+}
+
+void CVideo::getNextFrame ()
+{
+    bool eof = false;
+    AVPacket packet;
+    packet.data = nullptr;
+    
+    // Find video streams packet
+    do
+    {
+        if (packet.data != nullptr)
+            av_packet_unref (&packet);
+
+        int readError = av_read_frame (m_formatCtx, &packet);
+        if (readError == AVERROR_EOF)
+        {
+            eof = true;
+            break;
+        }
+        else if (readError < 0)
+        {
+            char err[AV_ERROR_MAX_STRING_SIZE];
+            throw std::runtime_error (av_make_error_string (err, AV_ERROR_MAX_STRING_SIZE, readError));
+        }
+        
+    } while (packet.stream_index != m_videoStream);
+
+    // Send video stream packet to codec
+    if (avcodec_send_packet (m_codecCtx, &packet) < 0)
+        return;
+    
+    // Receive frame from codec
+    if (avcodec_receive_frame (m_codecCtx, m_videoFrame) < 0)
+        return;
+
+    sws_scale (m_swsCtx, (uint8_t const* const*)m_videoFrame->data, m_videoFrame->linesize,
+            0, m_codecCtx->height, m_videoFrameRGB->data, m_videoFrameRGB->linesize);
+
+    av_packet_unref (&packet);
+
+    if (eof)
+        restartStream ();
+}
+
+void CVideo::writeFrameToImage (irr::video::IImage* image)
+{
+    uint8_t* frameData = m_videoFrameRGB->data[0];
+    if (frameData == nullptr)
+        return;
+
+    irr::u32 imgWidth = image->getDimension().Width;
+    irr::u32 imgHeight = image->getDimension().Height;
+    
+    unsigned char* data = (unsigned char*)image->lock ();
+    memcpy (data, frameData, imgWidth * imgHeight * 3);
+    image->unlock ();
+}
+
+void CVideo::restartStream ()
+{
+    av_seek_frame (m_formatCtx, m_videoStream, 0, AVSEEK_FLAG_FRAME);
+    avcodec_flush_buffers (m_codecCtx);
+}
+
+int CVideo::getWidth () const
+{
+    return this->m_width;
+}
+
+int CVideo::getHeight () const
+{
+    return this->m_height;
+}
+
+const std::string CVideo::Type = "video";

src/WallpaperEngine/Core/Objects/CVideo.h

@@ -0,0 +1,50 @@
+#pragma once
+
+#include <irrlicht/irrlicht.h>
+
+#include "WallpaperEngine/Core/Core.h"
+#include "WallpaperEngine/Core/CObject.h"
+
+extern "C"
+{
+    #include <libavcodec/avcodec.h>
+    #include <libavformat/avformat.h>
+    #include <libavutil/imgutils.h>
+    #include <libswscale/swscale.h>
+}
+
+namespace WallpaperEngine::Core::Objects
+{
+    class CVideo : public CObject
+    {
+        friend class CObject;
+
+    public:
+        CVideo (
+                const irr::io::path& filename
+        );
+
+        void initFrames (int width, int height);
+
+        void getNextFrame ();
+        void writeFrameToImage (irr::video::IImage* image);
+
+        int getWidth () const;
+        int getHeight () const;
+
+    protected:
+        void restartStream ();
+
+        static const std::string Type;
+
+    private:
+        AVFormatContext* m_formatCtx = nullptr;
+        AVCodecContext* m_codecCtx = nullptr;
+        AVFrame* m_videoFrame = nullptr;
+        AVFrame* m_videoFrameRGB = nullptr;
+        SwsContext* m_swsCtx = nullptr;
+
+        int m_videoStream = -1, m_audioStream = -1;
+        int m_width, m_height;
+    };
+};

src/WallpaperEngine/Core/Scenes/CCamera.cpp

@@ -2,6 +2,13 @@
 
 using namespace WallpaperEngine::Core::Scenes;
 
+CCamera::CCamera () :
+    m_center (0),
+    m_eye (0),
+    m_up (0)
+{
+}
+
 CCamera::CCamera (irr::core::vector3df center, irr::core::vector3df eye, irr::core::vector3df up) :
     m_center (center),
     m_eye (eye),

src/WallpaperEngine/Core/Scenes/CCamera.h

@@ -11,6 +11,7 @@ namespace WallpaperEngine::Core::Scenes
     class CCamera
     {
     public:
+        CCamera ();
         static CCamera* fromJSON (json data);
 
         const irr::core::vector3df& getCenter () const;

src/WallpaperEngine/Core/Scenes/CProjection.cpp

@@ -2,6 +2,12 @@
 
 using namespace WallpaperEngine::Core::Scenes;
 
+CProjection::CProjection () :
+    m_width (0),
+    m_height (0)
+{
+}
+
 CProjection::CProjection (irr::u32 width, irr::u32 height) :
     m_width (width),
     m_height (height)

src/WallpaperEngine/Core/Scenes/CProjection.h

@@ -11,6 +11,7 @@ namespace WallpaperEngine::Core::Scenes
     class CProjection
     {
     public:
+        CProjection ();
         static CProjection* fromJSON (json data);
 
         const irr::u32& getWidth () const;

src/WallpaperEngine/Render/CScene.cpp

@@ -1,9 +1,11 @@
 #include "WallpaperEngine/Irrlicht/CContext.h"
 
 #include "WallpaperEngine/Core/Objects/CImage.h"
+#include "WallpaperEngine/Core/Objects/CVideo.h"
 #include "WallpaperEngine/Core/Objects/CSound.h"
 
 #include "WallpaperEngine/Render/Objects/CImage.h"
+#include "WallpaperEngine/Render/Objects/CVideo.h"
 #include "WallpaperEngine/Render/Objects/CSound.h"
 
 #include "CScene.h"
@@ -40,6 +42,13 @@ CScene::CScene (const Core::CProject* project, Irrlicht::CContext* context) :
         {
             new Objects::CImage (this, (*cur)->as<Core::Objects::CImage>());
         }
+        else if ((*cur)->is<Core::Objects::CVideo>() == true)
+        {
+            Core::Objects::CVideo* video = (*cur)->as<Core::Objects::CVideo>();
+            video->initFrames (m_context->getDevice ()->getVideoDriver ()->getScreenSize().Width,
+                            m_context->getDevice ()->getVideoDriver ()->getScreenSize().Height);
+            new Objects::CVideo (this, video);
+        }
         else if ((*cur)->is<Core::Objects::CSound>() == true)
         {
             new Objects::CSound (this, (*cur)->as<Core::Objects::CSound>());

src/WallpaperEngine/Render/Objects/CVideo.cpp

@@ -0,0 +1,35 @@
+#include "CVideo.h"
+
+using namespace WallpaperEngine;
+
+using namespace WallpaperEngine::Render::Objects;
+
+CVideo::CVideo (CScene* scene, Core::Objects::CVideo* video) :
+    Render::CObject (scene, Type, video),
+    m_video (video)
+{
+    m_frameImage = this->getScene ()->getContext ()->getDevice ()->getVideoDriver ()->createImage (irr::video::ECOLOR_FORMAT::ECF_R8G8B8,
+                irr::core::dimension2du(m_video->getWidth(), m_video->getHeight()));
+
+    this->m_boundingBox = irr::core::aabbox3d<irr::f32> (0, 0, 0, 0, 0, 0);
+}
+
+void CVideo::render ()
+{
+    irr::video::IVideoDriver* driver = this->getScene ()->getContext ()->getDevice ()->getVideoDriver ();
+
+    m_video->getNextFrame ();
+    m_video->writeFrameToImage (m_frameImage);
+    
+    driver->removeTexture (m_frameTexture);
+    m_frameTexture = driver->addTexture ("frameTexture", m_frameImage);
+
+    driver->draw2DImage (m_frameTexture, irr::core::vector2di(0));
+}
+
+const irr::core::aabbox3d<irr::f32>& CVideo::getBoundingBox () const
+{
+    return this->m_boundingBox;
+}
+
+const std::string CVideo::Type = "video";

src/WallpaperEngine/Render/Objects/CVideo.h

@@ -0,0 +1,28 @@
+#pragma once
+
+#include "WallpaperEngine/Core/Objects/CVideo.h"
+
+#include "WallpaperEngine/Render/CScene.h"
+#include "WallpaperEngine/Render/CObject.h"
+
+namespace WallpaperEngine::Render::Objects
+{
+    class CVideo : public CObject
+    {
+    public:
+        CVideo (CScene* scene, Core::Objects::CVideo* video);
+
+        void render () override;
+        const irr::core::aabbox3d<irr::f32>& getBoundingBox () const override;
+
+    protected:
+        static const std::string Type;
+
+    private:
+        irr::video::IImage* m_frameImage;
+        irr::video::ITexture* m_frameTexture;
+
+        Core::Objects::CVideo* m_video;
+        irr::core::aabbox3d<irr::f32> m_boundingBox;
+    };
+};