diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml index 53110c2..8eaa9b9 100644 --- a/.github/workflows/cmake.yml +++ b/.github/workflows/cmake.yml @@ -21,7 +21,7 @@ jobs: - uses: actions/checkout@v3 - name: Install dependencies - run: sudo apt-get update && sudo apt-get -y install libxrandr-dev libfreeimage-dev libxinerama-dev libxcursor-dev libxi-dev libgl-dev libglew-dev freeglut3-dev libsdl1.2-dev libsdl-mixer1.2-dev liblz4-dev libavcodec-dev libavformat-dev libavutil-dev libswscale-dev libxxf86vm-dev libglm-dev libglfw3-dev libmpv-dev mpv libmpv1 + run: sudo apt-get update && sudo apt-get -y install libxrandr-dev libfreeimage-dev libxinerama-dev libxcursor-dev libxi-dev libgl-dev libglew-dev freeglut3-dev libsdl2-dev liblz4-dev libavcodec-dev libavformat-dev libavutil-dev libswscale-dev libxxf86vm-dev libglm-dev libglfw3-dev libmpv-dev mpv libmpv1 - name: Configure CMake # Configure CMake in a 'build' subdirectory. `CMAKE_BUILD_TYPE` is only required if you are using a single-configuration generator such as make. diff --git a/CMakeLists.txt b/CMakeLists.txt index 2c3a564..d70a08d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -7,7 +7,7 @@ set(OpenGL_GL_PREFERENCE "LEGACY") # if you're developing you might find this debug option useful for shader output, although RenderDoc is encouraged add_compile_definitions(DEBUG=1) -add_compile_definitions(ERRORONLY=0) +add_compile_definitions(ERRORONLY=1) find_package(X11 REQUIRED) find_package(Xrandr REQUIRED) @@ -15,7 +15,7 @@ find_package(OpenGL REQUIRED) find_package(GLEW REQUIRED) find_package(GLUT REQUIRED) find_package(ZLIB REQUIRED) -find_package(SDL REQUIRED) +find_package(SDL2 REQUIRED) find_package(MPV REQUIRED) find_package(LZ4 REQUIRED) find_package(FFMPEG REQUIRED) @@ -27,8 +27,7 @@ include_directories( ${XRANDR_INCLUDE_DIR} ${GLEW_INCLUDE_DIR} ${LZ4_INCLUDE_DIR} - ${SDL_INCLUDE_DIRS} - ${SDL_MIXER_INCLUDE_DIRS} + ${SDL2_INCLUDE_DIRS} ${FFMPEG_INCLUDE_DIR} ${FREEIMAGE_INCLUDE_DIR} src @@ -65,6 +64,12 @@ add_executable( src/WallpaperEngine/Core/Core.h src/WallpaperEngine/Core/Core.cpp + src/WallpaperEngine/Audio/Drivers/CAudioDriver.cpp + src/WallpaperEngine/Audio/Drivers/CAudioDriver.h + src/WallpaperEngine/Audio/Drivers/CSDLAudioDriver.cpp + src/WallpaperEngine/Audio/Drivers/CSDLAudioDriver.h + src/WallpaperEngine/Audio/CAudioContext.cpp + src/WallpaperEngine/Audio/CAudioContext.h src/WallpaperEngine/Audio/CAudioStream.cpp src/WallpaperEngine/Audio/CAudioStream.h @@ -227,8 +232,7 @@ target_link_libraries(linux-wallpaperengine ${GLUT_LIBRARIES} ${ZLIB_LIBRARIES} ${LZ4_LIBRARY} - ${SDL_LIBRARY} - ${SDL_MIXER_LIBRARIES} + ${SDL2_LIBRARIES} ${FFMPEG_LIBRARIES} ${FREEIMAGE_LIBRARIES} ${MPV_LIBRARY} diff --git a/README.md b/README.md index b8c2b23..be6ac3f 100644 --- a/README.md +++ b/README.md @@ -37,16 +37,8 @@ Wallpaper Engine is a software designed by [Kristjan Skutta](https://store.steam ## Commands ``` -sudo apt update -``` -``` sudo apt-get update -``` -``` -sudo apt -y install cmake lz4 zlib1g ffmpeg libxxf86vm-dev libglm-dev freeglut3-dev libxrandr-dev libavcodec-dev libavformat-dev libavfilter-dev libsdl1.2-dev libsdl-mixer1.2-dev -``` -``` -sudo apt-get -y install libsdl2-2.0 libsdl-image1.2-dev libsdl1.2-dev libglfw3 libglfw3-dev libmpv-dev mpv libmpv1 +sudo apt-get install build-essential cmake libxrandr-dev libfreeimage-dev libxinerama-dev libxcursor-dev libxi-dev libgl-dev libglew-dev freeglut3-dev libsdl2-dev liblz4-dev libavcodec-dev libavformat-dev libavutil-dev libswscale-dev libxxf86vm-dev libglm-dev libglfw3-dev libmpv-dev mpv libmpv1 ``` # 5. How to use diff --git a/main.cpp b/main.cpp index d309ac2..7c89adb 100644 --- a/main.cpp +++ b/main.cpp @@ -2,7 +2,6 @@ #include #include #include -#include #include #include #include @@ -23,8 +22,10 @@ #include "WallpaperEngine/Assets/CCombinedContainer.h" #include "WallpaperEngine/Assets/CPackageLoadException.h" -#include "WallpaperEngine/Render/Drivers/COpenGLDriver.h" #include "Steam/FileSystem/FileSystem.h" +#include "WallpaperEngine/Audio/CAudioContext.h" +#include "WallpaperEngine/Audio/Drivers/CSDLAudioDriver.h" +#include "WallpaperEngine/Render/Drivers/COpenGLDriver.h" #include "common.h" #define WORKSHOP_APP_ID 431960 @@ -34,7 +35,7 @@ float g_Time; float g_TimeLast; bool g_KeepRunning = true; bool g_AudioEnabled = true; -int g_AudioVolume = 15; +int g_AudioVolume = 128; void print_help (const char* route) { @@ -445,12 +446,10 @@ int main (int argc, char* argv[]) std::signal(SIGINT, signalhandler); std::signal(SIGTERM, signalhandler); - if (g_AudioEnabled == true && SDL_Init (SDL_INIT_AUDIO) < 0) - { - sLog.error ("Cannot initialize SDL audio system, SDL_GetError: ", SDL_GetError()); - sLog.error ("Continuing without audio support"); - } - + // initialize sdl audio driver + WallpaperEngine::Audio::Drivers::CSDLAudioDriver audioDriver; + // initialize audio context + WallpaperEngine::Audio::CAudioContext audioContext (&audioDriver); // initialize OpenGL driver WallpaperEngine::Render::Drivers::COpenGLDriver videoDriver ("Wallpaper Engine"); // initialize render context @@ -459,7 +458,7 @@ int main (int argc, char* argv[]) context.setMouse (new CMouseInput (videoDriver.getWindow ())); // ensure the context knows what wallpaper to render context.setWallpaper ( - WallpaperEngine::Render::CWallpaper::fromWallpaper (project->getWallpaper (), &context) + WallpaperEngine::Render::CWallpaper::fromWallpaper (project->getWallpaper (), &context, &audioContext) ); float startTime, endTime, minimumTime = 1.0f / maximumFPS; @@ -494,8 +493,6 @@ int main (int argc, char* argv[]) sLog.out ("Stop requested"); - // terminate SDL - SDL_QuitSubSystem(SDL_INIT_AUDIO); SDL_Quit (); return 0; diff --git a/src/WallpaperEngine/Audio/CAudioContext.cpp b/src/WallpaperEngine/Audio/CAudioContext.cpp new file mode 100644 index 0000000..d610c00 --- /dev/null +++ b/src/WallpaperEngine/Audio/CAudioContext.cpp @@ -0,0 +1,31 @@ +#include "CAudioContext.h" +#include "WallpaperEngine/Audio/Drivers/CAudioDriver.h" + +using namespace WallpaperEngine::Audio; +using namespace WallpaperEngine::Audio::Drivers; + +CAudioContext::CAudioContext (CAudioDriver* driver) : + m_driver (driver) +{ + +} + +void CAudioContext::addStream (CAudioStream* stream) +{ + this->m_driver->addStream (stream); +} + +AVSampleFormat CAudioContext::getFormat () const +{ + return this->m_driver->getFormat (); +} + +int CAudioContext::getSampleRate () const +{ + return this->m_driver->getSampleRate (); +} + +int CAudioContext::getChannels () const +{ + return this->m_driver->getChannels (); +} \ No newline at end of file diff --git a/src/WallpaperEngine/Audio/CAudioContext.h b/src/WallpaperEngine/Audio/CAudioContext.h new file mode 100644 index 0000000..1e59811 --- /dev/null +++ b/src/WallpaperEngine/Audio/CAudioContext.h @@ -0,0 +1,28 @@ +#pragma once + +#include +#include + +namespace WallpaperEngine::Audio::Drivers +{ + class CAudioDriver; +} + +namespace WallpaperEngine::Audio +{ + class CAudioStream; + + class CAudioContext + { + public: + CAudioContext (Drivers::CAudioDriver* driver); + + void addStream (CAudioStream* stream); + + AVSampleFormat getFormat () const; + int getSampleRate () const; + int getChannels () const; + private: + Drivers::CAudioDriver* m_driver; + }; +} \ No newline at end of file diff --git a/src/WallpaperEngine/Audio/CAudioStream.cpp b/src/WallpaperEngine/Audio/CAudioStream.cpp index fb154fa..710b150 100644 --- a/src/WallpaperEngine/Audio/CAudioStream.cpp +++ b/src/WallpaperEngine/Audio/CAudioStream.cpp @@ -4,74 +4,52 @@ #include #include +// maximum size of the queue to prevent reading too much data +#define MAX_QUEUE_SIZE (5 * 1024 * 1024) +#define MIN_FRAMES 25 + extern int g_AudioVolume; extern bool g_KeepRunning; using namespace WallpaperEngine::Audio; -// callback for sdl to play our audio -void audio_callback (void* userdata, uint8_t* stream, int length) -{ - auto audioStream = static_cast (userdata); - int len1, audio_size; - - static uint8_t audio_buf[(MAX_AUDIO_FRAME_SIZE * 3) / 2]; - static unsigned int audio_buf_size = 0; - static unsigned int audio_buf_index = 0; - - while (length > 0 && g_KeepRunning) - { - if (audio_buf_index >= audio_buf_size) - { - /* We have already sent all our data; get more */ - audio_size = audioStream->decodeFrame (audio_buf, sizeof (audio_buf)); - - if (audio_size < 0) - { - /* If error, output silence */ - audio_buf_size = 1024; // arbitrary? - memset(audio_buf, 0, audio_buf_size); - } - else - { - audio_buf_size = audio_size; - } - - audio_buf_index = 0; - } - - len1 = audio_buf_size - audio_buf_index; - - if (len1 > length) - len1 = length; - - // mix the audio so the volume is right - SDL_MixAudio (stream, (uint8_t*) audio_buf + audio_buf_index, len1, g_AudioVolume); - - length -= len1; - stream += len1; - audio_buf_index += len1; - - if (audioStream->isInitialized () == false) - break; - } -} - int audio_read_thread (void* arg) { + SDL_mutex* waitMutex = SDL_CreateMutex (); CAudioStream* stream = static_cast (arg); AVPacket* packet = av_packet_alloc (); int ret = 0; + if (waitMutex == nullptr) + sLog.exception ("Cannot create mutex for audio playback waiting"); + while (ret >= 0 && g_KeepRunning == true) { + // give the cpu some time to play the queued frames if there's enough info there + if ( + stream->getQueueSize () >= MAX_QUEUE_SIZE || + (stream->getQueuePacketCount () > MIN_FRAMES && + (av_q2d (stream->getTimeBase ()) * stream->getQueueDuration () > 1.0)) + ) + { + SDL_LockMutex (waitMutex); + SDL_CondWaitTimeout (stream->getWaitCondition (), waitMutex, 10); + SDL_UnlockMutex (waitMutex); + continue; + } + ret = av_read_frame (stream->getFormatContext (), packet); if (ret == AVERROR_EOF) { // seek to the beginning of the file again - av_seek_frame (stream->getFormatContext (), stream->getAudioStream (), 0, AVSEEK_FLAG_FRAME); + avformat_seek_file (stream->getFormatContext (), stream->getAudioStream (), 0, 0, 0, ~AVSEEK_FLAG_FRAME); avcodec_flush_buffers (stream->getContext ()); + + // ensure the thread is not killed if audio has to be looped + if (stream->isRepeat() == true) + ret = 0; + continue; } @@ -79,7 +57,7 @@ int audio_read_thread (void* arg) if (packet->stream_index == stream->getAudioStream ()) stream->queuePacket (packet); else - av_packet_unref (packet); + av_packet_free (&packet); if (stream->isInitialized () == false) break; @@ -87,6 +65,7 @@ int audio_read_thread (void* arg) // stop the audio too just in case stream->stop (); + SDL_DestroyMutex (waitMutex); return 0; } @@ -126,21 +105,15 @@ int64_t audio_seek_data_callback (void* streamarg, int64_t offset, int whence) return offset; } -CAudioStream::CAudioStream (const std::string& filename) +CAudioStream::CAudioStream (CAudioContext* context, const std::string& filename) : + m_audioContext (context) { - // do not do anything if sdl audio was not initialized - if (SDL_WasInit (SDL_INIT_AUDIO) != SDL_INIT_AUDIO) - return; - this->loadCustomContent (filename.c_str ()); } -CAudioStream::CAudioStream (const void* buffer, int length) +CAudioStream::CAudioStream (CAudioContext* context, const void* buffer, int length) : + m_audioContext (context) { - // do not do anything if sdl audio was not initialized - if (SDL_WasInit (SDL_INIT_AUDIO) != SDL_INIT_AUDIO) - return; - // setup a custom context first this->m_formatContext = avformat_alloc_context (); @@ -169,14 +142,11 @@ CAudioStream::CAudioStream (const void* buffer, int length) this->loadCustomContent (); } -CAudioStream::CAudioStream(AVCodecContext* context) - : m_context (context), - m_queue (new PacketQueue) +CAudioStream::CAudioStream(CAudioContext* audioContext, AVCodecContext* context) : + m_context (context), + m_queue (new PacketQueue), + m_audioContext (audioContext) { - // do not do anything if sdl audio was not initialized - if (SDL_WasInit (SDL_INIT_AUDIO) != SDL_INIT_AUDIO) - return; - this->initialize (); } @@ -222,38 +192,18 @@ void CAudioStream::loadCustomContent (const char* filename) this->initialize (); // initialize an SDL thread to read the file - SDL_CreateThread (audio_read_thread, this); + SDL_CreateThread (audio_read_thread, filename, this); } void CAudioStream::initialize () { // allocate the FIFO buffer - this->m_queue->packetList = av_fifo_alloc (sizeof (MyAVPacketList)); + this->m_queue->packetList = av_fifo_alloc2 (1, sizeof (MyAVPacketList), AV_FIFO_FLAG_AUTO_GROW); // setup the queue information this->m_queue->mutex = SDL_CreateMutex (); this->m_queue->cond = SDL_CreateCond (); - - // take control of the audio device - - SDL_AudioSpec requestedSpec, finalSpec; - - // Set audio settings from codec info - requestedSpec.freq = this->m_context->sample_rate; - requestedSpec.format = AUDIO_S16SYS; - requestedSpec.channels = this->m_context->channels; - requestedSpec.silence = 0; - requestedSpec.samples = SDL_AUDIO_BUFFER_SIZE; - requestedSpec.callback = audio_callback; - requestedSpec.userdata = this; - - if (SDL_OpenAudio (&requestedSpec, &finalSpec) < 0) - { - sLog.error ("SDL_OpenAudio: ", SDL_GetError ()); - return; - } - - SDL_PauseAudio (0); + this->m_queue->wait = SDL_CreateCond (); this->m_initialized = true; } @@ -281,21 +231,11 @@ void CAudioStream::queuePacket(AVPacket *pkt) bool CAudioStream::doQueue (AVPacket* pkt) { - MyAVPacketList entry; + MyAVPacketList entry { pkt }; - // ensure the FIFO has enough space to hold the new entry - if (av_fifo_space (this->m_queue->packetList) < sizeof (entry)) - { - if (av_fifo_grow (this->m_queue->packetList, sizeof (entry)) < 0) - { - return false; - } - } - - entry.packet = pkt; - - // write to the FIFO - av_fifo_generic_write (this->m_queue->packetList, &entry, sizeof (entry), nullptr); + // write the entry if possible + if (av_fifo_write (this->m_queue->packetList, &entry, 1) < 0) + return false; this->m_queue->nb_packets ++; this->m_queue->size += entry.packet->size + sizeof (entry); @@ -315,12 +255,11 @@ void CAudioStream::dequeuePacket (AVPacket* output) while (g_KeepRunning) { // enough data available, read it - if (av_fifo_size (this->m_queue->packetList) >= sizeof (entry)) + if (av_fifo_read (this->m_queue->packetList, &entry, 1) >= 0) { - av_fifo_generic_read (this->m_queue->packetList, &entry, sizeof (entry), nullptr); - this->m_queue->nb_packets --; - this->m_queue->size -= entry.packet->duration; + this->m_queue->size -= entry.packet->size + sizeof (entry); + this->m_queue->duration -= entry.packet->duration; // move the reference and free the old one av_packet_move_ref (output, entry.packet); @@ -329,7 +268,7 @@ void CAudioStream::dequeuePacket (AVPacket* output) } // make the thread wait if nothing was available - SDL_CondWaitTimeout (this->m_queue->cond, this->m_queue->mutex, 1000); + SDL_CondWait (this->m_queue->cond, this->m_queue->mutex); } SDL_UnlockMutex (this->m_queue->mutex); @@ -385,14 +324,46 @@ void CAudioStream::setPosition (int current) this->m_position = current; } +SDL_cond* CAudioStream::getWaitCondition () +{ + return this->m_queue->wait; +} + +int CAudioStream::getQueueSize () +{ + return this->m_queue->size; +} + +int CAudioStream::getQueuePacketCount () +{ + return this->m_queue->nb_packets; +} + +AVRational CAudioStream::getTimeBase () +{ + return this->m_formatContext->streams [this->m_audioStream]->time_base; +} + +int64_t CAudioStream::getQueueDuration () +{ + return this->m_queue->duration; +} + +bool CAudioStream::isQueueEmpty () +{ + return this->m_queue->nb_packets == 0; +} + +SDL_mutex* CAudioStream::getMutex () +{ + return this->m_queue->mutex; +} + void CAudioStream::stop () { if (this->isInitialized () == false) return; - // pause audio - SDL_PauseAudio (1); - // stop the threads running this->m_initialized = false; } @@ -421,7 +392,7 @@ int CAudioStream::resampleAudio ( if (!swr_ctx) { - printf("swr_alloc error.\n"); + sLog.error("swr_alloc error.\n"); return -1; } @@ -434,7 +405,7 @@ int CAudioStream::resampleAudio ( // check input audio channels correctly retrieved if (in_channel_layout <= 0) { - printf("in_channel_layout error.\n"); + sLog.error("in_channel_layout error.\n"); return -1; } @@ -456,7 +427,7 @@ int CAudioStream::resampleAudio ( in_nb_samples = decoded_audio_frame->nb_samples; if (in_nb_samples <= 0) { - printf("in_nb_samples error.\n"); + sLog.error("in_nb_samples error."); return -1; } @@ -513,7 +484,7 @@ int CAudioStream::resampleAudio ( ret = swr_init(swr_ctx);; if (ret < 0) { - printf("Failed to initialize the resampling context.\n"); + sLog.error("Failed to initialize the resampling context."); return -1; } @@ -527,7 +498,7 @@ int CAudioStream::resampleAudio ( // check rescaling was successful if (max_out_nb_samples <= 0) { - printf("av_rescale_rnd error.\n"); + sLog.error("av_rescale_rnd error."); return -1; } @@ -545,7 +516,7 @@ int CAudioStream::resampleAudio ( if (ret < 0) { - printf("av_samples_alloc_array_and_samples() error: Could not allocate destination samples.\n"); + sLog.error("av_samples_alloc_array_and_samples() error: Could not allocate destination samples."); return -1; } @@ -560,7 +531,7 @@ int CAudioStream::resampleAudio ( // check output samples number was correctly retrieved if (out_nb_samples <= 0) { - printf("av_rescale_rnd error\n"); + sLog.error("av_rescale_rnd error"); return -1; } @@ -582,50 +553,42 @@ int CAudioStream::resampleAudio ( // check samples buffer correctly allocated if (ret < 0) { - printf("av_samples_alloc failed.\n"); + sLog.error("av_samples_alloc failed."); return -1; } max_out_nb_samples = out_nb_samples; } - if (swr_ctx) + // do the actual audio data resampling + ret = swr_convert( + swr_ctx, + resampled_data, + out_nb_samples, + (const uint8_t **) decoded_audio_frame->data, + decoded_audio_frame->nb_samples + ); + + // check audio conversion was successful + if (ret < 0) { - // do the actual audio data resampling - ret = swr_convert( - swr_ctx, - resampled_data, - out_nb_samples, - (const uint8_t **) decoded_audio_frame->data, - decoded_audio_frame->nb_samples - ); - - // check audio conversion was successful - if (ret < 0) - { - printf("swr_convert_error.\n"); - return -1; - } - - // Get the required buffer size for the given audio parameters - resampled_data_size = av_samples_get_buffer_size( - &out_linesize, - out_nb_channels, - ret, - out_sample_fmt, - 1 - ); - - // check audio buffer size - if (resampled_data_size < 0) - { - printf("av_samples_get_buffer_size error.\n"); - return -1; - } + sLog.error("swr_convert_error."); + return -1; } - else + + // Get the required buffer size for the given audio parameters + resampled_data_size = av_samples_get_buffer_size( + &out_linesize, + out_nb_channels, + ret, + out_sample_fmt, + 1 + ); + + // check audio buffer size + if (resampled_data_size < 0) { - printf("swr_ctx null error.\n"); + sLog.error ("av_samples_get_buffer_size error."); return -1; } @@ -666,39 +629,28 @@ int CAudioStream::decodeFrame (uint8_t* audioBuffer, int bufferSize) avFrame = av_frame_alloc(); if (!avFrame) { - printf("Could not allocate AVFrame.\n"); + sLog.error("Could not allocate AVFrame.\n"); return -1; } - for (; g_KeepRunning;) { + // block until there's any data in the buffers + while (g_KeepRunning) { while (audio_pkt_size > 0) { int got_frame = 0; int ret = avcodec_receive_frame(this->getContext (), avFrame); if (ret == 0) - { got_frame = 1; - } if (ret == AVERROR(EAGAIN)) - { ret = 0; - } if (ret == 0) - { ret = avcodec_send_packet(this->getContext (), pkt); - } if (ret == AVERROR(EAGAIN)) - { ret = 0; - } else if (ret < 0) - { return -1; - } else - { len1 = pkt->size; - } if (len1 < 0) { @@ -715,18 +667,18 @@ int CAudioStream::decodeFrame (uint8_t* audioBuffer, int bufferSize) // audio resampling data_size = this->resampleAudio ( avFrame, - AV_SAMPLE_FMT_S16, - this->getContext ()->channels, - this->getContext ()->sample_rate, + this->m_audioContext->getFormat (), + this->m_audioContext->getChannels (), + this->m_audioContext->getSampleRate (), audioBuffer ); assert(data_size <= bufferSize); } if (data_size <= 0) { - /* No data yet, get more frames */ + // no data found, keep waiting continue; } - /* We have data, return it and come back for more later */ + // some data was found return data_size; } if (pkt->data) diff --git a/src/WallpaperEngine/Audio/CAudioStream.h b/src/WallpaperEngine/Audio/CAudioStream.h index a6db0f7..3c4652f 100644 --- a/src/WallpaperEngine/Audio/CAudioStream.h +++ b/src/WallpaperEngine/Audio/CAudioStream.h @@ -15,74 +15,79 @@ extern "C" #include #include -#define SDL_AUDIO_BUFFER_SIZE 1024 -#define MAX_AUDIO_FRAME_SIZE 192000 +#include "WallpaperEngine/Audio/CAudioContext.h" -namespace WallpaperEngine +namespace WallpaperEngine::Audio { - namespace Audio + class CAudioStream { - class CAudioStream + public: + CAudioStream (CAudioContext* context, const std::string& filename); + CAudioStream (CAudioContext* context, const void* buffer, int length); + CAudioStream (CAudioContext* audioContext, AVCodecContext* context); + + void queuePacket (AVPacket* pkt); + + /** + * Gets the next packet in the queue + * + * WARNING: BLOCKS UNTIL SOME DATA IS READ FROM IT + * + * @return + */ + void dequeuePacket (AVPacket* output); + + AVCodecContext* getContext (); + AVFormatContext* getFormatContext (); + int getAudioStream (); + bool isInitialized (); + void setRepeat (bool newRepeat = true); + bool isRepeat (); + void stop (); + const void* getBuffer (); + int getLength (); + int getPosition (); + void setPosition (int current); + SDL_cond* getWaitCondition (); + int getQueueSize (); + int getQueuePacketCount (); + int64_t getQueueDuration (); + AVRational getTimeBase (); + bool isQueueEmpty (); + SDL_mutex* getMutex (); + + int decodeFrame (uint8_t* audioBuffer, int bufferSize); + + private: + void loadCustomContent (const char* filename = nullptr); + int resampleAudio (AVFrame * decoded_audio_frame, enum AVSampleFormat out_sample_fmt, int out_channels, int out_sample_rate, uint8_t* out_buf); + bool doQueue (AVPacket* pkt); + void initialize (); + + CAudioContext* m_audioContext; + bool m_initialized; + bool m_repeat; + AVCodecContext* m_context = nullptr; + AVFormatContext* m_formatContext = nullptr; + int m_audioStream = -1; + const void* m_buffer; + int m_length; + int m_position = 0; + + struct MyAVPacketList { - public: - CAudioStream (const std::string& filename); - CAudioStream (const void* buffer, int length); - CAudioStream (AVCodecContext* context); - - void queuePacket (AVPacket* pkt); - - /** - * Gets the next packet in the queue - * - * WARNING: BLOCKS UNTIL SOME DATA IS READ FROM IT - * - * @return - */ - void dequeuePacket (AVPacket* output); - - AVCodecContext* getContext (); - AVFormatContext* getFormatContext (); - int getAudioStream (); - bool isInitialized (); - void setRepeat (bool newRepeat = true); - bool isRepeat (); - void stop (); - const void* getBuffer (); - int getLength (); - int getPosition (); - void setPosition (int current); - - int decodeFrame (uint8_t* audioBuffer, int bufferSize); - - private: - void loadCustomContent (const char* filename = nullptr); - int resampleAudio (AVFrame * decoded_audio_frame, enum AVSampleFormat out_sample_fmt, int out_channels, int out_sample_rate, uint8_t* out_buf); - bool doQueue (AVPacket* pkt); - void initialize (); - - bool m_initialized; - bool m_repeat; - AVCodecContext* m_context = nullptr; - AVFormatContext* m_formatContext = nullptr; - int m_audioStream = -1; - const void* m_buffer; - int m_length; - int m_position = 0; - - struct MyAVPacketList - { - AVPacket *packet; - }; - - struct PacketQueue - { - AVFifoBuffer* packetList; - int nb_packets; - int size; - int64_t duration; - SDL_mutex *mutex; - SDL_cond *cond; - } *m_queue; + AVPacket *packet; }; - } + + struct PacketQueue + { + AVFifo* packetList; + int nb_packets; + int size; + int64_t duration; + SDL_mutex *mutex; + SDL_cond *wait; + SDL_cond *cond; + } *m_queue; + }; }; diff --git a/src/WallpaperEngine/Audio/Drivers/CAudioDriver.cpp b/src/WallpaperEngine/Audio/Drivers/CAudioDriver.cpp new file mode 100644 index 0000000..fc14ce9 --- /dev/null +++ b/src/WallpaperEngine/Audio/Drivers/CAudioDriver.cpp @@ -0,0 +1,4 @@ +#include "CAudioDriver.h" + +using namespace WallpaperEngine::Audio; +using namespace WallpaperEngine::Audio::Drivers; diff --git a/src/WallpaperEngine/Audio/Drivers/CAudioDriver.h b/src/WallpaperEngine/Audio/Drivers/CAudioDriver.h new file mode 100644 index 0000000..b24172d --- /dev/null +++ b/src/WallpaperEngine/Audio/Drivers/CAudioDriver.h @@ -0,0 +1,24 @@ +#pragma once + +#include + +#include "WallpaperEngine/Audio/CAudioStream.h" + +namespace WallpaperEngine::Audio +{ + class CAudioStream; +} + +namespace WallpaperEngine::Audio::Drivers +{ + class CAudioDriver + { + public: + virtual void addStream (CAudioStream* stream) = 0; + + virtual AVSampleFormat getFormat () const = 0; + virtual int getSampleRate () const = 0; + virtual int getChannels () const = 0; + private: + }; +} \ No newline at end of file diff --git a/src/WallpaperEngine/Audio/Drivers/CSDLAudioDriver.cpp b/src/WallpaperEngine/Audio/Drivers/CSDLAudioDriver.cpp new file mode 100644 index 0000000..b66df03 --- /dev/null +++ b/src/WallpaperEngine/Audio/Drivers/CSDLAudioDriver.cpp @@ -0,0 +1,154 @@ +#include "common.h" +#include "CSDLAudioDriver.h" + +#define SDL_AUDIO_BUFFER_SIZE 4096 +#define MAX_AUDIO_FRAME_SIZE 192000 + +extern int g_AudioVolume; +extern bool g_KeepRunning; + +using namespace WallpaperEngine::Audio; +using namespace WallpaperEngine::Audio::Drivers; + +void audio_callback (void* userdata, uint8_t* streamData, int length) +{ + CSDLAudioDriver* driver = reinterpret_cast (userdata); + + memset (streamData, 0, length); + + for (const auto& buffer : driver->getStreams ()) + { + uint8_t* streamDataPointer = streamData; + int streamLength = length; + int len1, audio_size; + + // sound is not initialized or stopped and is not in loop mode + // ignore mixing it in + if (buffer->stream->isInitialized () == false) + continue; + + // check if thread is empty and signal the read thread + if (buffer->stream->isQueueEmpty () == true) + { + SDL_CondSignal (buffer->stream->getWaitCondition ()); + continue; + } + + while (streamLength > 0 && g_KeepRunning) + { + if (buffer->audio_buf_index >= buffer->audio_buf_size) + { + // get more data to fill the buffer + audio_size = buffer->stream->decodeFrame (buffer->audio_buf, sizeof (buffer->audio_buf)); + + if (audio_size < 0) + { + // fallback for errors, silence + buffer->audio_buf_size = 1024; + memset(buffer->audio_buf, 0, buffer->audio_buf_size); + } + else + { + buffer->audio_buf_size = audio_size; + } + + buffer->audio_buf_index = 0; + } + + len1 = buffer->audio_buf_size - buffer->audio_buf_index; + + if (len1 > streamLength) + len1 = streamLength; + + // mix the audio + SDL_MixAudioFormat ( + streamDataPointer, &buffer->audio_buf [buffer->audio_buf_index], + driver->getSpec ().format, len1, g_AudioVolume + ); + + streamLength -= len1; + streamDataPointer += len1; + buffer->audio_buf_index += len1; + } + } +} + +CSDLAudioDriver::CSDLAudioDriver () : + m_initialized (false) +{ + if (SDL_InitSubSystem (SDL_INIT_AUDIO) < 0) + { + sLog.error ("Cannot initialize SDL audio system, SDL_GetError: ", SDL_GetError ()); + sLog.error ("Continuing without audio support"); + + return; + } + + SDL_AudioSpec requestedSpec; + + memset (&requestedSpec, 0, sizeof (requestedSpec)); + + requestedSpec.freq = 48000; + requestedSpec.format = AUDIO_F32; + requestedSpec.channels = 2; + requestedSpec.samples = SDL_AUDIO_BUFFER_SIZE; + requestedSpec.callback = audio_callback; + requestedSpec.userdata = this; + + this->m_deviceID = SDL_OpenAudioDevice (nullptr, false, &requestedSpec, &this->m_audioSpec, SDL_AUDIO_ALLOW_ANY_CHANGE); + + if (this->m_deviceID == 0) + { + sLog.error ("SDL_OpenAudioDevice: ", SDL_GetError ()); + return; + } + + SDL_PauseAudioDevice (this->m_deviceID, 0); + + this->m_initialized = true; +} + +CSDLAudioDriver::~CSDLAudioDriver () +{ + if (this->m_initialized == false) + return; + + SDL_CloseAudioDevice (this->m_deviceID); + SDL_QuitSubSystem (SDL_INIT_AUDIO); +} + +void CSDLAudioDriver::addStream (CAudioStream* stream) +{ + this->m_streams.push_back (new CSDLAudioBuffer { stream }); +} +const std::vector & CSDLAudioDriver::getStreams () +{ + return this->m_streams; +} + +AVSampleFormat CSDLAudioDriver::getFormat () const +{ + switch (this->m_audioSpec.format) + { + case AUDIO_U8: case AUDIO_S8: return AV_SAMPLE_FMT_U8; + case AUDIO_U16MSB: case AUDIO_U16LSB: case AUDIO_S16LSB: case AUDIO_S16MSB: return AV_SAMPLE_FMT_S16; + case AUDIO_S32LSB: case AUDIO_S32MSB: return AV_SAMPLE_FMT_S32; + case AUDIO_F32LSB: case AUDIO_F32MSB: return AV_SAMPLE_FMT_FLT; + } + + sLog.exception ("Cannot convert from SDL format to ffmpeg format, aborting..."); +} + +int CSDLAudioDriver::getSampleRate () const +{ + return this->m_audioSpec.freq; +} + +int CSDLAudioDriver::getChannels () const +{ + return this->m_audioSpec.channels; +} +const SDL_AudioSpec& CSDLAudioDriver::getSpec () const +{ + return this->m_audioSpec; +} \ No newline at end of file diff --git a/src/WallpaperEngine/Audio/Drivers/CSDLAudioDriver.h b/src/WallpaperEngine/Audio/Drivers/CSDLAudioDriver.h new file mode 100644 index 0000000..e2f298a --- /dev/null +++ b/src/WallpaperEngine/Audio/Drivers/CSDLAudioDriver.h @@ -0,0 +1,42 @@ +#pragma once + +#include +#include + +#include "WallpaperEngine/Audio/CAudioStream.h" +#include "WallpaperEngine/Audio/Drivers/CAudioDriver.h" + +#include + +#define MAX_AUDIO_FRAME_SIZE 192000 + +namespace WallpaperEngine::Audio::Drivers +{ + struct CSDLAudioBuffer + { + CAudioStream* stream; + uint8_t audio_buf[(MAX_AUDIO_FRAME_SIZE * 3) / 2]; + unsigned int audio_buf_size = 0; + unsigned int audio_buf_index = 0; + }; + + class CSDLAudioDriver : public CAudioDriver + { + public: + CSDLAudioDriver (); + ~CSDLAudioDriver (); + + void addStream (CAudioStream* stream) override; + const std::vector & getStreams (); + + AVSampleFormat getFormat () const override; + int getSampleRate () const override; + int getChannels () const override; + const SDL_AudioSpec& getSpec () const; + private: + SDL_AudioDeviceID m_deviceID; + bool m_initialized; + SDL_AudioSpec m_audioSpec; + std::vector m_streams; + }; +} \ No newline at end of file diff --git a/src/WallpaperEngine/Core/Objects/CSound.cpp b/src/WallpaperEngine/Core/Objects/CSound.cpp index 31b05cc..6580fa4 100644 --- a/src/WallpaperEngine/Core/Objects/CSound.cpp +++ b/src/WallpaperEngine/Core/Objects/CSound.cpp @@ -5,14 +5,17 @@ using namespace WallpaperEngine::Core::Objects; CSound::CSound ( - CScene* scene, - CUserSettingBoolean* visible, - uint32_t id, - std::string name, - CUserSettingVector3* origin, - CUserSettingVector3* scale, - const glm::vec3& angles) : - CObject (scene, visible, id, std::move(name), Type, origin, scale, angles) + CScene* scene, + CUserSettingBoolean* visible, + uint32_t id, + std::string name, + CUserSettingVector3* origin, + CUserSettingVector3* scale, + const glm::vec3& angles, + bool repeat +) : + CObject (scene, visible, id, std::move(name), Type, origin, scale, angles), + m_repeat (repeat) { } @@ -26,8 +29,13 @@ WallpaperEngine::Core::CObject* CSound::fromJSON ( CUserSettingVector3* scale, const glm::vec3& angles) { + bool repeat = false; // TODO: PARSE AUDIO VOLUME auto sound_it = jsonFindRequired (data, "sound", "Sound information not present"); + auto playbackmode = jsonFindDefault (data, "playbackmode", ""); + + if (playbackmode == "loop") + repeat = true; if ((*sound_it).is_array () == false) sLog.exception ("Expected sound list on element ", name); @@ -39,7 +47,8 @@ WallpaperEngine::Core::CObject* CSound::fromJSON ( name, origin, scale, - angles + angles, + repeat ); for (const auto& cur : (*sound_it)) @@ -57,5 +66,9 @@ const std::vector& CSound::getSounds () const { return this->m_sounds; } +bool CSound::isRepeat () const +{ + return this->m_repeat; +} const std::string CSound::Type = "sound"; \ No newline at end of file diff --git a/src/WallpaperEngine/Core/Objects/CSound.h b/src/WallpaperEngine/Core/Objects/CSound.h index a30b877..38fc727 100644 --- a/src/WallpaperEngine/Core/Objects/CSound.h +++ b/src/WallpaperEngine/Core/Objects/CSound.h @@ -27,6 +27,7 @@ namespace WallpaperEngine::Core::Objects void insertSound (std::string filename); const std::vector& getSounds () const; + bool isRepeat () const; protected: CSound ( @@ -36,11 +37,13 @@ namespace WallpaperEngine::Core::Objects std::string name, CUserSettingVector3* origin, CUserSettingVector3* scale, - const glm::vec3& angles + const glm::vec3& angles, + bool repeat ); static const std::string Type; private: + bool m_repeat; std::vector m_sounds; }; } diff --git a/src/WallpaperEngine/Render/CScene.cpp b/src/WallpaperEngine/Render/CScene.cpp index 2f9a096..8d068f0 100644 --- a/src/WallpaperEngine/Render/CScene.cpp +++ b/src/WallpaperEngine/Render/CScene.cpp @@ -14,8 +14,8 @@ extern float g_TimeLast; using namespace WallpaperEngine; using namespace WallpaperEngine::Render; -CScene::CScene (Core::CScene* scene, CRenderContext* context) : - CWallpaper (scene, Type, context) +CScene::CScene (Core::CScene* scene, CRenderContext* context, CAudioContext* audioContext) : + CWallpaper (scene, Type, context, audioContext) { // setup the scene camera this->m_camera = new CCamera (this, scene->getCamera ()); diff --git a/src/WallpaperEngine/Render/CScene.h b/src/WallpaperEngine/Render/CScene.h index 623cb9c..22fb0f5 100644 --- a/src/WallpaperEngine/Render/CScene.h +++ b/src/WallpaperEngine/Render/CScene.h @@ -15,7 +15,7 @@ namespace WallpaperEngine::Render class CScene : public CWallpaper { public: - CScene (Core::CScene* scene, CRenderContext* context); + CScene (Core::CScene* scene, CRenderContext* context, CAudioContext* audioContext); CCamera* getCamera () const; diff --git a/src/WallpaperEngine/Render/CVideo.cpp b/src/WallpaperEngine/Render/CVideo.cpp index 3cdebee..287f5df 100644 --- a/src/WallpaperEngine/Render/CVideo.cpp +++ b/src/WallpaperEngine/Render/CVideo.cpp @@ -14,8 +14,8 @@ void* get_proc_address (void* ctx, const char* name) return reinterpret_cast (glfwGetProcAddress (name)); } -CVideo::CVideo (Core::CVideo* video, CRenderContext* context) : - CWallpaper (video, Type, context), +CVideo::CVideo (Core::CVideo* video, CRenderContext* context, CAudioContext* audioContext) : + CWallpaper (video, Type, context, audioContext), m_width (16), m_height (16) { diff --git a/src/WallpaperEngine/Render/CVideo.h b/src/WallpaperEngine/Render/CVideo.h index 5693df8..0764ac2 100644 --- a/src/WallpaperEngine/Render/CVideo.h +++ b/src/WallpaperEngine/Render/CVideo.h @@ -12,7 +12,7 @@ namespace WallpaperEngine::Render class CVideo : public CWallpaper { public: - CVideo (Core::CVideo* video, CRenderContext* context); + CVideo (Core::CVideo* video, CRenderContext* context, CAudioContext* audioContext); Core::CVideo* getVideo (); diff --git a/src/WallpaperEngine/Render/CWallpaper.cpp b/src/WallpaperEngine/Render/CWallpaper.cpp index 1f0b39d..b7673bf 100644 --- a/src/WallpaperEngine/Render/CWallpaper.cpp +++ b/src/WallpaperEngine/Render/CWallpaper.cpp @@ -9,11 +9,12 @@ using namespace WallpaperEngine::Render; -CWallpaper::CWallpaper (Core::CWallpaper* wallpaperData, std::string type, CRenderContext* context) : +CWallpaper::CWallpaper (Core::CWallpaper* wallpaperData, std::string type, CRenderContext* context, CAudioContext* audioContext) : m_wallpaperData (wallpaperData), m_type (std::move(type)), m_context (context), - m_destFramebuffer (GL_NONE) + m_destFramebuffer (GL_NONE), + m_audioContext (audioContext) { // generate the VAO to stop opengl from complaining glGenVertexArrays (1, &this->m_vaoBuffer); @@ -329,6 +330,11 @@ CRenderContext* CWallpaper::getContext () return this->m_context; } +CAudioContext* CWallpaper::getAudioContext () +{ + return this->m_audioContext; +} + CFBO* CWallpaper::createFBO (const std::string& name, ITexture::TextureFormat format, ITexture::TextureFlags flags, float scale, uint32_t realWidth, uint32_t realHeight, uint32_t textureWidth, uint32_t textureHeight) { CFBO* fbo = new CFBO (name, format, flags, scale, realWidth, realHeight, textureWidth, textureHeight); @@ -359,12 +365,12 @@ CFBO* CWallpaper::getFBO () const return this->m_sceneFBO; } -CWallpaper* CWallpaper::fromWallpaper (Core::CWallpaper* wallpaper, CRenderContext* context) +CWallpaper* CWallpaper::fromWallpaper (Core::CWallpaper* wallpaper, CRenderContext* context, CAudioContext* audioContext) { if (wallpaper->is () == true) - return new WallpaperEngine::Render::CScene (wallpaper->as (), context); + return new WallpaperEngine::Render::CScene (wallpaper->as (), context, audioContext); else if (wallpaper->is () == true) - return new WallpaperEngine::Render::CVideo (wallpaper->as (), context); + return new WallpaperEngine::Render::CVideo (wallpaper->as (), context, audioContext); else sLog.exception ("Unsupported wallpaper type"); } \ No newline at end of file diff --git a/src/WallpaperEngine/Render/CWallpaper.h b/src/WallpaperEngine/Render/CWallpaper.h index ea71acc..0721aa9 100644 --- a/src/WallpaperEngine/Render/CWallpaper.h +++ b/src/WallpaperEngine/Render/CWallpaper.h @@ -10,8 +10,10 @@ #include "CFBO.h" #include "CRenderContext.h" #include "WallpaperEngine/Assets/CContainer.h" +#include "WallpaperEngine/Audio/CAudioContext.h" using namespace WallpaperEngine::Assets; +using namespace WallpaperEngine::Audio; namespace WallpaperEngine::Render { @@ -25,7 +27,7 @@ namespace WallpaperEngine::Render template bool is () { return this->m_type == T::Type; } - CWallpaper (Core::CWallpaper* wallpaperData, std::string type, CRenderContext* context); + CWallpaper (Core::CWallpaper* wallpaperData, std::string type, CRenderContext* context, CAudioContext* audioContext); ~CWallpaper (); /** @@ -43,6 +45,11 @@ namespace WallpaperEngine::Render */ CRenderContext* getContext (); + /** + * @return The current audio context for this wallpaper + */ + CAudioContext* getAudioContext (); + /** * @return The scene's framebuffer */ @@ -105,7 +112,7 @@ namespace WallpaperEngine::Render * * @return */ - static CWallpaper* fromWallpaper (Core::CWallpaper* wallpaper, CRenderContext* context); + static CWallpaper* fromWallpaper (Core::CWallpaper* wallpaper, CRenderContext* context, CAudioContext* audioContext); protected: /** @@ -160,5 +167,9 @@ namespace WallpaperEngine::Render * Context that is using this wallpaper */ CRenderContext* m_context; + /* + * Audio context that is using this wallpaper + */ + CAudioContext* m_audioContext; }; } diff --git a/src/WallpaperEngine/Render/Objects/CSound.cpp b/src/WallpaperEngine/Render/Objects/CSound.cpp index f8b06d0..67dc8e0 100644 --- a/src/WallpaperEngine/Render/Objects/CSound.cpp +++ b/src/WallpaperEngine/Render/Objects/CSound.cpp @@ -18,8 +18,15 @@ void CSound::load () uint32_t filesize = 0; const void* filebuffer = this->getContainer ()->readFile (cur, &filesize); - this->m_audioStreams.push_back (new Audio::CAudioStream (filebuffer, filesize)); + auto stream = new Audio::CAudioStream (this->getScene ()->getAudioContext (), filebuffer, filesize); + + stream->setRepeat (this->m_sound->isRepeat ()); + + this->m_audioStreams.push_back (stream); this->m_soundBuffer.push_back (filebuffer); + + // add the stream to the context so it can be played + this->getScene ()->getAudioContext ()->addStream (stream); } }