Some cleanup in Gstreamer Audio, should be finished 100% now! Please test

Frames are now pulled in a thread for better performance. We still need to grab the right frame from the Framebuffer and we might need to switch Gstreamer's Frame stepping API.
refs #1

GstreamerHelper/GstreamerAudio.cpp

@@ -1,11 +1,5 @@
 #include "stdafx.h"
 #include "GstreamerAudio.h"
-#include "GstreamerAudioStream.h"
-#include "gst\gst.h"
-#include <string>
-#include <sstream>
-#include <map>
-#include <queue>
 
 #define DllExport extern "C" __declspec(dllexport)
 

GstreamerHelper/GstreamerAudio.h

@@ -1,6 +1,11 @@
 #pragma once
 #include <string>
+#include <map>
+#include <queue>
 #include "gst/gst.h"
+#include "GstreamerAudioStream.h"
+#include "gst\gst.h"
+
 #define DllExport extern "C" __declspec(dllexport)
 
 using namespace std;

GstreamerHelper/GstreamerAudioStream.cpp

@@ -1,8 +1,6 @@
 #include "stdafx.h"
 #include "GstreamerAudioStream.h"
-#include "GstreamerAudio.h"
+
-#include "gst/gst.h"
-#include <string>
 
 typedef enum {
   GST_PLAY_FLAG_VIDEO         = (1 << 0),
@@ -28,8 +26,6 @@ GstreamerAudioStream::GstreamerAudioStream(int id)
 	Running = true;
 
 	Finished = false;
-	Playing = false;
-	Paused = false;
 
 	//Fading
 	FadeTimer = g_timer_new();
@@ -76,7 +72,6 @@ int GstreamerAudioStream::Load(const wchar_t* Media)
 
 	gst_element_set_state(Element, GST_STATE_PAUSED);
 	gst_bus_timed_pop_filtered(Bus, -1, GST_MESSAGE_ASYNC_DONE);
-	Paused = true;
 	RefreshDuration();
 
 	return ID;
@@ -132,16 +127,12 @@ void GstreamerAudioStream::Play(void)
 
 void GstreamerAudioStream::Play(bool Loop)
 {
-	Playing = true;
-	Paused = false;
 	this->Loop = Loop;
 	Play();
 }
 
 void GstreamerAudioStream::Pause()
 {
-	Playing = false;
-	Paused = true;
 	Running = true;
 	if(Element)
 		gst_element_set_state(Element, GST_STATE_PAUSED);
@@ -154,8 +145,6 @@ void GstreamerAudioStream::Stop()
 	{
 		gst_element_set_state(Element, GST_STATE_NULL);
 		gst_element_seek_simple(Element, GST_FORMAT_TIME, GST_SEEK_FLAG_FLUSH, 0);
-		Running = true;
-		Playing = false;
 	}
 }
 

GstreamerHelper/GstreamerAudioStream.h

@@ -2,8 +2,10 @@
 #pragma warning(disable: 4244) // possible loss of data
 #pragma warning(disable: 4267) // possible loss of data
 #pragma warning(disable: 4800)
+
 #include <string>
 #include "gst/gst.h"
+#include "GstreamerAudio.h"
 
 using namespace std;
 
@@ -49,8 +51,6 @@ private:
 	gboolean Running;
 	gboolean Loop;
 
-	gboolean Playing;
-	gboolean Paused;
 	gboolean Finished;
 
 	gdouble MaxVolume;

GstreamerHelper/GstreamerVideo.cpp

@@ -23,6 +23,7 @@ DllExport bool InitVideo()
 #endif
 
 	gst_init(NULL, NULL);
+
 	GstRegistry* registry = gst_registry_get();
 
 #if _WIN64

GstreamerHelper/GstreamerVideo.h

@@ -11,7 +11,7 @@ using namespace std;
 	static LogCallback Log;
 
 	struct ApplicationFrame {
-		guint8* buffer;
+		guint8* data;
 		int size;
 		int width;
 		int height;

GstreamerHelper/GstreamerVideoStream.cpp

@@ -19,9 +19,6 @@ GstreamerVideoStream::GstreamerVideoStream(int ID)
 {
 	this->ID = ID;
 
-	g_main_loop_new(Context, false);
-	Clock = vocaluxe_clock_new("external");
-
 	Loop = false;
 
 	Running = true;
@@ -29,6 +26,15 @@ GstreamerVideoStream::GstreamerVideoStream(int ID)
 	Paused = false;
 	Finished = false;
 	Closed = false;
+	Copying = true;
+
+	BufferFull = false;
+	for(int i = 0; i < 5; i++)
+	{
+		FrameBuffer[i].Sample = NULL;
+		FrameBuffer[i].Buffer = NULL;
+		FrameBuffer[i].Displayed = true;
+	}
 
 	Duration = -1.0;
 }
@@ -42,12 +48,10 @@ GstreamerVideoStream::~GstreamerVideoStream(void)
 int GstreamerVideoStream::LoadVideo(const wchar_t* Filename)
 {
 	Element = gst_element_factory_make("playbin", "playbin");
-
-
 	Appsink = (GstAppSink*) gst_element_factory_make("appsink", "appsink");
 
 	Bus = gst_element_get_bus(Element);
-	if(!Element)
+	if(!Element || !Appsink)
 	{
 		LogVideoError ("Could not create element!");
 		return -1;
@@ -56,8 +60,7 @@ int GstreamerVideoStream::LoadVideo(const wchar_t* Filename)
 	g_object_set(Element, "uri", Filename, NULL);
 	g_object_set(Element, "video-sink", Appsink, NULL);
 	g_object_set(Element, "flags", GST_PLAY_FLAG_VIDEO, NULL); 
-	g_object_set (Appsink, "emit-signals", TRUE, "sync", TRUE, NULL);
+	g_object_set (Appsink, "emit-signals", FALSE, "sync", FALSE, NULL);
-	g_signal_connect(Appsink,"new-sample", G_CALLBACK(fake_callback),(gpointer)this);
 
 	GstCaps *caps;
 	caps = gst_caps_new_simple ("video/x-raw",
@@ -67,70 +70,74 @@ int GstreamerVideoStream::LoadVideo(const wchar_t* Filename)
 	gst_app_sink_set_caps(Appsink, caps);
 
 	gst_app_sink_set_drop(Appsink, false);
-	gst_app_sink_set_max_buffers(Appsink, 1);
+	gst_app_sink_set_max_buffers(Appsink, 5);
 
-	gst_pipeline_use_clock((GstPipeline *)Element, (GstClock*) Clock);
 	gst_element_set_state(Element, GST_STATE_PLAYING);
-	gst_element_set_state((GstElement*)Appsink, GST_STATE_PLAYING);
+	//gst_element_set_state(GST_ELEMENT (Appsink), GST_STATE_PLAYING);
 	gst_bus_timed_pop_filtered(Bus, -1, GST_MESSAGE_ASYNC_DONE);
+	CopyThread = thread (&GstreamerVideoStream::Copy, this);
 
-	Paused = true;
 	RefreshDuration();
 
 	return ID;
 }
 
-GstFlowReturn GstreamerVideoStream::fake_callback(GstAppSink *sink,gpointer data)  {
+void GstreamerVideoStream::Copy()
-
-     GstreamerVideoStream* instance = (GstreamerVideoStream*)data;
-	 return instance->NewBufferRecieved(sink);
-}
-
-GstFlowReturn GstreamerVideoStream::NewBufferRecieved(GstAppSink *sink)
 {
-	float VideoTime = 0.0;
+	while(Copying)
-	int Width = 0;
+	{
-	int Height = 0;
+		int num = -1;
-	int Size = -1;
+		for (int i = 0; i < 5; i++)
-	
-	Sample = gst_app_sink_pull_sample(Appsink);
-	if(Sample) {
-		BufferCaps = gst_sample_get_caps (Sample);
-		BufferStructure = gst_caps_get_structure (BufferCaps, 0);
-
-		Buffer = gst_sample_get_buffer(Sample);
-		gst_buffer_map(Buffer, &Mapinfo, GST_MAP_READ);
-
-		Size = Mapinfo.size;
-		gst_structure_get_int (BufferStructure, "width", &Width);
-		gst_structure_get_int (BufferStructure, "height", &Height);
-		VideoTime = (float) (Buffer->pts / GST_SECOND);
-
-		if(Buffer && &Mapinfo)
 		{
-			gst_buffer_unmap(Buffer, &Mapinfo);
+			if(FrameBuffer[i].Displayed)
-			gst_buffer_unref(Buffer);
+			{
+
+				num = i;
+				GstSample *sample;
+				sample = gst_app_sink_pull_sample(Appsink);
+				Mutex.lock();
+				if(FrameBuffer[i].Buffer)
+				{
+					if(FrameBuffer[i].Memory.data)
+						gst_buffer_unmap(FrameBuffer[i].Buffer, &FrameBuffer[i].Memory);
+					gst_buffer_unref(FrameBuffer[i].Buffer);
+				}
+				//if(FrameBuffer[i].Sample)
+					//gst_sample_unref(FrameBuffer[i].Sample);
+
+
+				FrameBuffer[i].Displayed = false;
+				FrameBuffer[i].Sample = sample;
+				Mutex.unlock();
+
+				//gst_element_send_event (Element, gst_event_new_step (GST_FORMAT_BUFFERS, 1, 1.0, TRUE, FALSE));
+				break;
+			}
 		}
-		if(BufferCaps)
+		if(num == -1)
-			gst_caps_unref(BufferCaps);
+			BufferFull = true;
-
-		if(BufferStructure)
-			gst_object_unref(BufferStructure);
 	}
-
-	Mutex.lock();
-	Frame.buffer = Mapinfo.data;
-	Frame.height = Height;
-	Frame.width = Width;
-	Frame.size = Size;
-	Frame.videotime = VideoTime;
-	Mutex.unlock();
-
-	return GST_FLOW_OK;
 }
 
 bool GstreamerVideoStream::CloseVideo()
 {
+	Copying = false;
+	CopyThread.join();
+
+	for(int i= 0; i< 5; i++)
+	{
+		if(FrameBuffer[i].Buffer)
+		{
+			if(FrameBuffer[i].Memory.data)
+				gst_buffer_unmap(FrameBuffer[i].Buffer, &FrameBuffer[i].Memory);
+			gst_buffer_unref(FrameBuffer[i].Buffer);
+		}
+		//if(FrameBuffer[i].Sample)
+			//gst_sample_unref(FrameBuffer[i].Sample);
+	}
+
+	if(Element)
+		gst_element_set_state(Element, GST_STATE_NULL);
 
 	if(Appsink)
 		gst_object_unref(Appsink);
@@ -138,16 +145,8 @@ bool GstreamerVideoStream::CloseVideo()
 		gst_object_unref(Bus);
 	//if(Buffer && &Mapinfo)
 		//gst_buffer_unmap(Buffer, &Mapinfo);
-	if(Buffer)
-		gst_buffer_unref(Buffer);
-	if(BufferCaps)
-		gst_caps_unref(BufferCaps);
 
-	if(BufferStructure)
-		gst_object_unref(BufferStructure);
 
-	if(Element)
-		gst_element_set_state(Element, GST_STATE_NULL);
 	if(Element)
 		gst_object_unref(Element);
 	return true;
@@ -162,8 +161,51 @@ float GstreamerVideoStream::GetVideoLength()
 
 struct ApplicationFrame GstreamerVideoStream::GetFrame(float Time)
 {
-	vocaluxe_clock_set_time(Clock, Time);
+	float VideoTime = 0.0;
-	return Frame;
+	int Width = 0;
+	int Height = 0;
+	int Size = -1;
+
+	Mutex.lock();
+	for(int i = 0; i < 5; i++)
+	{
+		if(!FrameBuffer[i].Displayed)
+		{
+			if(FrameBuffer[i].Sample) {
+				GstBuffer *Buffer = gst_sample_get_buffer(FrameBuffer[i].Sample);
+				GstMapInfo Mapinfo;
+
+				GstCaps *BufferCaps = gst_sample_get_caps (FrameBuffer[i].Sample);
+				GstStructure *BufferStructure = gst_caps_get_structure (BufferCaps, 0);
+
+				gst_buffer_map(Buffer, &Mapinfo, GST_MAP_READ);
+
+				Size = Mapinfo.size;
+				gst_structure_get_int (BufferStructure, "width", &Width);
+				gst_structure_get_int (BufferStructure, "height", &Height);
+				VideoTime = (float) (Buffer->pts / GST_SECOND);
+
+				ReturnFrame.data = Mapinfo.data;
+				ReturnFrame.height = Height;
+				ReturnFrame.width = Width;
+				ReturnFrame.size = Size;
+				ReturnFrame.videotime = VideoTime;
+
+				FrameBuffer[i].Memory = Mapinfo;
+				FrameBuffer[i].Buffer = Buffer;
+
+				if(BufferCaps)
+					gst_caps_unref(BufferCaps);
+
+				if(BufferStructure)
+					gst_object_unref(BufferStructure);
+			}
+			FrameBuffer[i].Displayed = true;
+			break;
+		}
+	}
+	Mutex.unlock();
+	return ReturnFrame;
 }
 
 bool GstreamerVideoStream::Skip(float Start, float Gap)
@@ -179,9 +221,9 @@ bool GstreamerVideoStream::Skip(float Start, float Gap)
 void GstreamerVideoStream::SetVideoLoop(bool Loop)
 {
 	if(Loop)
-		gst_pipeline_auto_clock((GstPipeline *)Element);
+		g_object_set(Element, "sync", TRUE, NULL);
 	else
-		gst_pipeline_use_clock((GstPipeline *)Element, (GstClock*) Clock);
+		g_object_set(Element, "sync", FALSE, NULL);
 	this->Loop = Loop;
 }
 
@@ -217,7 +259,6 @@ void GstreamerVideoStream::RefreshDuration()
 void GstreamerVideoStream::UpdateVideo()
 {
 	if(Running) {
-		g_main_context_iteration(Context, false);
 		Message = gst_bus_pop (Bus);
    
 		/* Parse message */

GstreamerHelper/GstreamerVideoStream.h

@@ -1,5 +1,6 @@
 #pragma once
 #include <mutex>
+#include <thread>
 #include "gst/gst.h"
 #include "GstreamerVideo.h"
 #include "gst\app\gstappsink.h"
@@ -32,36 +33,39 @@ public:
     bool IsFinished();
 
 private:
-	GMainLoop *MainLoop;
+	struct BufferFrame {
-	GMainContext *Context;
+		GstSample *Sample;
-	VocaluxeClock *Clock;
+		GstBuffer *Buffer;
+		GstMapInfo Memory;
+		gboolean Displayed;
+	};
+
+	thread CopyThread;
 
 	GstAppSink *Appsink;
 	GstElement *Element;
 	GstBus *Bus;
 	GstMessage *Message;
 
-	ApplicationFrame Frame;
+	BufferFrame FrameBuffer[5];
-	GstMapInfo Mapinfo;
+	gboolean BufferFull;
-	GstSample *Sample;
-	GstBuffer *Buffer;
-	GstCaps *BufferCaps;
-	GstStructure *BufferStructure;
 	mutex Mutex;
 
 	gboolean Loop;
 
 	gboolean Running;
+	volatile gboolean Copying;
 
 	gboolean Paused;
 	gboolean Finished;
 
 	gfloat Duration;
 
+	ApplicationFrame ReturnFrame;
+
 	void RefreshDuration();
 
-	GstFlowReturn NewBufferRecieved(GstAppSink *Sink);
+	void Copy();
-	static GstFlowReturn fake_callback(GstAppSink *sink ,gpointer data);
 };