Valkka  0.17.0
OpenSource Video Management
openglthread.h
Go to the documentation of this file.
1 #ifndef openglthread_HEADER_GUARD
2 #define openglthread_HEADER_GUARD
3 /*
4  * openglthread.h : The OpenGL thread for presenting frames and related data structures
5  *
6  * Copyright 2017, 2018 Valkka Security Ltd. and Sampsa Riikonen.
7  *
8  * Authors: Sampsa Riikonen <sampsa.riikonen@iki.fi>
9  *
10  * This file is part of the Valkka library.
11  *
12  * Valkka is free software: you can redistribute it and/or modify
13  * it under the terms of the GNU Affero General Public License as
14  * published by the Free Software Foundation, either version 3 of the
15  * License, or (at your option) any later version.
16  *
17  * This program is distributed in the hope that it will be useful,
18  * but WITHOUT ANY WARRANTY; without even the implied warranty of
19  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
20  * GNU Affero General Public License for more details.
21  *
22  * You should have received a copy of the GNU Affero General Public License
23  * along with this program. If not, see <https://www.gnu.org/licenses/>
24  *
25  */
26 
37 #include "tex.h"
38 #include "openglframefifo.h"
39 #include "constant.h"
40 #include "thread.h"
41 #include "shader.h"
42 #include "threadsignal.h"
43 #include "openglobject.h"
44 
45 
46 typedef void ( *PFNGLXSWAPINTERVALEXTPROC) (Display *dpy, GLXDrawable drawable, int interval);
47 // typedef void ( *PFNGLXSWAPINTERVALEXTPROC) (Display *dpy, GLXDrawable drawable, int interval);
48 
58 class SlotContext {
59 
60 public:
62  ~SlotContext();
63  ban_copy_ctor(SlotContext);
64  ban_copy_asm(SlotContext);
65 
66 public:
70  BitmapPars bmpars;
71  long int lastmstime;
72 
73 private:
74  uint ref_count;
75  bool active;
76  bool is_dead;
77  AVCodecID codec_id;
78  bool load_flag;
79  bool keep_flag;
80 
81 private:
82  long int prev_mstimestamp;
83 
84 public:
85  // void activate(BitmapPars bmpars, YUVShader* shader); ///< Allocate SlotContext::yuvtex (for a frame of certain size) and SlotContext::shader.
86  void activate(BitmapPars bmpars);
87  void deActivate();
88  void loadYUVFrame(YUVFrame* yuvframe);
89  YUVTEX* getTEX();
90  bool manageTimer(long int mstime);
91  void checkIfDead(long int mstime);
92  bool isPending(long int mstime);
93 
94 public: // getters
95  bool isUsed() const {return ref_count>0;}
96  bool isActive() const {return active;}
97  bool isDead() const {return is_dead;}
98 
99 public: // setters
100  void inc_ref_count() {ref_count++;}
101  void dec_ref_count() {ref_count--;}
102  void loadFlag(bool val);
103  void keepFlag(bool val);
104 };
105 
106 
107 
113 
114 public:
121  RenderContext(SlotContext* slot_context, int id, unsigned int z=0);
122  virtual ~RenderContext();
123  RenderContext(const RenderContext &f) : z(0) notice_ban_copy_ctor();
124  ban_copy_asm(RenderContext);
125 
126 public: // Initialized at constructor init list or at constructor
128  const unsigned int z;
129  // bool active;
130  int id;
131 
132 private: // Initialized at RenderContext::init that calls RenderContext::activate
133  GLuint VAO;
134  GLuint VBO;
135  GLuint EBO;
136  std::array<GLfloat,16> transform;
137  std::array<GLfloat,20> vertices;
138  std::array<GLuint, 6> indices;
139 
140 private: // some aux timing variables
141  long int render_mstime, render_mstime_old;
142 
143 private:
144  std::list<OverlayObject*> overlays;
145 
146 private:
147  void activate();
148 
149 public:
150  XWindowAttributes x_window_attr;
151 
152 public: // getters
153  int getId() const {return id;}
154  // bool isActive() {return active;} ///< Legacy. Now, if instantiated, is active!
155 
156 public:
157  // bool activate(); ///< Activate this RenderContext (now it can render). Legacy
158  // bool activateIf(); ///< Try to activate if not active already. Legacy
159 
160  void addRectangle(float left, float right, float top, float bottom);
161  void clearObjects();
162 
163  void renderTexture();
164  void renderObjects();
165 
166  void render(XWindowAttributes x_window_attr);
167  void bindTextures();
168  void bindVars();
169  void bindVarsObj();
170 
171  void bindVertexArray();
172 };
173 
174 // https://stackoverflow.com/questions/4421706/what-are-the-basic-rules-and-idioms-for-operator-overloading
175 // inline bool operator==(const RenderContext& lhs, const RenderContext& rhs){ return (lhs.getId()==rhs.getId()); }
176 
177 
182 class RenderGroup {
183 
184 public:
193  RenderGroup(Display *display_id, const GLXContext &glc, Window window_id, Window child_id, bool doublebuffer_flag=true);
194  ~RenderGroup();
195 
196 public:
197  Display* display_id;
198  const GLXContext &glc;
199  Window window_id;
200  Window child_id;
202  std::list<RenderContext*> render_contexes;
203  XWindowAttributes x_window_attr;
204 
205 public: // getters
206  Window getWindowId() const {return window_id;}
207  const std::list<RenderContext*>& getRenderContexes() const {return render_contexes;}
208 
209 public:
210  std::list<RenderContext*>::iterator getContext(int id);
211  bool addContext(RenderContext *render_context);
212  RenderContext* delContext(int id);
213  bool isEmpty();
214 
221  void render();
222 };
223 
224 
228 namespace swap_flavors {
229  const static unsigned none =0;
230  const static unsigned ext =1;
231  const static unsigned mesa =2;
232  const static unsigned sgi =3;
233 }
234 
235 
247 class OpenGLThread : public Thread { // <pyapi>
248 
249 public: // <pyapi>
257  OpenGLThread(const char* name, OpenGLFrameFifoContext fifo_ctx=OpenGLFrameFifoContext(), unsigned msbuftime=DEFAULT_OPENGLTHREAD_BUFFERING_TIME, const char* x_connection=""); // <pyapi>
258  virtual ~OpenGLThread();
259 
260 protected: // initialized at init list
263  std::string x_connection;
264 
265 protected: // Shaders. Initialized by makeShaders.
266  // Future developments: create a shader instance (and a program) per each stream, etc..?
267  // these could be reserved into the stack right here
268  // YUVShader yuv_shader;
269  // RGBShader rgb_shader;
270  // .. but, we want to do all OpenGL calls after differentiating the Thread
273 
274 protected:
276  std::string static_texture_file;
277 
278 protected: // Variables related to X11 and GLX. Initialized by initGLX.
279  Display* display_id;
280  bool doublebuffer_flag;
281  GLXContext glc;
282  int* att;
283  Window root_id;
284  XVisualInfo* vi;
285  GLXFBConfig* fbConfigs;
286  Colormap cmap;
288 
289  std::vector<SlotContext*> slots_;
290  // std::map<SlotNumber, SlotContext> slots_; ///< SlotNumber => SlotContext mapping TODO: start using this in the future
291  std::map<Window, RenderGroup> render_groups;
292  std::vector<std::list<RenderGroup*>> render_lists;
293  std::map<int, RenderContext*> render_contexes;
294 
295  // std::map<SlotNumber,std::list<RenderGroup*>> render_lists; ///< TODO: start using this in the future
296 
297 private: // function pointers for glx extensions
298  PFNGLXSWAPINTERVALEXTPROC pglXSwapIntervalEXT;
299  PFNGLXGETSWAPINTERVALMESAPROC pglXGetSwapIntervalMESA;
300  PFNGLXSWAPINTERVALMESAPROC pglXSwapIntervalMESA;
301 
302 public: // methods for getting/setting the swap interval .. these choose the correct method to call
303  unsigned swap_flavor;
305  unsigned getSwapInterval(GLXDrawable drawable=0);
306  void setSwapInterval(unsigned i, GLXDrawable drawable=0);
307 
308 protected: // Variables related to queing and presentation
309  unsigned msbuftime;
311  std::list<Frame*> presfifo;
312  long int calltime, callswaptime;
313 
314 protected: // debugging variables
315  bool debug;
316 
317 public: // manipulate RenderGroup(s)
318  bool hasRenderGroup(Window window_id);
319  RenderGroup& getRenderGroup(Window window_id);
320  bool newRenderGroup(Window window_id);
321  bool delRenderGroup(Window window_id);
322 
323 public: // manipulate RenderContex(es)
324  // int newRenderContext(SlotNumber slot, Window window_id, unsigned int z); ///< Creates a new render context
325  void newRenderContext(SlotNumber slot, Window window_id, int id, unsigned int z);
326  bool delRenderContext(int id);
327 
328 public: // add overlay objects (rectangles, etc)
329  bool contextAddRectangle(int id, float left, float right, float top, float bottom);
330  bool contextClearObjects(int id);
331 
332 public: // loading and rendering actions
333  void loadYUVFrame(SlotNumber n_slot, YUVFrame *yuvframe);
334  void render(SlotNumber n_slot);
335  void checkSlots(long int mstime);
336 
337 public: // getter methods
338  Display* getDisplayId() {return display_id;}
339  const GLXContext& getGlc() {return glc;}
340  const Window& getRootId() {return root_id;}
341 
342 public: // slot methods
343  bool slotUsed (SlotNumber i);
344  void activateSlot (SlotNumber i, YUVFrame *yuvframe);
345  void activateSlotIf (SlotNumber i, YUVFrame *yuvframe); // Activate slot if it's not already active or if the texture has changed
346  bool manageSlotTimer(SlotNumber i, long int mstime);
347 
348 public: // setter methods
349  void debugOn() {debug=true;}
350  void debugOff() {debug=false;}
351 
352 public: // Thread variables
353  std::deque<OpenGLSignalContext> signal_fifo;
354 
355 public: // Thread virtual methods
356  void run();
357  void preRun();
358  void postRun();
359  void handleSignal(OpenGLSignalContext &signal_ctx);
360  void handleSignals();
361  void sendSignal(OpenGLSignalContext signal_ctx);
362  void sendSignalAndWait(OpenGLSignalContext signal_ctx);
363 
364 public: // methods, internal : initializing / closing .. but we might want to test these separately, keep public
365  void initGLX();
366  void closeGLX();
367  void loadExtensions();
368  void VSyncOff();
369  int hasCompositor(int screen);
370  void makeShaders();
371  void delShaders();
372  void reserveFrames();
373  void releaseFrames();
374 
375 protected: // internal methods
376  void dumpPresFifo();
377  void diagnosis();
378  void resetCallTime();
379  void reportCallTime(unsigned i);
380  long unsigned insertFifo(Frame* f);
381  void readStaticTex();
382 
387  long unsigned handleFifo();
388  void delRenderContexes();
389 
390 public: // reporting
391  void reportSlots();
392  void reportRenderGroups();
393  void reportRenderList();
394 
395  void dumpYUVStacks() {infifo->dumpYUVStacks();}
396  void YUVdiagnosis() {infifo->YUVdiagnosis();}
397  void dumpInFifo() {infifo->dumpFifo();}
398 
399 public: // testing
400  void recycle(Frame* f) {infifo->recycle(f);}
401 
402 public: // for testing // <pyapi>
403  Window createWindow(bool map=true, bool show=false);
404  void makeCurrent(const Window window_id);
405  unsigned getVsyncAtStartup();
406  void reConfigWindow(Window window_id);
407  Window getChildWindow(Window parent_id);
408 
409 public: // API // <pyapi>
410  FifoFrameFilter &getFrameFilter();
411  void setStaticTexFile(const char* fname);
412 
415  void requestStopCall(); // <pyapi>
416 
419  void infoCall(); // <pyapi>
420 
424  bool newRenderGroupCall (Window window_id); // <pyapi>
425 
429  bool delRenderGroupCall (Window window_id); // <pyapi>
430 
440  int newRenderContextCall (SlotNumber slot, Window window_id, unsigned int z); // <pyapi>
441 
446  bool delRenderContextCall(int id); // <pyapi>
447 
453  void addRectangleCall(int id, float left, float right, float top, float bottom); // <pyapi>
454 
459  void clearObjectsCall(int id); // <pyapi>
460 
461 }; // <pyapi>
462 
463 #endif
int id
A unique id identifying this render context.
Definition: openglthread.h:130
Window window_id
X window id: render group id.
Definition: openglthread.h:199
std::map< int, RenderContext * > render_contexes
Shortcut to render contexes.
Definition: openglthread.h:293
Group of bitmaps that are rendered into the same X-window.
Definition: openglthread.h:182
std::list< Frame * > presfifo
Double-linked list of buffered frames about to be presented.
Definition: openglthread.h:311
GLuint EBO
id of the element buffer object
Definition: openglthread.h:135
unsigned msbuftime
Buffering time in milliseconds.
Definition: openglthread.h:309
virtual void recycle(Frame *f)
Like FrameFifo::recycle_ but with mutex protection.
Definition: framefifo.cpp:202
A class encapsulating information about an OpenGL texture set for a YUV pixmap (sizes, OpenGL reference ids, etc.)
Definition: tex.h:77
Window child_id
X window id: rendering target.
Definition: openglthread.h:200
virtual void dumpFifo()
Dump frames in the fifo.
Definition: framefifo.cpp:241
YUVFrame * dummyframe
A PBO membuf which we reserve from the GPU as the first membuf, but is never used.
Definition: openglthread.h:287
const GLXContext & glc
GLX Context.
Definition: openglthread.h:198
std::string static_texture_file
Name of the file where statictex is.
Definition: openglthread.h:276
YUVTEX * getTEX()
Returns the relevant texture set (static or live)
Definition: openglthread.cpp:145
std::array< GLuint, 6 > indices
data of the element buffer object
Definition: openglthread.h:138
A GPU YUV frame.
Definition: frame.h:485
Encapsulates data sent by the signal.
Definition: threadsignal.h:130
void checkIfDead(long int mstime)
Compares SlotContext::lastmstime to mstime and changes the state of SlotContext::is_dead.
Definition: openglthread.cpp:119
GLuint VBO
id of the vertex buffer object
Definition: openglthread.h:134
A FrameFifo managed and used by OpenGLThread.
Definition: openglframefifo.h:69
bool isActive() const
Check if active.
Definition: openglthread.h:96
Constant/default values, version numbers.
Definition: shader.h:108
For AVBitmapFrames, linesizes are the widths + padding bytes.
Definition: constant.h:129
bool load_flag
This flag is cleared when a new frame is loaded.
Definition: openglthread.h:78
long int prev_mstimestamp
for debugging: check if frames are fed in correct timestamp order
Definition: openglthread.h:82
std::array< GLfloat, 16 > transform
data of the transformation matrix
Definition: openglthread.h:136
FifoFrameFilter infilter
A FrameFilter for writing incoming frames.
Definition: openglthread.h:262
RGBShader * rgb_shader
Initialized by OpenGLThread::makeShaders.
Definition: openglthread.h:272
void dumpInFifo()
Incoming fifo: here we have all kinds of frames, including YUVFrame(s)
Definition: openglthread.h:397
Encapsulates data for rendering a single bitmap: vertex array object (VAO), vertex buffer object (VBO...
Definition: openglthread.h:112
const unsigned int z
Stacking order.
Definition: openglthread.h:128
bool is_dead
Has received frames or not for a while ?
Definition: openglthread.h:76
void YUVdiagnosis()
Brief resumen of the state of the YUV Frame stack.
Definition: openglthread.h:396
std::string x_connection
X-server connection string (i.e. ":0.0", ":0.1", etc.
Definition: openglthread.h:263
void dumpYUVStacks()
State of the YUV Frame stack.
Definition: openglthread.h:395
Definition: shader.h:92
std::vector< SlotContext * > slots_
index => SlotContext mapping (based on vector indices) // WARNING: not they&#39;re pointers ...
Definition: openglthread.h:289
void dumpYUVStacks()
Dump frames in OpenGLFrameFifo::yuv_stacks.
Definition: openglframefifo.cpp:223
bool keep_flag
Should we keep on showing the previous frame, even if the slot is_dead?
Definition: openglthread.h:79
Base class for multithreading.
std::array< GLfloat, 20 > vertices
data of the vertex buffer object
Definition: openglthread.h:137
unsigned swap_interval_at_startup
The value of swap interval when this thread was started.
Definition: openglthread.h:304
std::vector< std::list< RenderGroup * > > render_lists
Each vector element corresponds to a slot. Each list inside a vector element has pointers to RenderGr...
Definition: openglthread.h:292
SlotContext(YUVTEX *statictex, YUVShader *shader)
Default constructor.
Definition: openglthread.cpp:51
YUVTEX * yuvtex
This could be a base class for different kinds of textures (now only YUVTEX)
Definition: openglthread.h:67
bool active
Is activated or not. Active = has received a setup frame.
Definition: openglthread.h:75
bool isPending(long int mstime)
Compares SlotContext::lastmstime to mstime and returns true of a treshold is met (if last frame was r...
Definition: openglthread.cpp:130
void recycle(Frame *f)
Recycle a frame back to OpenGLFrameFifo.
Definition: openglthread.h:400
YUVTEX * statictex
Texture to be shown when there is no stream.
Definition: openglthread.h:275
Display * display_id
X display id.
Definition: openglthread.h:197
~SlotContext()
Default destructor.
Definition: openglthread.cpp:55
std::map< Window, RenderGroup > render_groups
window_id => RenderGroup mapping. RenderGroup objects are warehoused here.
Definition: openglthread.h:291
XWindowAttributes x_window_attr
X window size, etc.
Definition: openglthread.h:150
long int callswaptime
Debugging: when handleFifo was last called?
Definition: openglthread.h:312
void YUVdiagnosis()
Brief resumen of OpenGLFrameFifo::yuv_stacks.
Definition: openglframefifo.cpp:239
std::deque< OpenGLSignalContext > signal_fifo
Redefinition of signal fifo. Signal fifo of Thread::SignalContext(s) is now hidden.
Definition: openglthread.h:353
YUVTEX * statictex
A static texture to be shown on the screen if no video is received.
Definition: openglthread.h:68
std::list< RenderContext * > render_contexes
RenderContext instances in ascending z order. User created rendercontexes are warehoused here...
Definition: openglthread.h:202
GLX extensions for controlling the vertical sync / framerate issue are a mess.
Definition: openglthread.h:228
GLuint VAO
id of the vertex array object
Definition: openglthread.h:133
YUVShader * shader
Base class for the chosen Shader for this slot. Now always YUVShader.
Definition: openglthread.h:69
bool manageTimer(long int mstime)
Updates SlotContext::lastmstime. Returns false if too little time has passed since receiving the last...
Definition: openglthread.cpp:105
A class for multithreading with a signaling system.
Definition: thread.h:90
bool doublebuffer_flag
Double buffer rendering or not?
Definition: openglthread.h:201
Frame: An abstract queueable class.
Definition: frame.h:108
Passes frames to a FrameFifo.
Definition: framefilter.h:560
Describes the stack structure and fifo behaviour for an OpenGLFrameFifo.
Definition: openglframefifo.h:45
long int lastmstime
Last millisecond timestamp when this slot received a frame.
Definition: openglthread.h:71
unsigned future_ms_tolerance
If frame is this much in the future, it will be discarded. See OpenGLThread::OpenGLThread for its val...
Definition: openglthread.h:310
void deActivate()
Deallocate textures.
Definition: openglthread.cpp:76
void activate(BitmapPars bmpars)
Allocate SlotContext::yuvtex (for a frame of certain size)
Definition: openglthread.cpp:61
XWindowAttributes x_window_attr
X window attributes.
Definition: openglthread.h:203
AVCodecID codec_id
FFmpeg codec id.
Definition: openglthread.h:77
Handling OpenGL textures.
OpenGLFrameFifo * infifo
This thread reads from this communication fifo.
Definition: openglthread.h:261
This class does a lot of things:
Definition: openglthread.h:247
int getId() const
Returns the unique id of this render context.
Definition: openglthread.h:153
SlotContext * slot_context
Knows relevant Shader and YUVTEX.
Definition: openglthread.h:127
void loadYUVFrame(YUVFrame *yuvframe)
Load bitmap from YUVFrame to SlotContext::yuvtex.
Definition: openglthread.cpp:88
Each Frame carries information about it&#39;s slot number in Frame::slot.
Definition: openglthread.h:58
YUVShader * yuv_shader
Initialized by OpenGLThread::makeShaders.
Definition: openglthread.h:271
OpenGL objects, i.e. stuff that can be drawn on the OpenGL canvas on top the textures (boxes...