Valkka  0.17.0
OpenSource Video Management
frame.h
Go to the documentation of this file.
1 #ifndef frame_HEADER_GUARD
2 #define frame_HEADER_GUARD
3 
4 /*
5  * frame.h : Frame classes
6  *
7  * Copyright 2017, 2018 Valkka Security Ltd. and Sampsa Riikonen.
8  *
9  * Authors: Sampsa Riikonen <sampsa.riikonen@iki.fi>
10  *
11  * This file is part of the Valkka library.
12  *
13  * Valkka is free software: you can redistribute it and/or modify
14  * it under the terms of the GNU Affero General Public License as
15  * published by the Free Software Foundation, either version 3 of the
16  * License, or (at your option) any later version.
17  *
18  * This program is distributed in the hope that it will be useful,
19  * but WITHOUT ANY WARRANTY; without even the implied warranty of
20  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21  * GNU Affero General Public License for more details.
22  *
23  * You should have received a copy of the GNU Affero General Public License
24  * along with this program. If not, see <https://www.gnu.org/licenses/>
25  *
26  */
27 
37 #include "common.h"
38 #include "codec.h"
39 #include "threadsignal.h"
40 #include "constant.h"
41 #include "avdep.h"
42 #include "opengl.h"
43 #include "tools.h"
44 #include "macro.h"
45 #include "rawrite.h"
46 
51 enum class FrameClass
52 {
53  none,
54 
55  basic,
56 
57  avpkt,
58 
59  avmedia,
60  avbitmap,
61  // avbitmap_np, ///< child of avmedia: video, non-planar
62 
63  avaudio,
64 
65  avrgb,
66 
67  yuv,
68 
69  rgb,
70 
71  setup,
72  signal,
73 
74  marker,
75 
76  First = none,
77  Last = signal
78 };
79 
84 { // <pyapi>
85  none, // <pyapi>
86  smart, // <pyapi>
87  dummy // <pyapi>
88 }; // <pyapi>
89 
108 class Frame
109 {
110 
111 public:
112  Frame();
113  virtual ~Frame();
114  frame_essentials(FrameClass::none, Frame);
115  frame_clone(FrameClass::none, Frame);
116  /*Frame(const Frame &f); ///< Default copy ctor
117 
118 
119 public: // frame essentials : must be defined for each frame subclass
120  virtual FrameClass getFrameClass(); ///< Returns the subclass frame type. See Frame::frameclass
121  virtual void copyFrom(Frame *f); ///< Copies data to this frame from a frame of the same type (also metadata)
122  */
123 
124 public: // redefined virtual
125  virtual void print(std::ostream &os) const;
126  virtual std::string dumpPayload();
127  virtual void dumpPayloadToFile(std::ofstream &fout);
128  virtual void updateAux();
129  virtual void update();
130  virtual void reset();
131  virtual bool isSeekable();
132 
133 public:
134  void copyMetaFrom(Frame *f);
135 
136 protected:
138 
139 public: // public metadata
140  SlotNumber n_slot;
142  long int mstimestamp;
143 };
144 
145 inline std::ostream &operator<<(std::ostream &os, const Frame &f)
146 {
147  // https://stackoverflow.com/questions/4571611/making-operator-virtual
148  f.print(os);
149  return os;
150 }
151 
160 class BasicFrame : public Frame
161 {
162 
163 public:
164  BasicFrame();
165  virtual ~BasicFrame();
166  frame_essentials(FrameClass::basic, BasicFrame);
167  frame_clone(FrameClass::basic, BasicFrame);
168  /*BasicFrame(const BasicFrame &f); ///< Default copy ctor
169 
170 public: // frame essentials
171  virtual FrameClass getFrameClass(); ///< Returns the subclass frame type. See Frame::frameclass
172  virtual void copyFrom(Frame *f); ///< Copies data to this frame from a frame of the same type
173  */
174 
175 public: // redefined virtual
176  virtual void print(std::ostream &os) const;
177  virtual std::string dumpPayload();
178  virtual void dumpPayloadToFile(std::ofstream &fout);
179  virtual void reset();
180  virtual bool isSeekable();
181 
182 public: // payload handling
183  void reserve(std::size_t n_bytes);
184  void resize(std::size_t n_bytes);
185 
186 public: // frame variables
187  std::vector<uint8_t> payload;
188  AVMediaType media_type;
189  AVCodecID codec_id;
190 
191  /* // nopes ..
192 protected: // for filesystem debugging
193  bool force_seekable;
194 
195 public: // for filesystem debugging
196  void setSeekable();
197  void unSetSeekable();
198 */
199 
200 public: // codec-dependent parameters
202 
203 public: // codec-dependent functions
204  void fillPars();
205  void fillH264Pars();
206 
207 public:
208  void fillAVPacket(AVPacket *avpkt);
209  void copyFromAVPacket(AVPacket *avpkt);
210  void filterFromAVPacket(AVPacket *avpkt, AVCodecContext *codec_ctx, AVBitStreamFilterContext *filter);
211 
212 public: // frame serialization
213  std::size_t calcSize();
214  bool dump(IdNumber device_id, RaWriter &raw_writer);
215  IdNumber read(RawReader &raw_reader);
216 };
217 
222 /* // TODO
223 class MuxFrame : public Frame {
224 
225 public:
226  MuxFrame(); ///< Default ctor
227  virtual ~MuxFrame(); ///< Default virtual dtor
228  frame_essentials(FrameClass::mux, MuxFrame);
229  frame_clone(FrameClass::mux, MuxFrame);
230 
231 public: // redefined virtual
232  virtual void print(std::ostream& os) const; ///< Produces frame output
233  virtual std::string dumpPayload(); ///< Dumps internal payload data
234  virtual void dumpPayloadToFile(std::ofstream& fout); ///< Dumps internal payload data into a file
235  virtual void update(); ///< Update internal auxiliary state variables
236  virtual void reset(); ///< Reset the internal data
237  virtual bool isSeekable(); ///< Can we seek to this frame? (e.g. is it a key-frame .. for H264 sps packets are used as seek markers)
238 
239 
240 // container-dependent parameters
241 public:
242  MP4Pars mp4_pars;
243  WebMPars webm_pars;
244 
245 public:
246  void fillPars(); ///< Fill container-dependent parameters, based on the payload
247  void fillMP4Pars(); ///< Inspect MP4 payload & fill mp4_pars
248 };
249 */
250 
251 enum class SetupFrameType
252 {
253  none,
254  stream_init,
255  stream_state
256 };
257 
270 class SetupFrame : public Frame
271 {
272 
273 public:
274  SetupFrame();
275  virtual ~SetupFrame();
276  frame_essentials(FrameClass::setup, SetupFrame);
277  frame_clone(FrameClass::setup, SetupFrame);
278  /*
279  SetupFrame(const SetupFrame &f); ///< Default copy ctor
280 
281 public: // frame essentials
282  virtual FrameClass getFrameClass(); ///< Returns the subclass frame type. See Frame::frameclass
283  virtual void copyFrom(Frame *f); ///< Copies data to this frame from a frame of the same type
284  */
285 
286 public: // redefined virtual
287  virtual void print(std::ostream &os) const;
288  virtual void reset();
289 
290 public: // managed objects
292 
293  AVMediaType media_type;
294  AVCodecID codec_id;
295 
297 };
298 
308 class AVMediaFrame : public Frame
309 {
310 
311 public:
312  AVMediaFrame();
313  virtual ~AVMediaFrame();
314  //frame_essentials(FrameClass::avmedia,AVMediaFrame); // now this is a virtual class ..
315  //frame_clone(FrameClass::avmedia,AVMediaFrame);
316  /*AVMediaFrame(const AVMediaFrame &f); ///< Default copy ctor
317 
318  public: // frame essentials
319  virtual FrameClass getFrameClass(); ///< Returns the subclass frame type. See Frame::frameclass
320  virtual void copyFrom(Frame *f); ///< Copies data to this frame from a frame of the same type
321  */
322 
323 /*
324 public:
325  virtual void updateAux() = 0; ///< update any helper objects
326  virtual void update() = 0;
327 */
328 
329 public: // redefined virtual
330  virtual std::string dumpPayload();
331  virtual void print(std::ostream &os) const;
332  virtual void reset();
333 
334 public: // helper objects : values should correspond to member av_frame
335  AVMediaType media_type;
336  AVCodecID codec_id;
337 
338 public: // managed objects
339  AVFrame *av_frame;
340 };
341 
355 {
356 
357 public:
358  AVBitmapFrame();
359  virtual ~AVBitmapFrame();
360  frame_essentials(FrameClass::avbitmap, AVBitmapFrame);
361  frame_clone(FrameClass::avbitmap, AVBitmapFrame); // TODO: think about this!
362  /*
363  AVBitmapFrame(const AVBitmapFrame &f); ///< Default copy ctor
364 
365 public: // frame essentials
366  virtual FrameClass getFrameClass(); ///< Returns the subclass frame type. See Frame::frameclass
367  virtual void copyFrom(Frame *f); ///< Copies data to this frame from a frame of the same type
368  */
369 
370 public: // redefined virtual
371  virtual std::string dumpPayload();
372  virtual void print(std::ostream &os) const;
373  virtual void reset();
374  virtual void copyPayloadFrom(AVBitmapFrame *frame);
375  virtual void updateAux();
376  virtual void update();
377  virtual void reserve(int width, int height);
378 
379 public: // helper objects
380  AVPixelFormat av_pixel_format;
382  uint8_t *y_payload;
383  uint8_t *u_payload;
384  uint8_t *v_payload;
385 };
386 
430 class AVRGBFrame : public AVBitmapFrame
431 {
432 
433 public:
434  AVRGBFrame();
435  virtual ~AVRGBFrame();
436  frame_essentials(FrameClass::avrgb, AVRGBFrame);
437  frame_clone(FrameClass::avrgb, AVRGBFrame); // TODO: think about this!
438 
439 public:
440  virtual void reserve(int width, int height);
441 
442 public: // redefined virtual
443  virtual std::string dumpPayload();
444  virtual void print(std::ostream &os) const;
445 };
446 
447 /*
448 class AVAudioFrame : public AVMediaFrame {
449 
450 public:
451  AVAudioFrame();
452  virtual ~AVAudioFrame();
453 
454 public: // frame essentials
455  virtual FrameClass getFrameClass(); ///< Returns the subclass frame type. See Frame::frameclass
456  virtual void copyFrom(Frame *f); ///< Copies data to this frame from a frame of the same type
457  virtual std::string dumpPayload();
458  virtual void print(std::ostream& os) const; ///< How to print this frame to output stream
459 
460 public:
461  virtual void getParametersDecoder(const AVCodecContext *ctx); ///< Extract sample rate
462 
463 public: // ffmpeg media parameters
464  AVSampleFormat av_sample_fmt;
465 };
466 */
467 
468 // Decoder gives AVBitmapFrame => OpenGLThread internal FrameFilter => OpenGLThread FrameFifo::writeCopy(Frame *f) => .. returns once a copy has been made
469 
485 class YUVFrame : public Frame
486 {
487 
488 public:
489  YUVFrame(BitmapPars bmpars);
490  virtual ~YUVFrame();
491  frame_essentials(FrameClass::yuv, YUVFrame);
492  // TODO: frame_clone
493  /*
494  YUVFrame(const YUVFrame &f); ///< Default copy ctor
495 
496 public: // frame essentials
497  virtual FrameClass getFrameClass(); ///< Returns the subclass frame type. See Frame::frameclass
498  virtual void copyFrom(Frame *f); ///< Copies data to this frame from a frame of the same type
499 
500  */
501 
502 public: // variables filled at constructor time
503  BitmapPars bmpars; // the maximum, pre-reserved size
504  BitmapPars source_bmpars; // the actual size of the image
505 
506 private:
507  void reserve();
508  void release();
509 
510 public: // variables used/filled by reserve
511  GLuint y_index;
512  GLuint u_index;
513  GLuint v_index;
514 
515 private:
516  GLubyte *y_payload;
517  GLubyte *u_payload;
518  GLubyte *v_payload;
519 
520 public: // redefined virtual
521  virtual std::string dumpPayload();
522  virtual void print(std::ostream &os) const;
523  virtual void reset();
524 
525 public:
526  void fromAVBitmapFrame(AVBitmapFrame *f);
527 };
528 
535 class RGBFrame : public Frame
536 {
537 
538 public:
539  RGBFrame(int max_width, int max_height);
540  virtual ~RGBFrame();
541  frame_essentials(FrameClass::rgb, RGBFrame);
542 
543 private:
544  std::vector<uint8_t> payload;
545  int max_width, max_height;
546  int width, height;
547 
548 public: // redefined virtual
549  virtual std::string dumpPayload();
550  virtual void print(std::ostream &os) const;
551  virtual void reset();
552 
553 public:
554  void fromAVRGBFrame(AVRGBFrame *f);
555 };
556 
557 typedef std::vector<Frame *> Reservoir;
558 typedef std::deque<Frame *> Stack;
559 typedef std::deque<Frame *> Fifo;
560 typedef std::deque<Frame *> Cache;
561 
562 typedef std::vector<YUVFrame *> YUVReservoir;
563 typedef std::deque<YUVFrame *> YUVStack;
564 typedef std::vector<RGBFrame *> RGBReservoir;
565 typedef std::deque<RGBFrame *> RGBStack;
566 
570 class SignalFrame : public Frame
571 {
572 
573 public:
574  SignalFrame();
575  virtual ~SignalFrame();
576  frame_essentials(FrameClass::signal, SignalFrame);
577  frame_clone(FrameClass::signal, SignalFrame);
578 
579 public:
585 };
586 
587 class MarkerFrame : public Frame
588 {
589 
590 public:
591  MarkerFrame();
592  virtual ~MarkerFrame();
593  frame_essentials(FrameClass::marker, MarkerFrame);
594  frame_clone(FrameClass::marker, MarkerFrame);
595 
596 public: // redefined virtual
597  // virtual std::string dumpPayload();
598  virtual void print(std::ostream &os) const;
599  virtual void reset();
600 
601 public:
602  bool fs_start, fs_end;
603  bool tm_start, tm_end;
604 };
605 
606 #endif
data at payload
child of avmedia: audio
AVMediaType media_type
For subtype stream_init.
Definition: frame.h:293
uint8_t * u_payload
shortcut to AVMediaFrame::av_frame->data[1]
Definition: frame.h:383
Definition: codec.h:71
signal to AVThread or OpenGLThread. Also custom signals to custom Threads
Custom payload Frame.
Definition: frame.h:160
A GPU YUV frame.
Definition: frame.h:485
FrameClass frameclass
Declares frametype for correct typecast. Used by Frame::getFrameClass()
Definition: frame.h:137
Encapsulates data sent by the signal.
Definition: threadsignal.h:130
Definition: rawrite.h:96
data at ffmpeg avpkt
data at ffmpeg av_frame and ffmpeg av_codec_context
Constant/default values, version numbers.
Definition: rawrite.h:67
For AVBitmapFrames, linesizes are the widths + padding bytes.
Definition: constant.h:129
std::vector< uint8_t > payload
Raw payload data (use .data() to get the pointer from std::vector)
Definition: frame.h:187
virtual void print(std::ostream &os) const
Produces frame output.
Definition: frame.cpp:68
Codec definitions (slightly outdated)
OpenGL calls for reserving PBOs and TEXtures, plus some auxiliary routines.
TimeCorrectionType
Methods to correct frame timestamps.
Definition: frame.h:83
undefined (initial value)
Definition: usbthread.h:143
GLuint y_index
internal OpenGL/GPU index
Definition: frame.h:511
Redefinition of characteristic signal contexts (info that goes with the signal) for AVThread thread...
Definition: threadsignal.h:78
AVCodecID codec_id
helper object: codec id
Definition: frame.h:336
Decoded Frame in FFmpeg format.
Definition: frame.h:308
OpenGLSignalContext opengl_signal_ctx
Thread commands to OpenGLThread.
Definition: frame.h:580
bool fs_end
Filesystem start / end // this controlled better at the python level.
Definition: frame.h:602
AVPixelFormat av_pixel_format
From AVCodecContext .. this class implies YUV420P so this is not really required ..
Definition: frame.h:380
GLubyte * v_payload
direct memory access memory address, returned by GPU
Definition: frame.h:518
std::vector< uint8_t > payload
RGB24 data as continuous bytes. 3 bytes per pixel.
Definition: frame.h:544
H264Pars h264_pars
H264 parameters, extracted from the payload.
Definition: frame.h:201
GLubyte * y_payload
direct memory access memory address, returned by GPU
Definition: frame.h:516
Encapsulate data sent in the ValkkaFSReaderSignal.
Definition: threadsignal.h:196
bool tm_end
Transmission start / end.
Definition: frame.h:603
AVMediaType media_type
Type of the media (video/audio)
Definition: frame.h:188
AVFrame * av_frame
The decoded frame.
Definition: frame.h:339
GLuint v_index
internal OpenGL/GPU index
Definition: frame.h:513
FrameClass
Enumeration of Frame classes used by Valkka.
Definition: frame.h:51
our own RGB24 data structure
GLuint u_index
internal OpenGL/GPU index
Definition: frame.h:512
BitmapPars bmpars
Calculated bitmap plane dimensions, data sizes, etc.
Definition: frame.h:381
SetupFrameType
A muxed packet (in some container format)
Definition: frame.h:251
Definition: frame.h:587
AVMediaType media_type
helper object: media type
Definition: frame.h:335
uint8_t * v_payload
shortcut to AVMediaFrame::av_frame->data[2]
Definition: frame.h:384
Used when sending blocks of frames: mark filesystem and block start and end.
List of common ffmpeg/libav header files. Definition of some functions to call FFmpeg API directly fr...
AVCodecID codec_id
AVCodeCID of the media.
Definition: frame.h:189
Decoded YUV/RGB frame in FFMpeg format.
Definition: frame.h:354
child of avmedia: video
Encapsulate data sent in the ValkkaFSWriterSignal.
Definition: threadsignal.h:163
Setup frame.
Definition: frame.h:270
A frame signaling internal thread commands, states of recorded video, etc.
Definition: frame.h:570
Decoded YUV frame in a non-planar format (thus "NP")
Definition: frame.h:430
Write directly to files and devices with POSIX O_DIRECT.
void * custom_signal_ctx
For extensions: thread commands for any thread. TODO: migrate all signal contexes here...
Definition: frame.h:584
Frame: An abstract queueable class.
Definition: frame.h:108
GLubyte * u_payload
direct memory access memory address, returned by GPU
Definition: frame.h:517
List of common header files.
Auxiliary routines.
int subsession_index
Media subsession index.
Definition: frame.h:141
AVCodecID codec_id
For subtype stream_init.
Definition: frame.h:294
rgb interpolated from yuv
ValkkaFSWriterSignalContext valkkafswriter_signal_ctx
Thread commands to ValkkFSWriterThread.
Definition: frame.h:582
uint8_t * y_payload
shortcut to AVMediaFrame::av_frame->data[0]
Definition: frame.h:382
AbstractFileState stream_state
For subtype stream_state.
Definition: frame.h:296
AbstractFileState
Describes the state of a stream.
Definition: threadsignal.h:43
ValkkaFSReaderSignalContext valkkafsreader_signal_ctx
Thread commands to ValkkaFSReaderThread.
Definition: frame.h:583
long int mstimestamp
Presentation time stamp (PTS) in milliseconds.
Definition: frame.h:142
SetupFrameType sub_type
Type of the SetupFrame.
Definition: frame.h:291
data at the GPU
AVSignalContext av_signal_ctx
Thread commands to AVThread.
Definition: frame.h:581
Our own RGB24 structure.
Definition: frame.h:535
SlotNumber n_slot
Slot number identifying the media source.
Definition: frame.h:140