Valkka  0.17.0
OpenSource Video Management
valkkafs.h
Go to the documentation of this file.
1 #ifndef valkkafs_HEADER_GUARD
2 #define valkkafs_HEADER_GUARD
3 /*
4  * valkkafs.h :
5  *
6  * Copyright 2017, 2018 Valkka Security Ltd. and Sampsa Riikonen.
7  *
8  * Authors: Sampsa Riikonen <sampsa.riikonen@iki.fi>
9  *
10  * This file is part of the Valkka library.
11  *
12  * Valkka is free software: you can redistribute it and/or modify
13  * it under the terms of the GNU Affero General Public License as
14  * published by the Free Software Foundation, either version 3 of the
15  * License, or (at your option) any later version.
16  *
17  * This program is distributed in the hope that it will be useful,
18  * but WITHOUT ANY WARRANTY; without even the implied warranty of
19  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
20  * GNU Affero General Public License for more details.
21  *
22  * You should have received a copy of the GNU Affero General Public License
23  * along with this program. If not, see <https://www.gnu.org/licenses/>
24  *
25  */
26 
36 #include "common.h"
37 #include "thread.h"
38 #include "framefilter.h"
39 #include "framefifo.h"
40 #include "rawrite.h"
41 #include "logging.h"
42 #include "Python.h"
43 
44 /*
45 #include "boost/python/numpy.hpp"
46 namespace p = boost::python;
47 namespace np = boost::python::numpy;
48 // https://www.boost.org/doc/libs/1_64_0/libs/python/doc/html/numpy/reference/ndarray.html
49 // https://github.com/ndarray/Boost.NumPy/blob/master/libs/numpy/example/simple.cpp
50 // let's not use boost..
51 */
52 
78 class ValkkaFS { // <pyapi>
79 
80 public: // <pyapi>
81 
90  ValkkaFS(const char *device_file, const char *block_file, std::size_t blocksize, std::size_t n_blocks, bool init=false); // <pyapi>
91  ~ValkkaFS(); // <pyapi>
92 
93 protected:
94  std::string device_file;
95  std::string block_file;
96  std::size_t blocksize;
97  std::size_t n_blocks;
98  bool init;
99  std::size_t device_size;
100  std::vector<long int> tab;
101  std::fstream os;
102 
103  std::mutex mutex;
104  long int col_0;
105  long int col_1;
106  std::size_t current_row;
107  std::size_t prev_row;
108  PyObject *pyfunc;
109 
110 protected:
111  const static std::size_t n_cols = 2;
112 
113 protected:
114  std::size_t ind(std::size_t i, std::size_t j);
115 
116 public:
117  void setVal(std::size_t i, std::size_t j, long int val);
118  long int getVal(std::size_t i, std::size_t j);
119  std::size_t getBlockSeek(std::size_t n_block);
120  std::size_t getCurrentBlockSeek();
121 
122 public: // getters
123  const std::size_t getBlockSize() {return this->blocksize;} // <pyapi>
124 
125 public: // <pyapi>
126  std::size_t get_n_blocks(); // <pyapi>
127  std::size_t get_n_cols(); // <pyapi>
128 
130  void dumpTable_();
132  void updateDumpTable_(std::size_t n_block);
133 
135  void dumpTable(); // <pyapi>
137  void readTable(); // <pyapi>
139  std::string getDevice(); // <pyapi>
141  std::size_t getDeviceSize(); // <pyapi>
143  void clearDevice(bool writethrough=false, bool verbose=false); // <pyapi>
145  void clearTable(); // <pyapi>
147  std::size_t maxFrameSize(); // <pyapi>
149  void reportTable(std::size_t from=0, std::size_t to=0, bool show_all=false); // <pyapi>
155  void writeBlock(bool pycall=true, bool use_gil=true); // <pyapi>
156 
158  void markFrame(long int mstimestamp); // <pyapi>
159 
161  void markKeyFrame(long int mstimestamp); // <pyapi>
162 
164  void setCurrentBlock(std::size_t n_block); // <pyapi>
165 
167  void setBlockCallback(PyObject* pobj); // <pyapi>
168 
178  void setArrayCall(PyObject *pyobj); // <pyapi>
179 }; // <pyapi>
180 
181 
187 class ValkkaFSTool { // <pyapi>
188 
189 public: // <pyapi>
190  ValkkaFSTool(ValkkaFS &valkkafs); // <pyapi>
191  ~ValkkaFSTool(); // <pyapi>
192 
193 protected:
194  // std::fstream is;
195  RawReader raw_reader;
196  ValkkaFS &valkkafs;
197 
198 public: // <pyapi>
199  void dumpBlock(std::size_t n_block); // <pyapi>
200 }; // <pyapi>
201 
202 
203 
211 class ValkkaFSWriterThread : public Thread { // <pyapi>
212 
213 public: // <pyapi>
214  ValkkaFSWriterThread(const char *name, ValkkaFS &valkkafs, FrameFifoContext fifo_ctx=FrameFifoContext(), bool o_direct = false); // <pyapi>
215  ~ValkkaFSWriterThread(); // <pyapi>
216 
217 protected:
218  ValkkaFS &valkkafs;
219  // std::fstream filestream;
220  RaWriter raw_writer;
221  std::map<SlotNumber, IdNumber> slot_to_id;
222  std::size_t bytecount;
223 
224 protected: // frame input
228 
229 protected: // Thread member redefinitions
230  std::deque<ValkkaFSWriterSignalContext> signal_fifo;
231 
232 public: // redefined virtual functions
233  void run();
234  void preRun();
235  void postRun();
236  void preJoin();
237  void postJoin();
238  void sendSignal(ValkkaFSWriterSignalContext signal_ctx);
239 
240 protected:
241  void handleSignal(ValkkaFSWriterSignalContext &signal_ctx);
242  void handleSignals();
243 
244 protected:
245  void saveCurrentBlock(bool pycall=true, bool use_gil=true);
246  void setSlotId(SlotNumber slot, IdNumber id);
247  void unSetSlotId(SlotNumber slot);
248  void clearSlotId();
249  void reportSlotId();
250  void seek(std::size_t n_block);
251 
252 // API
253 public: // <pyapi>
254  FifoFrameFilter &getFrameFilter(); // <pyapi>
255  FifoFrameFilter &getBlockingFrameFilter(); // <pyapi>
257  void setSlotIdCall(SlotNumber slot, IdNumber id); // <pyapi>
259  void unSetSlotIdCall(SlotNumber slot); // <pyapi>
261  void clearSlotIdCall(); // <pyapi>
263  void reportSlotIdCall(); // <pyapi>
265  void seekCall(std::size_t n_block); // <pyapi>
266  void requestStopCall(); // <pyapi>
267 }; // <pyapi>
268 
269 
270 #endif
void setCurrentBlock(std::size_t n_block)
Set block number that&#39;s being written.
Definition: valkkafs.cpp:315
FrameFifo infifo
Incoming frames are read from here.
Definition: valkkafs.h:225
std::deque< ValkkaFSWriterSignalContext > signal_fifo
Redefinition of signal fifo.
Definition: valkkafs.h:230
Writes frames to ValkkaFS.
Definition: valkkafs.h:211
void reportTable(std::size_t from=0, std::size_t to=0, bool show_all=false)
print blocktable
Definition: valkkafs.cpp:150
void dumpTable()
dump blocktable to disk
Definition: valkkafs.cpp:94
void readTable()
read blocktable from disk
Definition: valkkafs.cpp:107
FifoFrameFilter infilter
Write incoming frames here // TODO: add a chain of correcting FrameFilter(s)
Definition: valkkafs.h:226
Describes the stack structure and fifo behaviour for a FrameFifo.
Definition: framefifo.h:45
long int col_0
Current column 0 value (max keyframe timestamp)
Definition: valkkafs.h:104
void updateDumpTable_(std::size_t n_block)
dump single row of bloctable to disk.
Definition: valkkafs.cpp:99
Definition: rawrite.h:96
Thread safe system of fifo and a stack.
PyObject * pyfunc
Python callback that&#39;s triggered at block write.
Definition: valkkafs.h:108
Definition: rawrite.h:67
std::fstream os
Write handle to blocktable file.
Definition: valkkafs.h:101
A thread-safe combination of a fifo (first-in-first-out) queue and an associated stack.
Definition: framefifo.h:72
void setVal(std::size_t i, std::size_t j, long int val)
set tab&#39;s value at block i, row j
Definition: valkkafs.cpp:125
std::size_t getDeviceSize()
returns device file size
Definition: valkkafs.cpp:273
Logging utilities.
long int col_1
Current column 1 value (max anyframe timestamp)
Definition: valkkafs.h:105
void markKeyFrame(long int mstimestamp)
Used by a writer class to inform that a key frame has been written.
Definition: valkkafs.cpp:262
Book-keeping for ValkkaFS.
Definition: valkkafs.h:78
Base class for multithreading.
Passes frames to a multiprocessing fifo.
Definition: framefilter.h:585
std::map< SlotNumber, IdNumber > slot_to_id
Map from slot numbers to ids.
Definition: valkkafs.h:221
Analyzer tool for ValkkaFS.
Definition: valkkafs.h:187
void setArrayCall(PyObject *pyobj)
Copy blocktable to a given numpy array.
Definition: valkkafs.cpp:340
void writeBlock(bool pycall=true, bool use_gil=true)
Used by a writer class to inform that a new block has been written.
Definition: valkkafs.cpp:173
void clearTable()
clears the blocktable and writes it to the disk
Definition: valkkafs.cpp:305
std::string getDevice()
returns device filename
Definition: valkkafs.cpp:269
void markFrame(long int mstimestamp)
Used by a writer class to inform that a non-key frame has been written.
Definition: valkkafs.cpp:257
BlockingFifoFrameFilter infilter_block
Incoming frames can also be written here. If stack runs out of frames, writing will block...
Definition: valkkafs.h:227
bool init
Clear the blocktable or not even if it exists.
Definition: valkkafs.h:98
Definition of FrameFilter and derived classes for various purposes.
void setBlockCallback(PyObject *pobj)
Set a python callable that&#39;s being triggered when a new block is written.
Definition: valkkafs.cpp:321
ValkkaFS(const char *device_file, const char *block_file, std::size_t blocksize, std::size_t n_blocks, bool init=false)
Default Constructor.
Definition: valkkafs.cpp:37
Encapsulate data sent in the ValkkaFSWriterSignal.
Definition: threadsignal.h:163
std::size_t ind(std::size_t i, std::size_t j)
first index: block number (row), second index: column
Definition: valkkafs.cpp:115
A class for multithreading with a signaling system.
Definition: thread.h:90
Write directly to files and devices with POSIX O_DIRECT.
Passes frames to a FrameFifo.
Definition: framefilter.h:560
List of common header files.
std::vector< long int > tab
Blocktable.
Definition: valkkafs.h:100
std::size_t prev_row
Previous row number (block)
Definition: valkkafs.h:107
void clearDevice(bool writethrough=false, bool verbose=false)
writes zero bytes to the device
Definition: valkkafs.cpp:277
std::size_t maxFrameSize()
returns maximum allowed frame size in bytes
Definition: valkkafs.cpp:145
std::size_t current_row
Row number (block) that&#39;s being written.
Definition: valkkafs.h:106
void dumpTable_()
dump blocktable to disk.
Definition: valkkafs.cpp:86