8000 Initial support for concatenated archives, and memory fixes by jhol · Pull Request #88 · USCiLab/cereal · GitHub
[go: up one dir, main page]
More Web Proxy on the site http://driver.im/
Skip to content

Initial support for concatenated archives, and memory fixes #88

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 9 commits into
base: master
Choose a base branch
from
Open

Initial support for concatenated archives, and memory fixes #88

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
179d839
XMLOutputArchive: Do not take c_str from a temporary string
jhol Mar 28, 2014
5ef2e50
Made XMLOutputArchive::NodeInfo::name a std::string
jho 8000 l Mar 31, 2014
a5d52c7
Made XMLInputArchive::NodeInfo::name a std::string
jhol Mar 31, 2014
3000a17
Made JSONOutputArchive::itsNextName a std::string
jhol Mar 31, 2014
b7f8480
Made JSONInputArchive::itsNextName a std::string
jhol Mar 31, 2014
c8b5ee5
XMLInputArchive: Removed commented code
jhol Mar 31, 2014
7853bea
JSONInputArchive: Corrected a typo
jhol Mar 31, 2014
1298bb2
Added the concatenation unit test
jhol Mar 31, 2014
0c7d5ad
XMLInputArchive: Support concatenated documents
jhol Mar 31, 2014
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 16 additions & 17 deletions include/cereal/archives/json.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -143,7 +143,7 @@ namespace cereal
OutputArchive<JSONOutputArchive>(this),
itsWriteStream(stream),
itsWriter(itsWriteStream, op 10000 tions.itsPrecision),
itsNextName(nullptr)
itsNextName()
{
itsWriter.SetIndent( options.itsIndentChar, options.itsIndentLength );
itsNameCounter.push(0);
Expand Down Expand Up @@ -215,7 +215,7 @@ namespace cereal
//! Sets the name for the next node created with startNode
void setNextName( const char * name )
{
itsNextName = name;
itsNextName = name ? std::string(name) : std::string();
}

//! Saves a bool to the current node
Expand Down Expand Up @@ -327,15 +327,15 @@ namespace cereal
// Array types do not output names
if(nodeType == NodeType::InArray) return;

if(itsNextName == nullptr)
if(itsNextName.empty())
{
std::string name = "value" + std::to_string( itsNameCounter.top()++ ) + "\0";
saveValue(name);
}
else
{
saveValue(itsNextName);
itsNextName = nullptr;
itsNextName.clear();
}
}

Expand All @@ -350,7 +350,7 @@ namespace cereal
private:
WriteStream itsWriteStream; //!< Rapidjson write stream
JSONWriter itsWriter; //!< Rapidjson writer
char const * itsNextName; //!< The next name
std::string itsNextName; //!< The next name
std::stack<uint32_t> itsNameCounter; //!< Counter for creating unique names for unnamed nodes
std::stack<NodeType> itsNodeStack;
}; // JSONOutputArchive
Expand Down Expand Up @@ -408,7 +408,7 @@ namespace cereal
/*! @param stream The stream to read from */
JSONInputArchive(std::istream & stream) :
InputArchive<JSONInputArchive>(this),
itsNextName( nullptr ),
itsNextName(),
itsReadStream(stream)
{
itsDocument.ParseStream<0>(itsReadStream);
Expand All @@ -423,7 +423,7 @@ namespace cereal
to loading in/out of order */
void loadBinaryValue( void * data, size_t size, const char * name = nullptr )
{
itsNextName = name;
itsNextName = name ? std::string(name) : std::string();

std::string encoded;
loadValue( encoded );
Expand All @@ -433,7 +433,7 @@ namespace cereal
throw Exception("Decoded binary data size does not match specified size");

std::memcpy( data, decoded.data(), decoded.size() );
itsNextName = nullptr;
itsNextName.clear();
};

private:
Expand Down Expand Up @@ -488,12 +488,11 @@ namespace cereal

//! Adjust our position such that we are at the node with the given name
/*! @throws Exception if no such named node exists */
inline void search( const char * searchName )
inline void search( const std::string &searchName )
{
const auto len = std::strlen( searchName );
size_t index = 0;
for( auto it = itsMemberItBegin; it != itsMemberItEnd; ++it, ++index )
if( std::strncmp( searchName, it->name.GetString(), len ) == 0 )
if( searchName == it->name.GetString() )
{
itsIndex = index;
return;
Expand Down Expand Up @@ -521,17 +520,17 @@ namespace cereal
inline void search()
{
// The name an NVP provided with setNextName()
if( itsNextName )
if( !itsNextName.empty() )
{
// The actual name of the current node
auto const actualName = itsIteratorStack.back().name();

// Do a search if we don't see a name coming up, or if the names don't match
if( !actualName || std::strcmp( itsNextName, actualName ) != 0 )
if( !actualName || itsNextName != actualName )
itsIteratorStack.back().search( itsNextName );
}

itsNextName = nullptr;
itsNextName.clear();
}

public:
Expand Down Expand Up @@ -565,7 +564,7 @@ namespace cereal
//! Sets the name for the next node created with startNode
void setNextName( const char * name )
{
itsNextName = name;
itsNextName = name ? std::string(name) : std::string();
}

//! Loads a value from the current node - small signed overload
Expand Down Expand Up @@ -637,8 +636,8 @@ namespace cereal
//! @}

private:
const char * itsNextName; //!< Next name set by NVP
ReadStream itsReadStream; //!< Rapidjson write stream
std::string itsNextName; //!< Next name set by NVP
ReadStream itsReadStream; //!< Rapidjson read stream
std::vector<Iterator> itsIteratorStack; //!< 'Stack' of rapidJSON iterators
rapidjson::Document itsDocument; //!< Rapidjson document
};
Expand Down
79 changes: 49 additions & 30 deletions include/cereal/archives/xml.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -227,7 +227,8 @@ namespace cereal
itsOS << value << std::ends;

// allocate strings for all of the data in the XML object
auto dataPtr = itsXML.allocate_string( itsOS.str().c_str() );
const std::string s = itsOS.str();
auto dataPtr = itsXML.allocate_string( s.c_str() );

// insert into the XML
itsNodes.top().node->append_node( itsXML.allocate_node( rapidxml::node_data, nullptr, dataPtr ) );
Expand Down Expand Up @@ -277,23 +278,23 @@ namespace cereal
const char * nm = nullptr ) :
node( n ),
counter( 0 ),
name( nm )
name( nm ? std::string(nm) : std::string() )
{ }

rapidxml::xml_node<> * node; //!< A pointer to this node
size_t counter; //!< The counter for naming child nodes
const char * name; //!< The name for the next child node
std::string name; //!< The name for the next child node

//! Gets the name for the next child node created from this node
/*! The name will be automatically generated using the counter if
a name has not been previously set. If a name has been previously
set, that name will be returned only once */
std::string getValueName()
{
if( name )
if( !name.empty() )
{
auto n = name;
name = nullptr;
name = std::string();
return {n};
}
else
Expand Down Expand Up @@ -361,25 +362,42 @@ namespace cereal

@param stream The stream to read from. Can be a stringstream or a file. */
XMLInputArchive( std::istream & stream ) :
InputArchive<XMLInputArchive>( this ),
itsData( std::istreambuf_iterator<char>( stream ), std::istreambuf_iterator<char>() )
InputArchive<XMLInputArchive>( this )
{
try
bool valid = false;
std::string line, document;

while (!valid && stream)
{
itsData.push_back('\0'); // rapidxml will do terrible things without the data being null terminated
itsXML.parse<rapidxml::parse_no_data_nodes | rapidxml::parse_declaration_node>( reinterpret_cast<char *>( itsData.data() ) );
// Read until we get an empty line - this may indicate that the document has ended
while (getline(stream, line))
{
document += line + '\n';
if (line.empty())
break;
}

// Set this as the document data
// Make a fresh copy every time, because RapidXml parsing is destructive
itsData.clear();
itsData.reserve(document.length() + 1);
itsData.insert(itsData.begin(), document.cbegin(), document.cend());
itsData.push_back('\0');

// If the line is empty, this may indicate the document has ended, if so try and parse it
try
{
itsXML.parse<rapidxml::parse_no_data_nodes | rapidxml::parse_declaration_node>(
reinterpret_cast<char *>(itsData.data()) );
valid = true;
}
catch( rapidxml::parse_error const & )
{
}
}
catch( rapidxml::parse_error const & )
{
//std::cerr << "-----Original-----" << std::endl;
//stream.seekg(0);
//std::cout << std::string( std::istreambuf_iterator<char>( stream ), std::istreambuf_iterator<char>() ) << std::endl;

//std::cerr << "-----Error-----" << std::endl;
//std::cerr << e.what() << std::endl;
//std::cerr << e.where<char>() << std::endl;
if (!valid)
throw Exception("XML Parsing failed - likely due to invalid characters or invalid naming");
}

// Parse the root
auto root = itsXML.first_node( xml_detail::CEREAL_XML_STRING );
Expand Down Expand Up @@ -432,12 +450,12 @@ namespace cereal
void startNode()
{
auto next = itsNodes.top().child; // By default we would move to the next child node
auto const expectedName = itsNodes.top().name; // this is the expected name from the NVP, if provided
const std::string &expectedName = itsNodes.top().name; // this is the expected name from the NVP, if provided

// If we were given an NVP name, look for it in the current level of the document.
// We only need to do this if either we have exhausted the siblings of the current level or
// the NVP name does not match the name of the node we would normally read next
if( expectedName && ( next == nullptr || std::strcmp( next->name(), expectedName ) != 0 ) )
if( !expectedName.empty() && ( next == nullptr || expectedName != next->name() ) )
{
F438 next = itsNodes.top().search( expectedName );

Expand All @@ -458,13 +476,13 @@ namespace cereal
itsNodes.top().advance();

// Reset name
itsNodes.top().name = nullptr;
itsNodes.top().name.clear();
}

//! Sets the name for the next node created with startNode
void setNextName( const char * name )
{
itsNodes.top().name = name;
itsNodes.top().name = name ? std::string(name) : std::string();
}

//! Loads a bool from the current top node
Expand Down Expand Up @@ -610,7 +628,7 @@ namespace cereal
node( n ),
child( n->first_node() ),
size( XMLInputArchive::getNumChildren( n ) ),
name( nullptr )
name()
{ }

//! Advances to the next sibling node of the child
Expand All @@ -625,18 +643,19 @@ namespace cereal
}

//! Searches for a child with the given name in this node
/*! @param searchName The name to search for (must be null terminated)
/*! @param searchName The name to search for
@return The node if found, nullptr otherwise */
rapidxml::xml_node<> * search( const char * searchName )
rapidxml::xml_node<> * search( const std::string &searchName )
{
if( searchName )
if( !searchName.empty() )
{
size_t new_size = XMLInputArchive::getNumChildren( node );
const size_t name_size = rapidxml::internal::measure( searchName );
const size_t name_size = rapidxml::internal::measure( searchName.c_str() );

for( auto new_child = node->first_node(); new_child != nullptr; new_child = new_child->next_sibling() )
{
if( rapidxml::internal::compare( new_child->name(), new_child->name_size(), searchName, name_size, true ) )
if( rapidxml::internal::compare( new_child->name(), new_child->name_size(),
searchName.c_str(), name_size, true ) )
{
size = new_size;
child = new_child;
Expand All @@ -653,7 +672,7 @@ namespace cereal
rapidxml::xml_node<> * node; //!< A pointer to this node
rapidxml::xml_node<> * child; //!< A pointer to its current child
size_t size; //!< The remaining number of children for this node
const char * name; //!< The NVP name for next next child node
std::string name; //!< The NVP name for next next child node
}; // NodeInfo

//! @}
Expand Down
87 changes: 87 additions & 0 deletions unittests/concatenation.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,87 @@
/*
Copyright (c) 2014, Randolph Voorhies, Shane Grant, Joel Holdsworth
All rights reserved.

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
* Neither the name of cereal nor the
names of its contributors may be used to endorse or promote products
derived from this software without specific prior written permission.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL RANDOLPH VOORHIES AND SHANE GRANT BE LIABLE FOR ANY
DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#include "common.hpp"
#include <boost/test/unit_test.hpp>

template <class IArchive, class OArchive>
void test_concatenation()
{
// Strings with newlines are being used to check that input archives to not
// treat them incorrectly as document terminators
std::basic_string<char> o_string = "Apples\n\nPears";
std::basic_string<char> o_string2 = "Oranges\n\nLemons";

// Create a stream containing to concatenated archives
std::ostringstream os;
{
OArchive oar(os);
oar(o_string);
}
{
OArchive oar(os);
oar(o_string2);
}

std::basic_string<char> i_string;
std::basic_string<char> i_string2;

// Now try and read them back
std::istringstream is(os.str());
{
IArchive iar(is);
iar(i_string);
}
{
IArchive iar(is);
iar(i_string2);
}

BOOST_CHECK_EQUAL(i_string, o_string);
BOOST_CHECK_EQUAL(i_string2, o_string2);
}

BOOST_AUTO_TEST_CASE( binary_string )
{
test_concatenation<cereal::BinaryInputArchive, cereal::BinaryOutputArchive>();
}

BOOST_AUTO_TEST_CASE( portable_binary_string )
{
test_concatenation<cereal::PortableBinaryInputArchive, cereal::PortableBinaryOutputArchive>();
}

BOOST_AUTO_TEST_CASE( xml_string_basic )
{
test_concatenation<cereal::XMLInputArchive, cereal::XMLOutputArchive>();
}

#if 0
BOOST_AUTO_TEST_CASE( json_string_basic )
{
test_concatenation<cereal::JSONInputArchive, cereal::JSONOutputArchive>();
}
#endif
0