Class that provides binary table I/O. More...

#include <BufferedColumnarFile.h>

Public Member Functions
	VarBufferedColumnarFile () noexcept=default
	VarBufferedColumnarFile (const VarBufferedColumnarFile &rhs)=delete
	VarBufferedColumnarFile (VarBufferedColumnarFile &&rhs) noexcept
	~VarBufferedColumnarFile () noexcept
VarBufferedColumnarFile &	operator= (VarBufferedColumnarFile &&rhs)
void	Close ()
	Close the file.
void	Read (uint64_t row, uint64_t column_offset, float *data, uint64_t n)
	Read one cell containing an array of floats.
void	Read (uint64_t row, uint64_t column_offset, double *data, uint64_t n)
	Read array of doubles.
void	Read (uint64_t row, uint64_t column_offset, int32_t *data, uint64_t n)
	Read array of int32_t.
void	Read (uint64_t row, uint64_t column_offset, std::complex< float > *data, uint64_t n)
	Read array of complex floats.
void	Read (uint64_t row, uint64_t column_offset, bool *data, uint64_t n)
	Read an array of bools.
void	Write (uint64_t row, uint64_t column_offset, const float *data, uint64_t n)
	Write one cell containing an array of floats.
void	Write (uint64_t row, uint64_t column_offset, const double *data, uint64_t n)
	Write an array of doubles.
void	Write (uint64_t row, uint64_t column_offset, const int32_t *data, uint64_t n)
	Write an array of int32_t.
void	Write (uint64_t row, uint64_t column_offset, const std::complex< float > *data, uint64_t n)
	Write an array of complex floats.
void	Write (uint64_t row, uint64_t column_offset, const std::complex< double > *data, uint64_t n)
	Write an array of complex doubles.
void	Write (uint64_t row, uint64_t column_offset, const bool *data, uint64_t n)
	Write an array of bools.
void	SetStride (uint64_t new_stride)
	Set the number of bytes per row for this file.
void	AddRows (uint64_t n_rows)
	Adds a given number of rows to the back of the file.
void	DeleteRow ()
	Deletes the last row.
const std::string &	Filename () const
bool	IsOpen () const
uint64_t	NRows () const
	Total number of rows stored in this file.
void	ReadHeader (unsigned char *data)
	Read an optional extra header to the file.
uint64_t	Stride () const
	Total number of bytes in one row.
void	WriteHeader (const unsigned char *data)
	Write an optional extra header to the file.

Static Public Member Functions
static VarBufferedColumnarFile	CreateNew (const std::string &filename, uint64_t header_size, uint64_t stride)
	Create a new file on disk.
static VarBufferedColumnarFile	OpenExisting (const std::string &filename, size_t header_size)
	Open an existing file from disk.

Private Member Functions
	VarBufferedColumnarFile (const std::string &filename, uint64_t header_size, uint64_t stride)
	Create or overwrite a new columnar file on disk.
	VarBufferedColumnarFile (const std::string &filename, size_t header_size)
	Open an existing columnar file.
void	ActivateBlock (uint64_t row)
template<typename ValueType>
void	ReadImplementation (uint64_t row, uint64_t column_offset, ValueType *data, uint64_t n)
template<typename ValueType>
void	WriteImplementation (uint64_t row, uint64_t column_offset, const ValueType *data, uint64_t n)
void	WriteActiveBlock ()
Private Member Functions inherited from casacore::RowBasedFile
	RowBasedFile ()=default
	RowBasedFile (const RowBasedFile &rhs)=delete
	RowBasedFile (RowBasedFile &&rhs) noexcept
	RowBasedFile (const std::string &filename, uint64_t header_size, uint64_t stride)
	Create or overwrite a new columnar file on disk.
	RowBasedFile (const std::string &filename, size_t header_size)
	Open an existing columnar file.
	~RowBasedFile () noexcept
RowBasedFile &	operator= (RowBasedFile &&rhs)
void	Close ()
	Close the file.
void	Truncate (uint64_t n_rows)
void	Seek (off_t pos, int seek_direction)
void	ReadData (unsigned char *data, uint64_t size)
void	WriteData (const unsigned char *data, uint64_t size)
bool	IsOpen () const
uint64_t	DataLocation () const
	Offset of the first row in the file.
const std::string &	Filename () const
uint64_t	HeaderSize () const
	Number of bytes reserved for an optional header.
void	WriteHeader (const unsigned char *data)
	Write an optional extra header to the file.
void	ReadHeader (unsigned char *data)
	Read an optional extra header to the file.
uint64_t	NRows () const
	Total number of rows stored in this file.
void	SetNRows (uint64_t new_n_rows)
uint64_t	Stride () const
	Total number of bytes in one row.
void	SetStride (uint64_t new_stride)
	Set the number of bytes per row for this file.
void	AddRows (uint64_t n_rows)
	Adds a given number of rows to the back of the file.
void	DeleteRow ()
	Deletes the last row.

Private Attributes
std::vector< unsigned char >	packed_buffer_
	This buffer is used temporarily for (un)packing booleans.
bool	block_changed_
uint64_t	active_block_
uint64_t	rows_per_block_
std::vector< unsigned char >	block_buffer_

Detailed Description

template<uint64_t BufferSize = 100 * 1024>
class casacore::VarBufferedColumnarFile< BufferSize >

Class that provides binary table I/O.

It is rather low-level, and requires the user to keep track of the columns and their datatype. It uses a simple binary format where the columns are interleaved into the output file. Booleans are written with bit-packing.

This class limits calls to the read/write functions by caching a block of data around the last accessed row. The size of the buffer is templated to make testing of the caching easier. Production code can normally use the alias BufferedColumnarFile, which uses a reasonable buffer size.

This class aims to be as fast as possible for large data files that are read from or written to consecutively. It does not try to optimize random access, partial data access and access is not transactional.

The class uses exceptions to handle any I/O errors.

Definition at line 33 of file BufferedColumnarFile.h.

Constructor & Destructor Documentation

◆ VarBufferedColumnarFile() [1/5]

template<uint64_t BufferSize = 100 * 1024>

casacore::VarBufferedColumnarFile< BufferSize >::VarBufferedColumnarFile ( )

defaultnoexcept

Referenced by casacore::VarBufferedColumnarFile< 100 *1024 >::CreateNew(), and casacore::VarBufferedColumnarFile< 100 *1024 >::OpenExisting().

◆ VarBufferedColumnarFile() [2/5]

template<uint64_t BufferSize = 100 * 1024>

casacore::VarBufferedColumnarFile< BufferSize >::VarBufferedColumnarFile ( const VarBufferedColumnarFile< BufferSize > & rhs )

delete

◆ VarBufferedColumnarFile() [3/5]

template<uint64_t BufferSize = 100 * 1024>

casacore::VarBufferedColumnarFile< BufferSize >::VarBufferedColumnarFile ( VarBufferedColumnarFile< BufferSize > && rhs )

inlinenoexcept

Definition at line 47 of file BufferedColumnarFile.h.

◆ ~VarBufferedColumnarFile()

template<uint64_t BufferSize = 100 * 1024>

casacore::VarBufferedColumnarFile< BufferSize >::~VarBufferedColumnarFile ( )

inlinenoexcept

Definition at line 58 of file BufferedColumnarFile.h.

◆ VarBufferedColumnarFile() [4/5]

template<uint64_t BufferSize = 100 * 1024>

casacore::VarBufferedColumnarFile< BufferSize >::VarBufferedColumnarFile	(	const std::string &	filename,
		uint64_t	header_size,
		uint64_t	stride )

inlineprivate

Create or overwrite a new columnar file on disk.

Definition at line 248 of file BufferedColumnarFile.h.

◆ VarBufferedColumnarFile() [5/5]

template<uint64_t BufferSize = 100 * 1024>

casacore::VarBufferedColumnarFile< BufferSize >::VarBufferedColumnarFile	(	const std::string &	filename,
		size_t	header_size )

inlineprivate

Open an existing columnar file.

Definition at line 257 of file BufferedColumnarFile.h.

Member Function Documentation

◆ ActivateBlock()

template<uint64_t BufferSize = 100 * 1024>

void casacore::VarBufferedColumnarFile< BufferSize >::ActivateBlock ( uint64_t row )

inlineprivate

Fill the remainder of block_buffer_ with zeroes. Doing it here makes the code robust and avoids the need for inserting zeroes when adding rows out-of-order, e.g., when adding row 5 while NRows() is 2."

Definition at line 267 of file BufferedColumnarFile.h.

Referenced by casacore::VarBufferedColumnarFile< 100 *1024 >::Read(), casacore::VarBufferedColumnarFile< 100 *1024 >::ReadImplementation(), casacore::VarBufferedColumnarFile< 100 *1024 >::Write(), and casacore::VarBufferedColumnarFile< 100 *1024 >::WriteImplementation().

◆ AddRows()

template<uint64_t BufferSize = 100 * 1024>

void casacore::RowBasedFile::AddRows ( uint64_t n_rows )

inline

Adds a given number of rows to the back of the file.

Definition at line 232 of file RowBasedFile.h.

◆ Close()

template<uint64_t BufferSize = 100 * 1024>

void casacore::VarBufferedColumnarFile< BufferSize >::Close ( )

inline

Close the file.

After closing, all calls to I/O functions cause undefined behaviour, untill the class is assigned to a new instance.

Even if writing fails, the file should still be closed to prevent the file to remain open after destruction.

Definition at line 87 of file BufferedColumnarFile.h.

Referenced by casacore::VarBufferedColumnarFile< 100 *1024 >::operator=().

◆ CreateNew()

template<uint64_t BufferSize = 100 * 1024>

VarBufferedColumnarFile casacore::VarBufferedColumnarFile< BufferSize >::CreateNew	(	const std::string &	filename,
		uint64_t	header_size,
		uint64_t	stride )

inlinestatic

Create a new file on disk.

If the file exists, it is overwritten.

Parameters

header	Optional header.
stride	The number of bytes in one row (total over all columns).

Definition at line 108 of file BufferedColumnarFile.h.

◆ DeleteRow()

template<uint64_t BufferSize = 100 * 1024>

void casacore::RowBasedFile::DeleteRow ( )

inline

Deletes the last row.

Definition at line 237 of file RowBasedFile.h.

◆ Filename()

template<uint64_t BufferSize = 100 * 1024>

const std::string & casacore::RowBasedFile::Filename ( ) const

inline

Definition at line 178 of file RowBasedFile.h.

◆ IsOpen()

template<uint64_t BufferSize = 100 * 1024>

bool casacore::RowBasedFile::IsOpen ( ) const

inline

Definition at line 172 of file RowBasedFile.h.

Referenced by casacore::VarBufferedColumnarFile< 100 *1024 >::Close(), and casacore::VarBufferedColumnarFile< 100 *1024 >::~VarBufferedColumnarFile().

◆ NRows()

template<uint64_t BufferSize = 100 * 1024>

uint64_t casacore::RowBasedFile::NRows ( ) const

inline

Total number of rows stored in this file.

Definition at line 203 of file RowBasedFile.h.

Referenced by casacore::VarBufferedColumnarFile< 100 *1024 >::ActivateBlock(), casacore::VarBufferedColumnarFile< 100 *1024 >::Read(), casacore::VarBufferedColumnarFile< 100 *1024 >::ReadImplementation(), casacore::VarBufferedColumnarFile< 100 *1024 >::Write(), casacore::VarBufferedColumnarFile< 100 *1024 >::WriteActiveBlock(), casacore::VarBufferedColumnarFile< 100 *1024 >::WriteImplementation(), and casacore::VarBufferedColumnarFile< 100 *1024 >::~VarBufferedColumnarFile().

◆ OpenExisting()

template<uint64_t BufferSize = 100 * 1024>

VarBufferedColumnarFile casacore::VarBufferedColumnarFile< BufferSize >::OpenExisting	(	const std::string &	filename,
		size_t	header_size )

inlinestatic

Open an existing file from disk.

If the file does not exists, an exception is thrown.

Parameters

header	Optional header.
stride	The number of bytes in one row (total over all columns).

Definition at line 120 of file BufferedColumnarFile.h.

◆ operator=()

template<uint64_t BufferSize = 100 * 1024>

VarBufferedColumnarFile & casacore::VarBufferedColumnarFile< BufferSize >::operator= ( VarBufferedColumnarFile< BufferSize > && rhs )

inline

Definition at line 71 of file BufferedColumnarFile.h.

◆ Read() [1/5]

template<uint64_t BufferSize = 100 * 1024>

void casacore::VarBufferedColumnarFile< BufferSize >::Read	(	uint64_t	row,
		uint64_t	column_offset,
		bool *	data,
		uint64_t	n )

inline

Read an array of bools.

See float version for documentation. Booleans are stored with bit-packing.

Definition at line 164 of file BufferedColumnarFile.h.

◆ Read() [2/5]

template<uint64_t BufferSize = 100 * 1024>

void casacore::VarBufferedColumnarFile< BufferSize >::Read	(	uint64_t	row,
		uint64_t	column_offset,
		double *	data,
		uint64_t	n )

inline

Read array of doubles.

See float version for documentation.

Definition at line 141 of file BufferedColumnarFile.h.

◆ Read() [3/5]

template<uint64_t BufferSize = 100 * 1024>

void casacore::VarBufferedColumnarFile< BufferSize >::Read	(	uint64_t	row,
		uint64_t	column_offset,
		float *	data,
		uint64_t	n )

inline

Read one cell containing an array of floats.

If the row was not written yet, zeros are returned.

Parameters

row	The cell's row index.
column_offset	The position of this column counted from the start of the row, in bytes.
data	Buffer in which the data will be stored.
n	Size of the column in number of elements (NOT in bytes!).

Definition at line 134 of file BufferedColumnarFile.h.

◆ Read() [4/5]

template<uint64_t BufferSize = 100 * 1024>

void casacore::VarBufferedColumnarFile< BufferSize >::Read	(	uint64_t	row,
		uint64_t	column_offset,
		int32_t *	data,
		uint64_t	n )

inline

Read array of int32_t.

See float version for documentation.

Definition at line 148 of file BufferedColumnarFile.h.

◆ Read() [5/5]

template<uint64_t BufferSize = 100 * 1024>

void casacore::VarBufferedColumnarFile< BufferSize >::Read	(	uint64_t	row,
		uint64_t	column_offset,
		std::complex< float > *	data,
		uint64_t	n )

inline

Read array of complex floats.

See float version for documentation.

Definition at line 155 of file BufferedColumnarFile.h.

◆ ReadHeader()

template<uint64_t BufferSize = 100 * 1024>

void casacore::RowBasedFile::ReadHeader ( unsigned char * data )

inline

Read an optional extra header to the file.

See also: WriteHeader().

Definition at line 196 of file RowBasedFile.h.

◆ ReadImplementation()

template<uint64_t BufferSize = 100 * 1024>

template<typename ValueType>

void casacore::VarBufferedColumnarFile< BufferSize >::ReadImplementation	(	uint64_t	row,
		uint64_t	column_offset,
		ValueType *	data,
		uint64_t	n )

inlineprivate

Definition at line 290 of file BufferedColumnarFile.h.

Referenced by casacore::VarBufferedColumnarFile< 100 *1024 >::Read(), casacore::VarBufferedColumnarFile< 100 *1024 >::Read(), casacore::VarBufferedColumnarFile< 100 *1024 >::Read(), and casacore::VarBufferedColumnarFile< 100 *1024 >::Read().

◆ SetStride()

template<uint64_t BufferSize = 100 * 1024>

void casacore::VarBufferedColumnarFile< BufferSize >::SetStride ( uint64_t new_stride )

inline

Set the number of bytes per row for this file.

This changes the format of the file, and because of this the file is emptied.

Definition at line 236 of file BufferedColumnarFile.h.

◆ Stride()

template<uint64_t BufferSize = 100 * 1024>

uint64_t casacore::RowBasedFile::Stride ( ) const

inline

Total number of bytes in one row.

This value is also stored in the file, and is read from the file in OpenExisting().

Definition at line 214 of file RowBasedFile.h.

◆ Write() [1/6]

template<uint64_t BufferSize = 100 * 1024>

void casacore::VarBufferedColumnarFile< BufferSize >::Write	(	uint64_t	row,
		uint64_t	column_offset,
		const bool *	data,
		uint64_t	n )

inline

Write an array of bools.

Bools are stored with bit-packing. See float version for documentation.

Definition at line 221 of file BufferedColumnarFile.h.

◆ Write() [2/6]

template<uint64_t BufferSize = 100 * 1024>

void casacore::VarBufferedColumnarFile< BufferSize >::Write	(	uint64_t	row,
		uint64_t	column_offset,
		const double *	data,
		uint64_t	n )

inline

Write an array of doubles.

See float version for documentation.

Definition at line 188 of file BufferedColumnarFile.h.

◆ Write() [3/6]

template<uint64_t BufferSize = 100 * 1024>

void casacore::VarBufferedColumnarFile< BufferSize >::Write	(	uint64_t	row,
		uint64_t	column_offset,
		const float *	data,
		uint64_t	n )

inline

Write one cell containing an array of floats.

If the row is past the end of the file, the file is enlarged (making NRows() = row + 1).

Definition at line 180 of file BufferedColumnarFile.h.

◆ Write() [4/6]

template<uint64_t BufferSize = 100 * 1024>

void casacore::VarBufferedColumnarFile< BufferSize >::Write	(	uint64_t	row,
		uint64_t	column_offset,
		const int32_t *	data,
		uint64_t	n )

inline

Write an array of int32_t.

See float version for documentation.

Definition at line 196 of file BufferedColumnarFile.h.

◆ Write() [5/6]

template<uint64_t BufferSize = 100 * 1024>

void casacore::VarBufferedColumnarFile< BufferSize >::Write	(	uint64_t	row,
		uint64_t	column_offset,
		const std::complex< double > *	data,
		uint64_t	n )

inline

Write an array of complex doubles.

See float version for documentation.

Definition at line 212 of file BufferedColumnarFile.h.

◆ Write() [6/6]

template<uint64_t BufferSize = 100 * 1024>

void casacore::VarBufferedColumnarFile< BufferSize >::Write	(	uint64_t	row,
		uint64_t	column_offset,
		const std::complex< float > *	data,
		uint64_t	n )

inline

Write an array of complex floats.

See float version for documentation.

Definition at line 204 of file BufferedColumnarFile.h.

◆ WriteActiveBlock()

template<uint64_t BufferSize = 100 * 1024>

void casacore::VarBufferedColumnarFile< BufferSize >::WriteActiveBlock ( )

inlineprivate

Definition at line 319 of file BufferedColumnarFile.h.

Referenced by casacore::VarBufferedColumnarFile< 100 *1024 >::ActivateBlock(), and casacore::VarBufferedColumnarFile< 100 *1024 >::Close().

◆ WriteHeader()

template<uint64_t BufferSize = 100 * 1024>

void casacore::RowBasedFile::WriteHeader ( const unsigned char * data )

inline

Write an optional extra header to the file.

When creating the file, the requested space is saved to store this header.

Parameters

data	An array equal to the size of the header given in the CreateNew() and OpenExisting() calls.

Definition at line 189 of file RowBasedFile.h.

◆ WriteImplementation()

template<uint64_t BufferSize = 100 * 1024>

template<typename ValueType>

void casacore::VarBufferedColumnarFile< BufferSize >::WriteImplementation	(	uint64_t	row,
		uint64_t	column_offset,
		const ValueType *	data,
		uint64_t	n )

inlineprivate

Definition at line 306 of file BufferedColumnarFile.h.

Referenced by casacore::VarBufferedColumnarFile< 100 *1024 >::Write(), casacore::VarBufferedColumnarFile< 100 *1024 >::Write(), casacore::VarBufferedColumnarFile< 100 *1024 >::Write(), casacore::VarBufferedColumnarFile< 100 *1024 >::Write(), and casacore::VarBufferedColumnarFile< 100 *1024 >::Write().

Member Data Documentation

◆ active_block_

template<uint64_t BufferSize = 100 * 1024>

uint64_t casacore::VarBufferedColumnarFile< BufferSize >::active_block_

private

Definition at line 335 of file BufferedColumnarFile.h.

◆ block_buffer_

template<uint64_t BufferSize = 100 * 1024>

std::vector<unsigned char> casacore::VarBufferedColumnarFile< BufferSize >::block_buffer_

private

Definition at line 337 of file BufferedColumnarFile.h.

◆ block_changed_

template<uint64_t BufferSize = 100 * 1024>

bool casacore::VarBufferedColumnarFile< BufferSize >::block_changed_

private

Definition at line 334 of file BufferedColumnarFile.h.

◆ packed_buffer_

template<uint64_t BufferSize = 100 * 1024>

std::vector<unsigned char> casacore::VarBufferedColumnarFile< BufferSize >::packed_buffer_

private

This buffer is used temporarily for (un)packing booleans.

Storing it as a member avoids memory allocations.

Definition at line 332 of file BufferedColumnarFile.h.

◆ rows_per_block_

template<uint64_t BufferSize = 100 * 1024>

uint64_t casacore::VarBufferedColumnarFile< BufferSize >::rows_per_block_

private

Definition at line 336 of file BufferedColumnarFile.h.

The documentation for this class was generated from the following file:

tables/AlternateMans/BufferedColumnarFile.h

Public Member Functions

Static Public Member Functions

Private Member Functions

Private Attributes

Detailed Description

Constructor & Destructor Documentation

◆ VarBufferedColumnarFile() [1/5]

◆ VarBufferedColumnarFile() [2/5]

◆ VarBufferedColumnarFile() [3/5]

◆ ~VarBufferedColumnarFile()

◆ VarBufferedColumnarFile() [4/5]

◆ VarBufferedColumnarFile() [5/5]

Member Function Documentation

◆ ActivateBlock()

◆ AddRows()

◆ Close()

◆ CreateNew()

◆ DeleteRow()

◆ Filename()

◆ IsOpen()

◆ NRows()

◆ OpenExisting()

◆ operator=()

◆ Read() [1/5]

◆ Read() [2/5]

◆ Read() [3/5]

◆ Read() [4/5]

◆ Read() [5/5]

◆ ReadHeader()

◆ ReadImplementation()

◆ SetStride()

◆ Stride()

◆ Write() [1/6]

◆ Write() [2/6]

◆ Write() [3/6]

◆ Write() [4/6]

◆ Write() [5/6]

◆ Write() [6/6]

◆ WriteActiveBlock()

◆ WriteHeader()

◆ WriteImplementation()

Member Data Documentation

◆ active_block_

◆ block_buffer_

◆ block_changed_

◆ packed_buffer_

◆ rows_per_block_