#include "cp_svfs.h"
#include <ctime>
#include <memory>
#include "svf.h"
#include "svfs_util.h"

Classes
struct	cp_SparseVirtualFile
	Python wrapper around a C++ SparseVirtualFile. More...

class	AcquireLockSVF
	A RAII wrapper around the PyThread_type_lock for the CPython SVF. More...

Macros
#define	SVFS_SVF_METHOD_SIZE_T_WRAPPER(method_name, docstring)

#define	SVFS_SVF_METHOD_SIZE_T_REGISTER(method_name)

#define	ASSERT_FUNCTION_ENTRY_SVF(member)

Functions
static PyObject *	cp_SparseVirtualFile_new (PyTypeObject type, PyObject Py_UNUSED(args), PyObject *Py_UNUSED(kwds))

static int	cp_SparseVirtualFile_init (cp_SparseVirtualFile self, PyObject args, PyObject *kwargs)

static void	cp_SparseVirtualFile_dealloc (cp_SparseVirtualFile *self)

	PyDoc_STRVAR (cp_SparseVirtualFile_id_docstring, "id(self) -> str\n\n" "Returns the ID of the Sparse Virtual File.")

static PyObject *	cp_SparseVirtualFile_id (cp_SparseVirtualFile *self)

	SVFS_SVF_METHOD_SIZE_T_WRAPPER (size_of, "Returns the estimate of total memory usage of the Sparse Virtual File.")

	SVFS_SVF_METHOD_SIZE_T_WRAPPER (num_bytes, "Returns the total number of file bytes held by the Sparse Virtual File.")

	SVFS_SVF_METHOD_SIZE_T_WRAPPER (num_blocks, "Returns the total number of blocks of data held by the Sparse Virtual File System.")

	SVFS_SVF_METHOD_SIZE_T_WRAPPER (last_file_position, "Returns the file position immediately past the last block.")

	PyDoc_STRVAR (cp_SparseVirtualFile_has_data_docstring, "has_data(self, file_position: int, length: int) -> bool\n\n" "Checks if the Sparse Virtual File of the ID has data at the given ``file_position`` and ``length``.\n\n" "Parameters\n\n" "file_position: int\n" " The absolute file position of the start of the data.\n\n" "length: int\n" " The length of the required data in bytes.\n\n" "Returns\n\n" "bool: True if the SVF contains the data, False otherwise.")

static PyObject *	cp_SparseVirtualFile_has_data (cp_SparseVirtualFile self, PyObject args, PyObject *kwargs)

	PyDoc_STRVAR (cp_SparseVirtualFile_write_docstring, "write(self, file_position: int, data: bytes) -> None\n\n" "Writes the data to the Sparse Virtual File of the given ID at ``file_position`` and ``data`` as a ``bytes`` object." " This will raise an ``IOError`` if ``self.compare_for_diff`` is True and given data is different than" " that seen before and only new data up to this point will be written." " If the ``byte`` data is empty nothing will be done." " This will raise a RuntimeError if the data can not be written for any other reason")

static PyObject *	cp_SparseVirtualFile_write (cp_SparseVirtualFile self, PyObject args, PyObject *kwargs)

	PyDoc_STRVAR (cp_SparseVirtualFile_read_docstring, "read(self, file_position: int, length: int) -> bytes\n\n" "Read the data from the Sparse Virtual File at ``file_position`` and ``length`` returning a ``bytes`` object." " This takes a file position and a length." " This will raise an ``IOError`` if any data is not present" " This will raise a ``RuntimeError`` if the data can not be read for any other reason")

static PyObject *	private_SparseVirtualFile_svf_read_as_py_bytes (cp_SparseVirtualFile *self, unsigned long long fpos, unsigned long long len)

static PyObject *	cp_SparseVirtualFile_read (cp_SparseVirtualFile self, PyObject args, PyObject *kwargs)

	PyDoc_STRVAR (cp_SparseVirtualFile_erase_docstring, "erase(self, file_position: int) -> None\n\n" "Erase the data from the Sparse Virtual File at the given ``file_position``" " which must be the beginning of a block.\n" "This will raise an ``IOError`` if a block is not present at that file position.")

static PyObject *	cp_SparseVirtualFile_erase (cp_SparseVirtualFile self, PyObject args, PyObject *kwargs)

	PyDoc_STRVAR (cp_SparseVirtualFile_clear_docstring, "clear(self) -> None\n\n" "Clear all the data from the Sparse Virtual File." " This removes all data and resets the internal counters." " Modification times are maintained.")

static PyObject *	cp_SparseVirtualFile_clear (cp_SparseVirtualFile *self)

	PyDoc_STRVAR (cp_SparseVirtualFile_need_docstring, "need(self, file_position: int, length: int, greedy_length: int = 0) -> typing.Tuple[typing.Tuple[int, int], ...]\n\n" "Given a file_position and length this returns a ordered list ``[(file_position, length), ...]`` of seek/read" " instructions of data that is required to be written to the Sparse Virtual File so that a subsequent read will" " succeed.\n" " If greedy_length is > 0 then, if possible, blocks will be coalesced to reduce the size of the return value." "\n\n" ".. note::\n" " If a greedy_length is given this will be the minimum size of the length of the required block." " If the length of the required block is so large (because of existing blocks likely to be coalesced)" " then the user might want to split the length, for example, into multiple (smaller) GET requests which" " will then be coalesced on ``write()``" "\n\n" ".. warning::\n" " The SVF has no knowledge of the the actual file size so when using a greedy length the need list might" " include positions beyond EOF.\n\n" " For example a file 1024 bytes long and a greedy length of 256 then ``need(1000, 24, 256)`` will create" " a need list of ``[(1000, 256),]``." " This should generate a ``write(1000, 24)`` not a ``write(1000, 256)``.\n\n" " It is up to the caller to handle this, however, ``reads()`` in C/C++/Python will ignore read lengths past" " EOF so the caller does not have to do anything.\n\n" "\n\nUsage::\n\n" " if not svf.has_data(position, length):\n" " for read_fpos, read_length in svf.need(position, length):\n" " # Somehow get data as a bytes object at read_fpos, read_length...\n" " svf.write(read_fpos, data)\n" " return svf.read(position, length):\n")

static PyObject *	cp_SparseVirtualFile_need_internal (const SVFS::SparseVirtualFile *pSvf, unsigned long long fpos, unsigned long long len, unsigned long long greedy_len)

static PyObject *	cp_SparseVirtualFile_need (cp_SparseVirtualFile self, PyObject args, PyObject *kwargs)

	PyDoc_STRVAR (cp_SparseVirtualFile_need_many_docstring, "need_many(self, seek_reads: typing.List[typing.Tuple[int, int]], greedy_length: int = 0) -> typing.Tuple[typing.Tuple[int, int], ...]\n\n" "Given a list of (file_position, length) this returns a ordered list ``[(file_position, length), ...]`` of seek/read" " instructions of data that is required to be written to the Sparse Virtual File so that a subsequent read will" " succeed.\n\n" "If greedy_length is > 0 then, if possible, blocks will be coalesced to reduce the size of the return value." "\n\n" "See also :py:meth:`svfsc.cSVF.need`")

static PyObject *	cp_SparseVirtualFile_need_many_internal (PyObject py_seek_reads, const SVFS::SparseVirtualFile pSvf, unsigned long long greedy_len)

static PyObject *	cp_SparseVirtualFile_need_many (cp_SparseVirtualFile self, PyObject args, PyObject *kwargs)

	PyDoc_STRVAR (cp_SparseVirtualFile_blocks_docstring, "blocks(self) -> typing.Tuple[typing.Tuple[int, int], ...]\n\n" "This returns a ordered tuple ``((file_position, length), ...)``" " of the shape of the blocks held by the SVF in file position order.")

static PyObject *	cp_SparseVirtualFile_blocks (cp_SparseVirtualFile *self)

	SVFS_SVF_METHOD_SIZE_T_WRAPPER (block_touch, "Return the latest value of the monotonically increasing block_touch value.")

	PyDoc_STRVAR (cp_SparseVirtualFile_block_touches_docstring, "block_touches(self) -> typing.Dict[int, int]\n\n" "This returns a dict ``{touch_int: file_position, ...}``" " of the touch integer of each block mapped to the file position.\n" "The caller can decide what older blocks can be used the erase(file_position).")

static PyObject *	cp_SparseVirtualFile_block_touches (cp_SparseVirtualFile *self)

	PyDoc_STRVAR (cp_SparseVirtualFile_lru_punt_docstring, "lru_punt(self, cache_size_upper_bound: int) -> int\n\n" "Reduces the size of the cache to < the given size by removing older blocks, at least one block will be left.\n" "There are limitations to this tactic, see the documentation in Technical Notes -> Cache Punting.")

static PyObject *	cp_SparseVirtualFile_lru_punt (cp_SparseVirtualFile self, PyObject args, PyObject *kwargs)

	PyDoc_STRVAR (cp_SparseVirtualFile_file_mod_time_matches_docstring, "file_mod_time_matches(self, file_mod_time: float) -> bool\n\n" "Returns True if the file modification time of the Sparse Virtual File matches the given time as a float.")

static PyObject *	cp_SparseVirtualFile_file_mod_time_matches (cp_SparseVirtualFile self, PyObject args, PyObject *kwargs)

	PyDoc_STRVAR (cp_SparseVirtualFile_file_mod_time_docstring, "file_mod_time(self) -> float\n\n" "Returns the file modification time as a float in UNIX time of the Sparse Virtual File.")

static PyObject *	cp_SparseVirtualFile_file_mod_time (cp_SparseVirtualFile *self)

	SVFS_SVF_METHOD_SIZE_T_WRAPPER (count_write, "Returns the count of write operations on the Sparse Virtual File.")

	SVFS_SVF_METHOD_SIZE_T_WRAPPER (count_read, "Returns the count of read operations on the Sparse Virtual File.")

	SVFS_SVF_METHOD_SIZE_T_WRAPPER (bytes_write, "Returns the count of the number of bytes writen to the Sparse Virtual File.")

	SVFS_SVF_METHOD_SIZE_T_WRAPPER (bytes_read, "Returns the count of the number of bytes read from the Sparse Virtual File.")

	SVFS_SVF_METHOD_SIZE_T_WRAPPER (blocks_erased, "Returns the The total count of blocks that have been erased either directly or by punting.")

	SVFS_SVF_METHOD_SIZE_T_WRAPPER (bytes_erased, "Returns the The total count of bytes that have been erased either directly or by punting.")

	SVFS_SVF_METHOD_SIZE_T_WRAPPER (blocks_punted, "Returns the The total count of blocks that have been erased by punting.")

	SVFS_SVF_METHOD_SIZE_T_WRAPPER (bytes_punted, "Returns the The total count of bytes that have been erased by punting.")

	PyDoc_STRVAR (cp_SparseVirtualFile_time_write_docstring, "time_write(self) -> typing.Optional[datetime.datetime]\n\n" "Returns the timestamp of the last write to the Sparse Virtual File as a ``datetime.datetime``" " or ``None`` if no write has taken place.")

static PyObject *	cp_SparseVirtualFile_time_write (cp_SparseVirtualFile *self)

	PyDoc_STRVAR (cp_SparseVirtualFile_time_read_docstring, "time_read(self) -> typing.Optional[datetime.datetime]\n\n" "Returns the timestamp of the last read from the Sparse Virtual File as a ``datetime.datetime``" " or ``None`` if no read has taken place.")

static PyObject *	cp_SparseVirtualFile_time_read (cp_SparseVirtualFile *self)

	PyDoc_STRVAR (cp_SparseVirtualFile_config_docstring, "config(self) -> typing.Dict[str, bool]\n\n" "Returns the SVF configuration as a dict.")

static PyObject *	cp_SparseVirtualFile_config (cp_SparseVirtualFile *self)

static PyObject *	cp_SparseVirtualFile___getstate__ (cp_SparseVirtualFile self, PyObject Py_UNUSED(ignored))

static PyObject *	cp_SparseVirtualFile___setstate__ (cp_SparseVirtualFile self, PyObject state)

	PyDoc_STRVAR (svfs_cSVF_doc, "This class implements a Sparse Virtual File (SVF)." " This is an in-memory file that has fragments of a real file." " It has read/write operations and can describe what file fragments are needed, if any, before any read operation." "\n\n" "The constructor takes a string as an ID and optionally:\n" " - A file modification time as a float (default 0.0)." " This can be used for checking if the actual file might been changed which might invalidate the SVF.\n" " - ``overwrite_on_exit``, a boolean that will overwrite the memory on destruction (default ``False``)." " If ``True`` then ``clear()`` on a 1Mb SVF typically takes 35 µs, if ``False`` 1.5 µs.\n" " - ``compare_for_diff``, a boolean that will check that overlapping writes match (default ``True``)." " If ``True`` this adds about 25% time to an overlapping write but gives better chance of catching changes to the" " original file.\n" "\n\n" "For example::" "\n\n" " import svfsc\n" " \n" " svf = svfsc.cSVF('some ID')\n" " svf.write(12, b'ABCD')\n" " svf.read(13, 2) # Returns b'BC'\n" " svf.need(10, 12) # Returns ((10, 2), 16, 6)), the file positions and lengths the the SVF needs\n" " svf.read(1024, 18) # SVF raises an error as it has no data here.\n" "\n" "Signature:\n\n``svfsc.cSVF(id: str, mod_time: float = 0.0, overwrite_on_exit: bool = False, compare_for_diff: bool = True)``")

Variables
static const char *	PICKLE_ID_KEY = "id"

static const char *	PICKLE_FILE_MOD_TIME_KEY = "file_mod_time"

static const char *	PICKLE_BLOCKS_KEY = "blocks"

static const char *	PICKLE_VERSION_KEY = "pickle_version"

static int	PICKLE_VERSION = 1

static PyMemberDef	cp_SparseVirtualFile_members []

static PyMethodDef	cp_SparseVirtualFile_methods []

static PyTypeObject	svfsc_cSVF

Detailed Description

Python wrapper around a C++ SparseVirtualFile.

Naming convention:

SVF functions are named cp_SparseVirtualFile_...

   MIT License

   Copyright (c) 2020-2025 Paul Ross

   Permission is hereby granted, free of charge, to any person obtaining a copy
   of this software and associated documentation files (the "Software"), to deal
   in the Software without restriction, including without limitation the rights
   to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
   copies of the Software, and to permit persons to whom the Software is
   furnished to do so, subject to the following conditions:

   The above copyright notice and this permission notice shall be included in all
   copies or substantial portions of the Software.

   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
   IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
   FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
   AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
   LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
   OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
   SOFTWARE.

Definition in file _cSVF.cpp.

Macro Definition Documentation

◆ ASSERT_FUNCTION_ENTRY_SVF

#define ASSERT_FUNCTION_ENTRY_SVF ( member )

Value:

    do { \
    assert(self); \
    assert(((cp_SparseVirtualFile *)self)->member); \
    assert(! PyErr_Occurred()); \
} while (0)

Function entry point test macro. After construction, we expect this invariant at the entry to each function. The cast is necessary when used with functions that take a SVFS as a PyObject* such as cp_SparseVirtualFile_mapping_length

Definition at line 165 of file _cSVF.cpp.

◆ SVFS_SVF_METHOD_SIZE_T_REGISTER

#define SVFS_SVF_METHOD_SIZE_T_REGISTER ( method_name )

Value:

    {                                                \
        #method_name, \
        (PyCFunction) cp_SparseVirtualFile_##method_name, \
        METH_NOARGS, \
        cp_SparseVirtualFile_##method_name##_docstring \
    }

Complimentary to SVFS_SVF_METHOD_SIZE_T_WRAPPER this is for adding methods into the methods table cp_SparseVirtualFile_methods

Definition at line 92 of file _cSVF.cpp.

◆ SVFS_SVF_METHOD_SIZE_T_WRAPPER

#define SVFS_SVF_METHOD_SIZE_T_WRAPPER	(	method_name,
		docstring
	)

Value:

PyDoc_STRVAR(                                                  \
    cp_SparseVirtualFile_##method_name##_docstring,            \
    #method_name"(self) -> int\n\n"                                \
    docstring                                                  \
);\
static PyObject * \
cp_SparseVirtualFile_##method_name(cp_SparseVirtualFile *self) { \
    ASSERT_FUNCTION_ENTRY_SVF(pSvf); \
    PyObject *ret = NULL; \
    try { \
        ret = PyLong_FromLong(self->pSvf->method_name()); \
    if (!ret) { \
        PyErr_Format(PyExc_RuntimeError, "%s: Can not create integer from size_t.", __FUNCTION__); \
        goto except; \
    } \
    } catch (const std::exception &err) { \
        PyErr_Format(PyExc_RuntimeError, "%s: FATAL caught std::exception %s", __FUNCTION__, err.what()); \
        goto except; \
    } \
    assert(! PyErr_Occurred()); \
    assert(ret); \
    goto finally; \
except: \
    assert(PyErr_Occurred()); \
    Py_XDECREF(ret); \
    ret = NULL; \
finally: \
    return ret; \
}

TODO: Implement the Buffer Protocol rather than returning a copy of the bytes? Look for PyBytes_FromStringAndSize(). This macro is for functions that return a size_t type such as count_write, count_read, bytes_write, bytes_read.

Usage:

PyDoc_STRVAR( cp_SparseVirtualFile_count_write_docstring, "count_write(self) -> int\n\n" "Returns the count of write operations on the Sparse Virtual File." );

Perhaps ret = Py_BuildValue("K", self->pSvf->method_name());

Definition at line 57 of file _cSVF.cpp.

Function Documentation

◆ cp_SparseVirtualFile_getstate()

static PyObject* cp_SparseVirtualFile___getstate__	(	cp_SparseVirtualFile *	self,
		PyObject *	Py_UNUSEDignored
	)

static

Returns a Python dict suitable for pickling. Key/values are: id, file_mod_time, list_of_file_pos/bytes

Parameters

self
_unused_ignored

Returns

Definition at line 1243 of file _cSVF.cpp.

◆ cp_SparseVirtualFile_setstate()

static PyObject* cp_SparseVirtualFile___setstate__	(	cp_SparseVirtualFile *	self,
		PyObject *	state
	)

static

Definition at line 1295 of file _cSVF.cpp.

◆ cp_SparseVirtualFile_block_touches()

static PyObject* cp_SparseVirtualFile_block_touches ( cp_SparseVirtualFile * self )

static

Definition at line 895 of file _cSVF.cpp.

◆ cp_SparseVirtualFile_blocks()

static PyObject* cp_SparseVirtualFile_blocks ( cp_SparseVirtualFile * self )

static

Definition at line 838 of file _cSVF.cpp.

◆ cp_SparseVirtualFile_clear()

static PyObject* cp_SparseVirtualFile_clear ( cp_SparseVirtualFile * self )

static

Definition at line 558 of file _cSVF.cpp.

◆ cp_SparseVirtualFile_config()

static PyObject* cp_SparseVirtualFile_config ( cp_SparseVirtualFile * self )

static

Definition at line 1211 of file _cSVF.cpp.

◆ cp_SparseVirtualFile_dealloc()

static void cp_SparseVirtualFile_dealloc ( cp_SparseVirtualFile * self )

static

Deallocate the SparseVirtualFile.

Parameters

self	The Python SparseVirtualFile.

Definition at line 270 of file _cSVF.cpp.

◆ cp_SparseVirtualFile_erase()

static PyObject* cp_SparseVirtualFile_erase	(	cp_SparseVirtualFile *	self,
		PyObject *	args,
		PyObject *	kwargs
	)

static

Definition at line 518 of file _cSVF.cpp.

◆ cp_SparseVirtualFile_file_mod_time()

static PyObject* cp_SparseVirtualFile_file_mod_time ( cp_SparseVirtualFile * self )

static

Definition at line 1051 of file _cSVF.cpp.

◆ cp_SparseVirtualFile_file_mod_time_matches()

static PyObject* cp_SparseVirtualFile_file_mod_time_matches	(	cp_SparseVirtualFile *	self,
		PyObject *	args,
		PyObject *	kwargs
	)

static

Definition at line 997 of file _cSVF.cpp.

◆ cp_SparseVirtualFile_has_data()

static PyObject* cp_SparseVirtualFile_has_data	(	cp_SparseVirtualFile *	self,
		PyObject *	args,
		PyObject *	kwargs
	)

static

Definition at line 352 of file _cSVF.cpp.

◆ cp_SparseVirtualFile_id()

static PyObject* cp_SparseVirtualFile_id ( cp_SparseVirtualFile * self )

static

Definition at line 294 of file _cSVF.cpp.

◆ cp_SparseVirtualFile_init()

static int cp_SparseVirtualFile_init	(	cp_SparseVirtualFile *	self,
		PyObject *	args,
		PyObject *	kwargs
	)

static

Initialise the cp_SparseVirtualFile.

See the defaults for SVFS::SparseVirtualFileConfig for how that works.

Arguments/keywords:

id Mandatory, str, ID.
mod_time Optional, float, modification time, defaults to 0.0
overwrite_on_exit Optional, bool, See the defaults for SVFS::SparseVirtualFileConfig
compare_for_diff Optional, bool, See the defaults for SVFS::SparseVirtualFileConfig

Parameters

self	The cp_SparseVirtualFile.
args	Order: "id", "mod_time", "overwrite_on_exit", "compare_for_diff".
kwargs	Can be "id", "mod_time", "overwrite_on_exit", "compare_for_diff".

Returns: Zero on success, non-zero on failure.

Definition at line 219 of file _cSVF.cpp.

◆ cp_SparseVirtualFile_lru_punt()

static PyObject* cp_SparseVirtualFile_lru_punt	(	cp_SparseVirtualFile *	self,
		PyObject *	args,
		PyObject *	kwargs
	)

static

See cp_SparseVirtualFile_lru_punt_docstring

Parameters

self	The cp_SparseVirtualFile
args	The upper bound of the number of bytes held by the cache.
kwargs	"cache_size_upper_bound".

Returns: Number of bytes removed.

Definition at line 958 of file _cSVF.cpp.

◆ cp_SparseVirtualFile_need()

static PyObject* cp_SparseVirtualFile_need	(	cp_SparseVirtualFile *	self,
		PyObject *	args,
		PyObject *	kwargs
	)

static

See cp_SparseVirtualFile_need_docstring

Parameters

self	The cp_SparseVirtualFile
args	The file_position and length. Optionally a greedy_length.
kwargs	"file_position", "length", "greedy_length".

Returns: List of tuples (file_position, length).

Definition at line 647 of file _cSVF.cpp.

◆ cp_SparseVirtualFile_need_internal()

static PyObject* cp_SparseVirtualFile_need_internal	(	const SVFS::SparseVirtualFile *	pSvf,
		unsigned long long	fpos,
		unsigned long long	len,
		unsigned long long	greedy_len
	)

static

Definition at line 600 of file _cSVF.cpp.

◆ cp_SparseVirtualFile_need_many()

static PyObject* cp_SparseVirtualFile_need_many	(	cp_SparseVirtualFile *	self,
		PyObject *	args,
		PyObject *	kwargs
	)

static

See cp_SparseVirtualFile_need_many_docstring

Parameters

self	The cp_SparseVirtualFile
args	The list of (file_position, length). Optionally a greedy_length.
kwargs	"seek_reads", "greedy_length".

Returns: List of tuples (file_position, length).

Definition at line 775 of file _cSVF.cpp.

◆ cp_SparseVirtualFile_need_many_internal()

static PyObject* cp_SparseVirtualFile_need_many_internal	(	PyObject *	py_seek_reads,
		const SVFS::SparseVirtualFile *	pSvf,
		unsigned long long	greedy_len
	)

static

Shared by SVF and SVFS.

Definition at line 696 of file _cSVF.cpp.

◆ cp_SparseVirtualFile_new()

static PyObject* cp_SparseVirtualFile_new	(	PyTypeObject *	type,
		PyObject *	Py_UNUSEDargs,
		PyObject *	Py_UNUSEDkwds
	)

static

Python new() for a cp_SparseVirtualFile. Contents will be NULL.

Parameters

type	The cp_SparseVirtualFile type.
_unused_args	Unused.
_unused_kwds	Unused.

Returns: An empty cp_SparseVirtualFile.

Definition at line 185 of file _cSVF.cpp.

◆ cp_SparseVirtualFile_read()

static PyObject* cp_SparseVirtualFile_read	(	cp_SparseVirtualFile *	self,
		PyObject *	args,
		PyObject *	kwargs
	)

static

Definition at line 480 of file _cSVF.cpp.

◆ cp_SparseVirtualFile_time_read()

static PyObject* cp_SparseVirtualFile_time_read ( cp_SparseVirtualFile * self )

static

Definition at line 1168 of file _cSVF.cpp.

◆ cp_SparseVirtualFile_time_write()

static PyObject* cp_SparseVirtualFile_time_write ( cp_SparseVirtualFile * self )

static

Definition at line 1123 of file _cSVF.cpp.

◆ cp_SparseVirtualFile_write()

static PyObject* cp_SparseVirtualFile_write	(	cp_SparseVirtualFile *	self,
		PyObject *	args,
		PyObject *	kwargs
	)

static

Definition at line 397 of file _cSVF.cpp.

◆ private_SparseVirtualFile_svf_read_as_py_bytes()

static PyObject* private_SparseVirtualFile_svf_read_as_py_bytes	(	cp_SparseVirtualFile *	self,
		unsigned long long	fpos,
		unsigned long long	len
	)

static

Definition at line 450 of file _cSVF.cpp.

◆ PyDoc_STRVAR() [1/17]

PyDoc_STRVAR	(	cp_SparseVirtualFile_block_touches_docstring	,
		"block_touches(self) -> typing.Dict\n\n" "This returns a dict ``{touch_int: file_position, ...}``" " of the touch integer of each block mapped to the file position.\n" "The caller can decide what older blocks can be used the erase(file_position)."	[int, int]
	)

◆ PyDoc_STRVAR() [2/17]

PyDoc_STRVAR	(	cp_SparseVirtualFile_blocks_docstring	,
		"blocks(self) -> typing.	Tuple[typing.Tuple[int, int],
		...]\n\n" "This returns a ordered tuple ``((file_position, length),...)``" " of the shape of the blocks held by the SVF in file position order."
	)

-— Meta information about the SVF -— The existing blocks. t_seek_reads blocks() const noexcept;

Information about memory used: size_of() gives best guess of total memory usage. size_t size_of() const noexcept;

Gives exact number of data bytes held. size_t num_bytes() const noexcept { return m_bytes_total; };

Gives exact number of blocks used. size_t num_blocks() const noexcept { return m_svf.size(); } t_fpos last_file_position() const noexcept;

Check the clients file modification time has changed. Caller has to decide what to do... bool file_mod_time_matches(const double &file_mod_time) const noexcept { return file_mod_time == m_file_mod_time; }

◆ PyDoc_STRVAR() [3/17]

PyDoc_STRVAR	(	cp_SparseVirtualFile_clear_docstring	,
		"clear(self) -> None\n\n" "Clear all the data from the Sparse Virtual File." " This removes all data and resets the internal counters." " Modification times are maintained."
	)

◆ PyDoc_STRVAR() [4/17]

PyDoc_STRVAR	(	cp_SparseVirtualFile_config_docstring	,
		"config(self) -> typing.Dict\n\n" "Returns the SVF configuration as a dict."	[str, bool]
	)

◆ PyDoc_STRVAR() [5/17]

PyDoc_STRVAR	(	cp_SparseVirtualFile_erase_docstring	,
		"erase(self, file_position: int) -> None\n\n" "Erase the data from the Sparse Virtual File at the given ``file_position``" " which must be the beginning of a block.\n" "This will raise an ``IOError`` if a block is not present at that file position."
	)

◆ PyDoc_STRVAR() [6/17]

PyDoc_STRVAR	(	cp_SparseVirtualFile_file_mod_time_docstring	,
		"file_mod_time(self) -> float\n\n" "Returns the file modification time as a float in UNIX time of the Sparse Virtual File."
	)

-— Attribute access -— const std::string id() const noexcept { return m_id; } double file_mod_time() const noexcept { return m_file_mod_time; }

size_t count_write() const noexcept { return m_count_write; } size_t count_read() const noexcept { return m_count_read; } size_t bytes_write() const noexcept { return m_bytes_write; } size_t bytes_read() const noexcept { return m_bytes_read; }

std::chrono::time_point<std::chrono::system_clock> time_write() const noexcept { return m_time_write; } std::chrono::time_point<std::chrono::system_clock> time_read() const noexcept { return m_time_read; }

◆ PyDoc_STRVAR() [7/17]

PyDoc_STRVAR	(	cp_SparseVirtualFile_file_mod_time_matches_docstring	,
		"file_mod_time_matches(self, file_mod_time: float) -> bool\n\n" "Returns True if the file modification time of the Sparse Virtual File matches the given time as a float."
	)

◆ PyDoc_STRVAR() [8/17]

PyDoc_STRVAR	(	cp_SparseVirtualFile_has_data_docstring	,
		"has_data(self, file_position: int, length: int) -> bool\n\n" "Checks if the Sparse Virtual File of the ID has data at the given ``file_position`` and ``length``.\n\n" "Parameters\n\n" "file_position: int\n" " The absolute file position of the start of the data.\n\n" "length: int\n" " The length of the required data in bytes.\n\n" "Returns\n\n" "bool: True if the SVF contains the	data,
		False otherwise."
	)

◆ PyDoc_STRVAR() [9/17]

PyDoc_STRVAR	(	cp_SparseVirtualFile_id_docstring	,
		"id(self) -> str\n\n" "Returns the ID of the Sparse Virtual File."
	)

◆ PyDoc_STRVAR() [10/17]

PyDoc_STRVAR	(	cp_SparseVirtualFile_lru_punt_docstring	,
		"lru_punt(self, cache_size_upper_bound: int) -> int\n\n" "Reduces the size of the cache to < the given size by removing older blocks, at least one block will be left.\n" "There are limitations to this tactic, see the documentation in Technical Notes -> Cache Punting."
	)

◆ PyDoc_STRVAR() [11/17]

PyDoc_STRVAR	(	cp_SparseVirtualFile_need_docstring	,
		"need(self, file_position: int, length: int, greedy_length: int = 0) -> typing.	Tuple[typing.Tuple[int, int],
		...]\n\n" "Given a file_position and length this returns a ordered list ```` of seek/read" " instructions of data that is required to be written to the Sparse Virtual File so that a subsequent read will" " succeed.\n" " If greedy_length	is[(file_position, length),...],
		0	then,
		if	possible,
		blocks will be coalesced to reduce the size of the return value." "\n\n" ".. note::\n" " If a greedy_length is given this will be the minimum size of the length of the required block." " If the length of the required block is so large(because of existing blocks likely to be coalesced)" " then the user might want to split the	length,
		for	example,
		into multiple(smaller) GET requests which" " will then be coalesced on ``write()``" "\n\n" ".. warning::\n" " The SVF has no knowledge of the the actual file size so when using a greedy length the need list might" " include positions beyond EOF.\n\n" " For example a file 1024 bytes long and a greedy length of 256 then ``need(1000, 24, 256)`` will create" " a need list of ````." " This should generate a ``write(1000, 24)`` not a ``write(1000, 256)``.\n\n" " It is up to the caller to handle	this[(1000, 256),],
		however	,
		``reads()`` in C/C++/Python will ignore read lengths past" " EOF so the caller does not have to do anything.\n\n" "\n\nUsage::\n\n" " if not svf.has_data(position, length):\n" " for	read_fpos,
		read_length in svf.need(position, length):\n" " # Somehow get data as a bytes object at	read_fpos,
		read_length...\n" " svf.write(read_fpos, data)\n" " return svf.read(position, length):\n"
	)

◆ PyDoc_STRVAR() [12/17]

PyDoc_STRVAR	(	cp_SparseVirtualFile_need_many_docstring	,
		"need_many(self, seek_reads: typing.List[typing.Tuple[int, int]], greedy_length: int = 0) -> typing.	Tuple[typing.Tuple[int, int],
		...]\n\n" "Given a list of(file_position, length) this returns a ordered list ```` of seek/read" " instructions of data that is required to be written to the Sparse Virtual File so that a subsequent read will" " succeed.\n\n" "If greedy_length	is[(file_position, length),...],
		0	then,
		if	possible,
		blocks will be coalesced to reduce the size of the return value." "\n\n" "See also :py:meth:`svfsc.cSVF.need`"
	)

◆ PyDoc_STRVAR() [13/17]

PyDoc_STRVAR	(	cp_SparseVirtualFile_read_docstring	,
		"read(self, file_position: int, length: int) -> bytes\n\n" "Read the data from the Sparse Virtual File at ``file_position`` and ``length`` returning a ``bytes`` object." " This takes a file position and a length." " This will raise an ``IOError`` if any data is not present" " This will raise a ``RuntimeError`` if the data can not be read for any other reason"
	)

◆ PyDoc_STRVAR() [14/17]

PyDoc_STRVAR	(	cp_SparseVirtualFile_time_read_docstring	,
		"time_read(self) -> typing.Optional\n\n" "Returns the timestamp of the last read from the Sparse Virtual File as a ``datetime.datetime``" " or ``None`` if no read has taken place."	[datetime.datetime]
	)

◆ PyDoc_STRVAR() [15/17]

PyDoc_STRVAR	(	cp_SparseVirtualFile_time_write_docstring	,
		"time_write(self) -> typing.Optional\n\n" "Returns the timestamp of the last write to the Sparse Virtual File as a ``datetime.datetime``" " or ``None`` if no write has taken place."	[datetime.datetime]
	)

◆ PyDoc_STRVAR() [16/17]

PyDoc_STRVAR	(	cp_SparseVirtualFile_write_docstring	,
		"write(self, file_position: int, data: bytes) -> None\n\n" "Writes the data to the Sparse Virtual File of the given ID at ``file_position`` and ``data`` as a ``bytes`` object." " This will raise an ``IOError`` if ``self.compare_for_diff`` is True and given data is different than" " that seen before and only new data up to this point will be written." " If the ``byte`` data is empty nothing will be done." " This will raise a RuntimeError if the data can not be written for any other reason"
	)

◆ PyDoc_STRVAR() [17/17]

PyDoc_STRVAR	(	svfs_cSVF_doc	,
		"This class implements a Sparse Virtual File (SVF)." " This is an in-memory file that has fragments of a real file." " It has read/write operations and can describe what file fragments are	needed,
		if	any,
		before any read operation." "\n\n" "The constructor takes a string as an ID and optionally:\n" " - A file modification time as a float(default 0.0)." " This can be used for checking if the actual file might been changed which might invalidate the SVF.\n" " - ``overwrite_on_exit``	,
		a boolean that will overwrite the memory on destruction(default ``False``)." " If ``True`` then ``clear()`` on a 1Mb SVF typically takes 35	µs,
		if ``False`` 1.5 µs.\n" " - ``compare_for_diff``	,
		a boolean that will check that overlapping writes match(default ``True``)." " If ``True`` this adds about 25% time to an overlapping write but gives better chance of catching changes to the" " original file.\n" "\n\n" "For example::" "\n\n" " import svfsc\n" " \n" "	svf = `svfsc.cSVF('some ID')\n" " svf.write(12, b'ABCD')\n" " svf.read(13, 2) # Returns b'BC'\n" " svf.need(10, 12) # Returns ((10, 2), 16, 6)`
	)

◆ SVFS_SVF_METHOD_SIZE_T_WRAPPER() [1/13]

SVFS_SVF_METHOD_SIZE_T_WRAPPER	(	block_touch	,
		"Return the latest value of the monotonically increasing block_touch value."
	)

◆ SVFS_SVF_METHOD_SIZE_T_WRAPPER() [2/13]

SVFS_SVF_METHOD_SIZE_T_WRAPPER	(	blocks_erased	,
		"Returns the The total count of blocks that have been erased either directly or by punting."
	)

◆ SVFS_SVF_METHOD_SIZE_T_WRAPPER() [3/13]

SVFS_SVF_METHOD_SIZE_T_WRAPPER	(	blocks_punted	,
		"Returns the The total count of blocks that have been erased by punting."
	)

◆ SVFS_SVF_METHOD_SIZE_T_WRAPPER() [4/13]

SVFS_SVF_METHOD_SIZE_T_WRAPPER	(	bytes_erased	,
		"Returns the The total count of bytes that have been erased either directly or by punting."
	)

◆ SVFS_SVF_METHOD_SIZE_T_WRAPPER() [5/13]

SVFS_SVF_METHOD_SIZE_T_WRAPPER	(	bytes_punted	,
		"Returns the The total count of bytes that have been erased by punting."
	)

◆ SVFS_SVF_METHOD_SIZE_T_WRAPPER() [6/13]

SVFS_SVF_METHOD_SIZE_T_WRAPPER	(	bytes_read	,
		"Returns the count of the number of bytes read from the Sparse Virtual File."
	)

◆ SVFS_SVF_METHOD_SIZE_T_WRAPPER() [7/13]

SVFS_SVF_METHOD_SIZE_T_WRAPPER	(	bytes_write	,
		"Returns the count of the number of bytes writen to the Sparse Virtual File."
	)

◆ SVFS_SVF_METHOD_SIZE_T_WRAPPER() [8/13]

SVFS_SVF_METHOD_SIZE_T_WRAPPER	(	count_read	,
		"Returns the count of read operations on the Sparse Virtual File."
	)

◆ SVFS_SVF_METHOD_SIZE_T_WRAPPER() [9/13]

SVFS_SVF_METHOD_SIZE_T_WRAPPER	(	count_write	,
		"Returns the count of write operations on the Sparse Virtual File."
	)

◆ SVFS_SVF_METHOD_SIZE_T_WRAPPER() [10/13]

SVFS_SVF_METHOD_SIZE_T_WRAPPER	(	last_file_position	,
		"Returns the file position immediately past the last block."
	)

◆ SVFS_SVF_METHOD_SIZE_T_WRAPPER() [11/13]

SVFS_SVF_METHOD_SIZE_T_WRAPPER	(	num_blocks	,
		"Returns the total number of blocks of data held by the Sparse Virtual File System."
	)

◆ SVFS_SVF_METHOD_SIZE_T_WRAPPER() [12/13]

SVFS_SVF_METHOD_SIZE_T_WRAPPER	(	num_bytes	,
		"Returns the total number of file bytes held by the Sparse Virtual File."
	)

◆ SVFS_SVF_METHOD_SIZE_T_WRAPPER() [13/13]

SVFS_SVF_METHOD_SIZE_T_WRAPPER	(	size_of	,
		"Returns the estimate of total memory usage of the Sparse Virtual File."
	)

Variable Documentation

◆ cp_SparseVirtualFile_members

PyMemberDef cp_SparseVirtualFile_members[]

static

Initial value:

= {
 
 
        {NULL, 0, 0, 0, NULL}  
}

Definition at line 1407 of file _cSVF.cpp.

◆ cp_SparseVirtualFile_methods

PyMethodDef cp_SparseVirtualFile_methods[]

static

Definition at line 1413 of file _cSVF.cpp.

◆ PICKLE_BLOCKS_KEY

const char* PICKLE_BLOCKS_KEY = "blocks"

static

Definition at line 1230 of file _cSVF.cpp.

◆ PICKLE_FILE_MOD_TIME_KEY

const char* PICKLE_FILE_MOD_TIME_KEY = "file_mod_time"

static

Definition at line 1229 of file _cSVF.cpp.

◆ PICKLE_ID_KEY

const char* PICKLE_ID_KEY = "id"

static

Definition at line 1228 of file _cSVF.cpp.

◆ PICKLE_VERSION

int PICKLE_VERSION = 1

static

Definition at line 1232 of file _cSVF.cpp.

◆ PICKLE_VERSION_KEY

const char* PICKLE_VERSION_KEY = "pickle_version"

static

Definition at line 1231 of file _cSVF.cpp.

◆ svfsc_cSVF

PyTypeObject svfsc_cSVF

static

Initial value:

= {
        PyVarObject_HEAD_INIT(NULL, 0)
        .tp_name = "svfsc.cSVF",
        .tp_basicsize = sizeof(cp_SparseVirtualFile),
        .tp_itemsize = 0,
        .tp_dealloc = (destructor) cp_SparseVirtualFile_dealloc,
        .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE,
        .tp_doc = svfs_cSVF_doc,
        .tp_methods = cp_SparseVirtualFile_methods,
        .tp_members = cp_SparseVirtualFile_members,
        .tp_init = (initproc) cp_SparseVirtualFile_init,
        .tp_new = cp_SparseVirtualFile_new,
}

Definition at line 1544 of file _cSVF.cpp.

Classes

Macros

Functions

Variables

Detailed Description

Macro Definition Documentation

◆ ASSERT_FUNCTION_ENTRY_SVF

◆ SVFS_SVF_METHOD_SIZE_T_REGISTER

◆ SVFS_SVF_METHOD_SIZE_T_WRAPPER

Function Documentation

◆ cp_SparseVirtualFile___getstate__()

◆ cp_SparseVirtualFile___setstate__()

◆ cp_SparseVirtualFile_block_touches()

◆ cp_SparseVirtualFile_blocks()

◆ cp_SparseVirtualFile_clear()

◆ cp_SparseVirtualFile_config()

◆ cp_SparseVirtualFile_dealloc()

◆ cp_SparseVirtualFile_erase()

◆ cp_SparseVirtualFile_file_mod_time()

◆ cp_SparseVirtualFile_file_mod_time_matches()

◆ cp_SparseVirtualFile_has_data()

◆ cp_SparseVirtualFile_id()

◆ cp_SparseVirtualFile_init()

◆ cp_SparseVirtualFile_lru_punt()

◆ cp_SparseVirtualFile_need()

◆ cp_SparseVirtualFile_need_internal()

◆ cp_SparseVirtualFile_need_many()

◆ cp_SparseVirtualFile_need_many_internal()

◆ cp_SparseVirtualFile_new()

◆ cp_SparseVirtualFile_read()

◆ cp_SparseVirtualFile_time_read()

◆ cp_SparseVirtualFile_time_write()

◆ cp_SparseVirtualFile_write()

◆ private_SparseVirtualFile_svf_read_as_py_bytes()

◆ PyDoc_STRVAR() [1/17]

◆ PyDoc_STRVAR() [2/17]

◆ PyDoc_STRVAR() [3/17]

◆ PyDoc_STRVAR() [4/17]

◆ PyDoc_STRVAR() [5/17]

◆ PyDoc_STRVAR() [6/17]

◆ PyDoc_STRVAR() [7/17]

◆ PyDoc_STRVAR() [8/17]

◆ PyDoc_STRVAR() [9/17]

◆ PyDoc_STRVAR() [10/17]

◆ PyDoc_STRVAR() [11/17]

◆ PyDoc_STRVAR() [12/17]

◆ PyDoc_STRVAR() [13/17]

◆ PyDoc_STRVAR() [14/17]

◆ PyDoc_STRVAR() [15/17]

◆ PyDoc_STRVAR() [16/17]

◆ PyDoc_STRVAR() [17/17]

◆ SVFS_SVF_METHOD_SIZE_T_WRAPPER() [1/13]

◆ SVFS_SVF_METHOD_SIZE_T_WRAPPER() [2/13]

◆ SVFS_SVF_METHOD_SIZE_T_WRAPPER() [3/13]

◆ SVFS_SVF_METHOD_SIZE_T_WRAPPER() [4/13]

◆ SVFS_SVF_METHOD_SIZE_T_WRAPPER() [5/13]

◆ SVFS_SVF_METHOD_SIZE_T_WRAPPER() [6/13]

◆ SVFS_SVF_METHOD_SIZE_T_WRAPPER() [7/13]

◆ SVFS_SVF_METHOD_SIZE_T_WRAPPER() [8/13]

◆ SVFS_SVF_METHOD_SIZE_T_WRAPPER() [9/13]

◆ SVFS_SVF_METHOD_SIZE_T_WRAPPER() [10/13]

◆ SVFS_SVF_METHOD_SIZE_T_WRAPPER() [11/13]

◆ SVFS_SVF_METHOD_SIZE_T_WRAPPER() [12/13]

◆ SVFS_SVF_METHOD_SIZE_T_WRAPPER() [13/13]

Variable Documentation

◆ cp_SparseVirtualFile_members

◆ cp_SparseVirtualFile_methods

◆ PICKLE_BLOCKS_KEY

◆ PICKLE_FILE_MOD_TIME_KEY

◆ PICKLE_ID_KEY

◆ PICKLE_VERSION

◆ PICKLE_VERSION_KEY

◆ svfsc_cSVF

◆ cp_SparseVirtualFile_getstate()

◆ cp_SparseVirtualFile_setstate()