_cSVFS.cpp File Reference

#include "cp_svfs.h"
#include <ctime>
#include <memory>
#include "svfs.h"
#include "svfs_util.h"

Go to the source code of this file.

Classes
struct	cp_SparseVirtualFileSystem
	Python wrapper around a C++ SparseVirtualFile. More...
class	AcquireLockSVFS
	A RAII wrapper around the PyThread_type_lock for the CPython SVFS. More...

Macros
#define	SVFS_SVFS_METHOD_SIZE_T_WRAPPER(method_name, docstring)
#define	ASSERT_FUNCTION_ENTRY_SVFS(member)
#define	SVFS_SVFS_METHOD_DATETIME_WRAPPER(method_name)

Functions
static PyObject *	cp_SparseVirtualFileSystem_new (PyTypeObject *type, PyObject *Py_UNUSED(args), PyObject *Py_UNUSED(kwds))
static int	cp_SparseVirtualFileSystem_init (cp_SparseVirtualFileSystem *self, PyObject *args, PyObject *kwargs)
static void	cp_SparseVirtualFileSystem_dealloc (cp_SparseVirtualFileSystem *self)
	PyDoc_STRVAR (cp_SparseVirtualFileSystem_keys_docstring, "keys(self) -> typing.List[str]\n\n" "Returns the IDs of all the Sparse Virtual Files in the Sparse Virtual File System.")
static PyObject *	cp_SparseVirtualFileSystem_keys (cp_SparseVirtualFileSystem *self)
	PyDoc_STRVAR (cp_SparseVirtualFileSystem_insert_docstring, "insert(self, id: str) -> None\n\n" "Inserts a Sparse Virtual File of ID and Unix file modification time as a float.")
static PyObject *	cp_SparseVirtualFileSystem_insert (cp_SparseVirtualFileSystem *self, PyObject *args, PyObject *kwargs)
	PyDoc_STRVAR (cp_SparseVirtualFileSystem_remove_docstring, "remove(self, id: str) -> None\n\n" "Removes a Sparse Virtual File of ID freeing that file's memory. Will raise an ``IndexError`` if the ID is absent.")
static PyObject *	cp_SparseVirtualFileSystem_remove (cp_SparseVirtualFileSystem *self, PyObject *args, PyObject *kwargs)
	PyDoc_STRVAR (cp_SparseVirtualFileSystem_has_docstring, "has(self, id: str) -> bool\n\n" "Returns True if the Sparse Virtual File for the ID is in the Sparse Virtual File System.")
static PyObject *	cp_SparseVirtualFileSystem_has (cp_SparseVirtualFileSystem *self, PyObject *args, PyObject *kwargs)
	PyDoc_STRVAR (cp_SparseVirtualFileSystem_total_size_of_docstring, "total_size_of(self) -> int\n\n" "Returns the estimate of total memory usage of the Sparse Virtual File System.")
static PyObject *	cp_SparseVirtualFileSystem_total_size_of (cp_SparseVirtualFileSystem *self)
	PyDoc_STRVAR (cp_SparseVirtualFileSystem_total_bytes_docstring, "total_bytes(self) -> int\n\n" "Returns the total number of file bytes held by the Sparse Virtual File System.")
static PyObject *	cp_SparseVirtualFileSystem_total_bytes (cp_SparseVirtualFileSystem *self)
	PyDoc_STRVAR (cp_SparseVirtualFileSystem_total_blocks_docstring, "total_blocks(self) -> int\n\n" "Returns the total number of blocks of data held by the Sparse Virtual File System.")
static PyObject *	cp_SparseVirtualFileSystem_total_blocks (cp_SparseVirtualFileSystem *self)
	PyDoc_STRVAR (cp_SparseVirtualFileSystem_svf_has_data_docstring, "has_data(self, id: str, file_position: int, length: int) -> bool\n\n" "Checks if the Sparse Virtual File of the ID has data at the given file_position and length." " This takes a string as an id, a file position and a length." " This returns True if the Sparse Virtual File of that id has the data, False otherwise." " This will raise an ``IndexError`` if the SVF of that id does not exist.")
static PyObject *	cp_SparseVirtualFileSystem_svf_has_data (cp_SparseVirtualFileSystem *self, PyObject *args, PyObject *kwargs)
	PyDoc_STRVAR (cp_SparseVirtualFileSystem_svf_write_docstring, "write(self, id: str, file_position: int, data: bytes) -> None\n\n" "Writes the data to the Sparse Virtual File of the given ID at file_position and length.\n\n" "This takes a string as an id, a file position and data as a bytes object.\n" "This will raise an ``IndexError`` if the SVF of that id does not exist.\n" "This will raise an ``IOError`` if the given data is different than that seen before and only\n" "new data up to this point will be written.\n" "This will raise a ``RuntimeError`` if the data can not be written for any other reason.\n")
static PyObject *	cp_SparseVirtualFileSystem_svf_write (cp_SparseVirtualFileSystem *self, PyObject *args, PyObject *kwargs)
	PyDoc_STRVAR (cp_SparseVirtualFileSystem_svf_read_docstring, "read(self, id: str, file_position: int, length: int) -> bytes\n\n" "Read the data to the Sparse Virtual File at file_position and length returning a bytes object.\n" "This takes a string as an id, a file position and a length.\n" "\nThis will raise an ``IndexError`` if the Sparse Virtual File of that id does not exist.\n" "This will raise an ``IOError`` if any data is not present\n" "This will raise a ``RuntimeError`` if the data can not be read for any other reason.\n")
static PyObject *	cp_SparseVirtualFileSystem_svf_read (cp_SparseVirtualFileSystem *self, PyObject *args, PyObject *kwargs)
	PyDoc_STRVAR (cp_SparseVirtualFileSystem_svf_erase_docstring, "erase(self, id: str, file_position: int) -> None\n\n" "Erases the data block in the Sparse Virtual File at a file position.\n" "This takes a string as an id and a file_position.\n" "This will raise an ``IndexError`` if the Sparse Virtual File of that id does not exist.\n" "This will raise an ``IOError`` if there is not a block at the position.\n" "This will raise a ``RuntimeError`` if the data can not be read for any other reason.\n")
static PyObject *	cp_SparseVirtualFileSystem_svf_erase (cp_SparseVirtualFileSystem *self, PyObject *args, PyObject *kwargs)
	PyDoc_STRVAR (cp_SparseVirtualFileSystem_svf_need_docstring, "need(self, id: str, file_position: int, length: int, greedy_length: int = 0) -> typing.Tuple[typing.Tuple[int, int], ...]\n\n" "Given a file_position and length this returns a ordered list ``[(file_position, length), ...]`` of seek/read" " instructions of data that is required to be written to the Sparse Virtual File so that a subsequent read will succeed.\n" "If greedy_length is > 0 then, if possible, blocks will be coalesced to reduce the size of the return value." "\nUsage::\n\n" " if not svfs.has(identity, file_position, length):\n" " for fpos, read_len in svfs.need(identity, file_position, length):\n" " # Somehow get the data at that seek/read position...\n" " svfs.write(identity, fpos, data)\n" " return svfs.read(identity, file_position, length):\n")
static PyObject *	cp_SparseVirtualFileSystem_svf_need (cp_SparseVirtualFileSystem *self, PyObject *args, PyObject *kwargs)
	PyDoc_STRVAR (cp_SparseVirtualFileSystem_svf_need_many_docstring, "need_many(self, id: str, seek_reads: typing.List[typing.Tuple[int, int]], greedy_length: int = 0) -> typing.Tuple[typing.Tuple[int, int], ...]\n\n" "Given a list of (file_position, length) this returns a ordered list ``[(file_position, length), ...]`` of seek/read" " instructions of data that is required to be written to the Sparse Virtual File so that a subsequent read will" " succeed.\n\n" "If greedy_length is > 0 then, if possible, blocks will be coalesced to reduce the size of the return value." "\n\n" "See also :py:meth:`svfsc.cSVFS.need`")
static PyObject *	cp_SparseVirtualFileSystem_svf_need_many (cp_SparseVirtualFileSystem *self, PyObject *args, PyObject *kwargs)
	PyDoc_STRVAR (cp_SparseVirtualFileSystem_svf_blocks_docstring, "blocks(self, id: str) -> typing.Tuple[typing.Tuple[int, int], ...]\n\n" "This returns a ordered tuple ((file_position, length), ...) of all the blocks held by the SVF identified" "by the given id.\n" "This will raise an ``IndexError`` if the Sparse Virtual File of that id does not exist.")
static PyObject *	cp_SparseVirtualFileSystem_svf_blocks (cp_SparseVirtualFileSystem *self, PyObject *args, PyObject *kwargs)
	SVFS_SVFS_METHOD_SIZE_T_WRAPPER (size_of, "Returns the best guess of total memory usage used by the Sparse Virtual File identified by the given id." " This will raise an ``IndexError`` if the Sparse Virtual File of that id does not exist.")
	SVFS_SVFS_METHOD_SIZE_T_WRAPPER (num_bytes, "Returns the number of bytes of data held by the Sparse Virtual File identified by the given id." " This will raise an ``IndexError`` if the Sparse Virtual File of that id does not exist.")
	SVFS_SVFS_METHOD_SIZE_T_WRAPPER (num_blocks, "Returns the number of data blocks held by the Sparse Virtual File identified by the given id." " This will raise an ``IndexError`` if the Sparse Virtual File of that id does not exist.")
	PyDoc_STRVAR (cp_SparseVirtualFileSystem_svf_block_touches_docstring, "block_touches(self, id: str) -> typing.Dict[int, int]\n\n" "This returns a dict ``{touch_int: file_position, ...}``" " of the touch integer of each block mapped to the file position.\n" "The caller can decide what older blocks can be used the erase(file_position).")
static PyObject *	cp_SparseVirtualFileSystem_svf_block_touches (cp_SparseVirtualFileSystem *self, PyObject *args, PyObject *kwargs)
	PyDoc_STRVAR (cp_SparseVirtualFileSystem_svf_lru_punt_docstring, "lru_punt(self, id: str, cache_size_upper_bound: int) -> int\n\n" "Reduces the size of the cache to < the given size by removing older blocks, at least one block will be left.\n" "There are limitations to this tactic, see the documentation in Technical Notes -> Cache Punting.")
static PyObject *	cp_SparseVirtualFileSystem_svf_lru_punt (cp_SparseVirtualFileSystem *self, PyObject *args, PyObject *kwargs)
	PyDoc_STRVAR (cp_SparseVirtualFileSystem_svf_lru_punt_all_docstring, "lru_punt_all(self, cache_size_upper_bound: int) -> int\n\n" "Reduces the size of all IDs in the cache to < the given size by removing older blocks." " At least one block will be left for each ID.\n" "There are limitations to this tactic, see the documentation in Technical Notes -> Cache Punting.")
static PyObject *	cp_SparseVirtualFileSystem_svf_lru_punt_all (cp_SparseVirtualFileSystem *self, PyObject *args, PyObject *kwargs)
	PyDoc_STRVAR (cp_SparseVirtualFileSystem_svf_file_mod_time_matches_docstring, "file_mod_time_matches(self, id: str) -> bool\n\n" "Returns True if the file modification time of the Sparse Virtual File identified by the given id the matches" "the given time as a float.\n" "This will raise an ``IndexError`` if the Sparse Virtual File of that id does not exist.")
static PyObject *	cp_SparseVirtualFileSystem_svf_file_mod_time_matches (cp_SparseVirtualFileSystem *self, PyObject *args, PyObject *kwargs)
	PyDoc_STRVAR (cp_SparseVirtualFileSystem_file_mod_time_docstring, "file_mod_time(self, id: str) -> float\n\n" "Returns the file modification time as a float in UNIX time of the Sparse Virtual File identified by the given id.\n" "This will raise an ``IndexError`` if the Sparse Virtual File of that id does not exist.")
static PyObject *	cp_SparseVirtualFileSystem_file_mod_time (cp_SparseVirtualFileSystem *self, PyObject *args, PyObject *kwargs)
	SVFS_SVFS_METHOD_SIZE_T_WRAPPER (count_write, "Returns the count of write operations on the Sparse Virtual File identified by the given id." " This will raise an ``IndexError`` if the Sparse Virtual File of that id does not exist.")
	SVFS_SVFS_METHOD_SIZE_T_WRAPPER (count_read, "Returns the count of read operations on the Sparse Virtual File identified by the given id." " This will raise an ``IndexError`` if the Sparse Virtual File of that id does not exist.")
	SVFS_SVFS_METHOD_SIZE_T_WRAPPER (bytes_write, "Returns the count of the number of bytes writen to the Sparse Virtual File identified by the given id." " This will raise an ``IndexError`` if the Sparse Virtual File of that id does not exist.")
	SVFS_SVFS_METHOD_SIZE_T_WRAPPER (bytes_read, "Returns the count of the number of bytes read from the Sparse Virtual File identified by the given id." " This will raise an ``IndexError`` if the Sparse Virtual File of that id does not exist.")
	PyDoc_STRVAR (cp_SparseVirtualFileSystem_svf_time_write_docstring, "time_write(self, id: str) -> typing.Optional[datetime.datetime]\n\n" "Returns the timestamp of the last write to the Sparse Virtual File identified by the given id as a datetime.datetime." "or None if no write has taken place.\n" "This will raise an ``IndexError`` if the Sparse Virtual File of that id does not exist.")
static PyObject *	cp_SparseVirtualFileSystem_svf_time_write (cp_SparseVirtualFileSystem *self, PyObject *args, PyObject *kwargs)
	PyDoc_STRVAR (cp_SparseVirtualFileSystem_svf_time_read_docstring, "time_read(self, id: str) -> typing.Optional[datetime.datetime]\n\n" "Returns the timestamp of the last read from the Sparse Virtual File identified by the given id as a datetime.datetime." "or None if no read has taken place.\n" "This will raise an ``IndexError`` if the Sparse Virtual File of that id does not exist.")
static PyObject *	cp_SparseVirtualFileSystem_svf_time_read (cp_SparseVirtualFileSystem *self, PyObject *args, PyObject *kwargs)
	PyDoc_STRVAR (cp_SparseVirtualFileSystem_config_docstring, "config(self) -> typing.Dict[str, bool]\n\n" "Returns the SVFS configuration as a dict.")
static PyObject *	cp_SparseVirtualFileSystem_config (cp_SparseVirtualFileSystem *self)
static Py_ssize_t	cp_SparseVirtualFileSystem_mapping_length (PyObject *self)
	PyDoc_STRVAR (svfs_cSVFS_doc, "This class implements a Sparse Virtual File System where Sparse Virtual Files are mapped to a key (a string).\n" "This can be constructed with an optional boolean overwrite flag that ensures in-memory data is overwritten" " on destruction of any SVF.")

Variables
static PyMemberDef	cp_SparseVirtualFileSystem_members []
static PyMethodDef	cp_SparseVirtualFileSystem_methods []
PyMappingMethods	svfs_mapping_methods
static PyTypeObject	svfsc_cSVFS

Detailed Description

A Sparse Virtual File implementation.

   MIT License

   Copyright (c) 2020-2024 Paul Ross

   Permission is hereby granted, free of charge, to any person obtaining a copy
   of this software and associated documentation files (the "Software"), to deal
   in the Software without restriction, including without limitation the rights
   to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
   copies of the Software, and to permit persons to whom the Software is
   furnished to do so, subject to the following conditions:

   The above copyright notice and this permission notice shall be included in all
   copies or substantial portions of the Software.

   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
   IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
   FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
   AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
   LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
   OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
   SOFTWARE.

Naming convention:

SVFS functions are named cp_SparseVirtualFileSystem_... SVF functions are named cp_SparseVirtualFileSystem_svf_...

Definition in file _cSVFS.cpp.

Macro Definition Documentation

ASSERT_FUNCTION_ENTRY_SVFS

#define ASSERT_FUNCTION_ENTRY_SVFS

(

member

)

Value:

    do { \
    assert(self); \
    assert(((cp_SparseVirtualFileSystem *)self)->member); \
    assert(! PyErr_Occurred()); \
} while (0)

Definition at line 145 of file _cSVFS.cpp.

SVFS_SVFS_METHOD_DATETIME_WRAPPER

#define SVFS_SVFS_METHOD_DATETIME_WRAPPER

(

method_name

)

Definition at line 1237 of file _cSVFS.cpp.

SVFS_SVFS_METHOD_SIZE_T_WRAPPER

#define SVFS_SVFS_METHOD_SIZE_T_WRAPPER	(		method_name,
			docstring
	)

This macro is for functions that return a size_t type such as count_write, count_read, bytes_write, bytes_read.

Definition at line 50 of file _cSVFS.cpp.

Function Documentation

cp_SparseVirtualFileSystem_config()

static PyObject* cp_SparseVirtualFileSystem_config ( cp_SparseVirtualFileSystem *  self )

static

Definition at line 1400 of file _cSVFS.cpp.

cp_SparseVirtualFileSystem_dealloc()

static void cp_SparseVirtualFileSystem_dealloc ( cp_SparseVirtualFileSystem *  self )

static

Definition at line 211 of file _cSVFS.cpp.

cp_SparseVirtualFileSystem_file_mod_time()

static PyObject* cp_SparseVirtualFileSystem_file_mod_time ( cp_SparseVirtualFileSystem *  self, PyObject *  args, PyObject *  kwargs  )

static

Definition at line 1176 of file _cSVFS.cpp.

cp_SparseVirtualFileSystem_has()

static PyObject* cp_SparseVirtualFileSystem_has ( cp_SparseVirtualFileSystem *  self, PyObject *  args, PyObject *  kwargs  )

static

Definition at line 373 of file _cSVFS.cpp.

cp_SparseVirtualFileSystem_init()

static int cp_SparseVirtualFileSystem_init ( cp_SparseVirtualFileSystem *  self, PyObject *  args, PyObject *  kwargs  )

static

Definition at line 173 of file _cSVFS.cpp.

cp_SparseVirtualFileSystem_insert()

static PyObject* cp_SparseVirtualFileSystem_insert ( cp_SparseVirtualFileSystem *  self, PyObject *  args, PyObject *  kwargs  )

static

Definition at line 287 of file _cSVFS.cpp.

cp_SparseVirtualFileSystem_keys()

static PyObject* cp_SparseVirtualFileSystem_keys ( cp_SparseVirtualFileSystem *  self )

static

Definition at line 235 of file _cSVFS.cpp.

cp_SparseVirtualFileSystem_mapping_length()

static Py_ssize_t cp_SparseVirtualFileSystem_mapping_length ( PyObject *  self )

static

Definition at line 1418 of file _cSVFS.cpp.

cp_SparseVirtualFileSystem_new()

static PyObject* cp_SparseVirtualFileSystem_new ( PyTypeObject *  type, PyObject *  Py_UNUSEDargs, PyObject *  Py_UNUSEDkwds  )

static

Definition at line 156 of file _cSVFS.cpp.

cp_SparseVirtualFileSystem_remove()

static PyObject* cp_SparseVirtualFileSystem_remove ( cp_SparseVirtualFileSystem *  self, PyObject *  args, PyObject *  kwargs  )

static

Definition at line 331 of file _cSVFS.cpp.

cp_SparseVirtualFileSystem_svf_block_touches()

static PyObject* cp_SparseVirtualFileSystem_svf_block_touches ( cp_SparseVirtualFileSystem *  self, PyObject *  args, PyObject *  kwargs  )

static

See cp_SparseVirtualFile_lru_punt_docstring

Parameters

self	The cp_SparseVirtualFile
args	The upper bound of the number of bytes held by the cache.
kwargs	"cache_size_upper_bound".

Returns

Number of bytes removed.

Definition at line 930 of file _cSVFS.cpp.

cp_SparseVirtualFileSystem_svf_blocks()

static PyObject* cp_SparseVirtualFileSystem_svf_blocks ( cp_SparseVirtualFileSystem *  self, PyObject *  args, PyObject *  kwargs  )

static

Definition at line 838 of file _cSVFS.cpp.

cp_SparseVirtualFileSystem_svf_erase()

static PyObject* cp_SparseVirtualFileSystem_svf_erase ( cp_SparseVirtualFileSystem *  self, PyObject *  args, PyObject *  kwargs  )

static

Definition at line 646 of file _cSVFS.cpp.

cp_SparseVirtualFileSystem_svf_file_mod_time_matches()

static PyObject* cp_SparseVirtualFileSystem_svf_file_mod_time_matches ( cp_SparseVirtualFileSystem *  self, PyObject *  args, PyObject *  kwargs  )

static

Definition at line 1109 of file _cSVFS.cpp.

cp_SparseVirtualFileSystem_svf_has_data()

static PyObject* cp_SparseVirtualFileSystem_svf_has_data ( cp_SparseVirtualFileSystem *  self, PyObject *  args, PyObject *  kwargs  )

static

Definition at line 467 of file _cSVFS.cpp.

cp_SparseVirtualFileSystem_svf_lru_punt()

static PyObject* cp_SparseVirtualFileSystem_svf_lru_punt ( cp_SparseVirtualFileSystem *  self, PyObject *  args, PyObject *  kwargs  )

static

See cp_SparseVirtualFile_lru_punt_docstring

Parameters

self	The cp_SparseVirtualFile
args	The upper bound of the number of bytes held by the cache.
kwargs	"cache_size_upper_bound".

Returns

Number of bytes removed.

Definition at line 1003 of file _cSVFS.cpp.

cp_SparseVirtualFileSystem_svf_lru_punt_all()

static PyObject* cp_SparseVirtualFileSystem_svf_lru_punt_all ( cp_SparseVirtualFileSystem *  self, PyObject *  args, PyObject *  kwargs  )

static

See cp_SparseVirtualFile_lru_punt_docstring

Parameters

self	The cp_SparseVirtualFile
args	The upper bound of the number of bytes held by the cache.
kwargs	"cache_size_upper_bound".

Returns

Number of bytes removed.

Definition at line 1062 of file _cSVFS.cpp.

cp_SparseVirtualFileSystem_svf_need()

static PyObject* cp_SparseVirtualFileSystem_svf_need ( cp_SparseVirtualFileSystem *  self, PyObject *  args, PyObject *  kwargs  )

static

Definition at line 710 of file _cSVFS.cpp.

cp_SparseVirtualFileSystem_svf_need_many()

static PyObject* cp_SparseVirtualFileSystem_svf_need_many ( cp_SparseVirtualFileSystem *  self, PyObject *  args, PyObject *  kwargs  )

static

Definition at line 764 of file _cSVFS.cpp.

cp_SparseVirtualFileSystem_svf_read()

static PyObject* cp_SparseVirtualFileSystem_svf_read ( cp_SparseVirtualFileSystem *  self, PyObject *  args, PyObject *  kwargs  )

static

Definition at line 585 of file _cSVFS.cpp.

cp_SparseVirtualFileSystem_svf_time_read()

static PyObject* cp_SparseVirtualFileSystem_svf_time_read ( cp_SparseVirtualFileSystem *  self, PyObject *  args, PyObject *  kwargs  )

static

Definition at line 1343 of file _cSVFS.cpp.

cp_SparseVirtualFileSystem_svf_time_write()

static PyObject* cp_SparseVirtualFileSystem_svf_time_write ( cp_SparseVirtualFileSystem *  self, PyObject *  args, PyObject *  kwargs  )

static

Definition at line 1282 of file _cSVFS.cpp.

cp_SparseVirtualFileSystem_svf_write()

static PyObject* cp_SparseVirtualFileSystem_svf_write ( cp_SparseVirtualFileSystem *  self, PyObject *  args, PyObject *  kwargs  )

static

Definition at line 523 of file _cSVFS.cpp.

cp_SparseVirtualFileSystem_total_blocks()

static PyObject* cp_SparseVirtualFileSystem_total_blocks ( cp_SparseVirtualFileSystem *  self )

static

Definition at line 447 of file _cSVFS.cpp.

cp_SparseVirtualFileSystem_total_bytes()

static PyObject* cp_SparseVirtualFileSystem_total_bytes ( cp_SparseVirtualFileSystem *  self )

static

Definition at line 430 of file _cSVFS.cpp.

cp_SparseVirtualFileSystem_total_size_of()

static PyObject* cp_SparseVirtualFileSystem_total_size_of ( cp_SparseVirtualFileSystem *  self )

static

Definition at line 413 of file _cSVFS.cpp.

PyDoc_STRVAR() [1/23]

PyDoc_STRVAR	(	cp_SparseVirtualFileSystem_config_docstring	,
		"config(self) -> typing.Dict\n\n" "Returns the SVFS configuration as a dict."	[str, bool]
	)

PyDoc_STRVAR() [2/23]

PyDoc_STRVAR	(	cp_SparseVirtualFileSystem_file_mod_time_docstring	,
		"file_mod_time(self, id: str) -> float\n\n" "Returns the file modification time as a float in UNIX time of the Sparse Virtual File identified by the given id.\n" "This will raise an ``IndexError`` if the Sparse Virtual File of that id does not exist."
	)

-— Attribute access -— const std::string id() const noexcept { return m_id; } double file_mod_time() const noexcept { return m_file_mod_time; }

size_t count_write() const noexcept { return m_count_write; } size_t count_read() const noexcept { return m_count_read; } size_t bytes_write() const noexcept { return m_bytes_write; } size_t bytes_read() const noexcept { return m_bytes_read; }

std::chrono::time_point<std::chrono::system_clock> time_write() const noexcept { return m_time_write; } std::chrono::time_point<std::chrono::system_clock> time_read() const noexcept { return m_time_read; }

PyDoc_STRVAR() [3/23]

PyDoc_STRVAR	(	cp_SparseVirtualFileSystem_has_docstring	,
		"has(self, id: str) -> bool\n\n" "Returns True if the Sparse Virtual File for the ID is in the Sparse Virtual File System."
	)

PyDoc_STRVAR() [4/23]

PyDoc_STRVAR	(	cp_SparseVirtualFileSystem_insert_docstring	,
		"insert(self, id: str) -> None\n\n" "Inserts a Sparse Virtual File of ID and Unix file modification time as a float."
	)

PyDoc_STRVAR() [5/23]

PyDoc_STRVAR	(	cp_SparseVirtualFileSystem_keys_docstring	,
		"keys(self) -> typing.List\n\n" "Returns the IDs of all the Sparse Virtual Files in the Sparse Virtual File System."	[str]
	)

PyDoc_STRVAR() [6/23]

PyDoc_STRVAR	(	cp_SparseVirtualFileSystem_remove_docstring	,
		"remove(self, id: str) -> None\n\n" "Removes a Sparse Virtual File of ID freeing that file's memory. Will raise an ``IndexError`` if the ID is absent."
	)

PyDoc_STRVAR() [7/23]

PyDoc_STRVAR	(	cp_SparseVirtualFileSystem_svf_block_touches_docstring	,
		"block_touches(self, id: str) -> typing.Dict\n\n" "This returns a dict ``{touch_int: file_position, ...}``" " of the touch integer of each block mapped to the file position.\n" "The caller can decide what older blocks can be used the erase(file_position)."	[int, int]
	)

PyDoc_STRVAR() [8/23]

PyDoc_STRVAR	(	cp_SparseVirtualFileSystem_svf_blocks_docstring	,
		"blocks(self, id: str) -> typing.	Tuple[typing.Tuple[int, int],
		...]\n\n" "This returns a ordered tuple((file_position, length),...) of all the blocks held by the SVF identified" "by the given id.\n" "This will raise an ``IndexError`` if the Sparse Virtual File of that id does not exist."
	)

-— Meta information about the SVF -— The existing blocks. t_seek_reads blocks() const noexcept;

Information about memory used: size_of() gives best guess of total memory usage. size_t size_of() const noexcept;

Gives exact number of data bytes held. size_t num_bytes() const noexcept { return m_bytes_total; };

Gives exact number of blocks used. size_t num_blocks() const noexcept { return m_svf.size(); } t_fpos last_file_position() const noexcept;

Check the clients file modification time has changed. Caller has to decide what to do... bool file_mod_time_matches(const double &file_mod_time) const noexcept { return file_mod_time == m_file_mod_time; }

PyDoc_STRVAR() [9/23]

PyDoc_STRVAR	(	cp_SparseVirtualFileSystem_svf_erase_docstring	,
		"erase(self, id: str, file_position: int) -> None\n\n" "Erases the data block in the Sparse Virtual File at a file position.\n" "This takes a string as an id and a file_position.\n" "This will raise an ``IndexError`` if the Sparse Virtual File of that id does not exist.\n" "This will raise an ``IOError`` if there is not a block at the position.\n" "This will raise a ``RuntimeError`` if the data can not be read for any other reason.\n"
	)

PyDoc_STRVAR() [10/23]

PyDoc_STRVAR	(	cp_SparseVirtualFileSystem_svf_file_mod_time_matches_docstring	,
		"file_mod_time_matches(self, id: str) -> bool\n\n" "Returns True if the file modification time of the Sparse Virtual File identified by the given id the matches" "the given time as a float.\n" "This will raise an ``IndexError`` if the Sparse Virtual File of that id does not exist."
	)

PyDoc_STRVAR() [11/23]

PyDoc_STRVAR	(	cp_SparseVirtualFileSystem_svf_has_data_docstring	,
		"has_data(self, id: str, file_position: int, length: int) -> bool\n\n" "Checks if the Sparse Virtual File of the ID has data at the given file_position and length." " This takes a string as an	id,
		a file position and a length." " This returns True if the Sparse Virtual File of that id has the	data,
		False otherwise." " This will raise an ``IndexError`` if the SVF of that id does not exist."
	)

PyDoc_STRVAR() [12/23]

PyDoc_STRVAR	(	cp_SparseVirtualFileSystem_svf_lru_punt_all_docstring	,
		"lru_punt_all(self, cache_size_upper_bound: int) -> int\n\n" "Reduces the size of all IDs in the cache to < the given size by removing older blocks." " At least one block will be left for each ID.\n" "There are limitations to this tactic, see the documentation in Technical Notes -> Cache Punting."
	)

PyDoc_STRVAR() [13/23]

PyDoc_STRVAR	(	cp_SparseVirtualFileSystem_svf_lru_punt_docstring	,
		"lru_punt(self, id: str, cache_size_upper_bound: int) -> int\n\n" "Reduces the size of the cache to < the given size by removing older blocks, at least one block will be left.\n" "There are limitations to this tactic, see the documentation in Technical Notes -> Cache Punting."
	)

PyDoc_STRVAR() [14/23]

PyDoc_STRVAR	(	cp_SparseVirtualFileSystem_svf_need_docstring	,
		"need(self, id: str, file_position: int, length: int, greedy_length: int = 0) -> typing.	Tuple[typing.Tuple[int, int],
		...]\n\n" "Given a file_position and length this returns a ordered list ```` of seek/read" " instructions of data that is required to be written to the Sparse Virtual File so that a subsequent read will succeed.\n" "If greedy_length	is[(file_position, length),...],
		0	then,
		if	possible,
		blocks will be coalesced to reduce the size of the return value." "\nUsage::\n\n" " if not svfs.has(identity, file_position, length):\n" " for	fpos,
		read_len in svfs.need(identity, file_position, length):\n" " # Somehow get the data at that seek/read position...\n" " svfs.write(identity, fpos, data)\n" " return svfs.read(identity, file_position, length):\n"
	)

PyDoc_STRVAR() [15/23]

PyDoc_STRVAR	(	cp_SparseVirtualFileSystem_svf_need_many_docstring	,
		"need_many(self, id: str, seek_reads: typing.List[typing.Tuple[int, int]], greedy_length: int = 0) -> typing.	Tuple[typing.Tuple[int, int],
		...]\n\n" "Given a list of(file_position, length) this returns a ordered list ```` of seek/read" " instructions of data that is required to be written to the Sparse Virtual File so that a subsequent read will" " succeed.\n\n" "If greedy_length	is[(file_position, length),...],
		0	then,
		if	possible,
		blocks will be coalesced to reduce the size of the return value." "\n\n" "See also :py:meth:`svfsc.cSVFS.need`"
	)

PyDoc_STRVAR() [16/23]

PyDoc_STRVAR	(	cp_SparseVirtualFileSystem_svf_read_docstring	,
		"read(self, id: str, file_position: int, length: int) -> bytes\n\n" "Read the data to the Sparse Virtual File at file_position and length returning a bytes object.\n" "This takes a string as an	id,
		a file position and a length.\n" "\nThis will raise an ``IndexError`` if the Sparse Virtual File of that id does not exist.\n" "This will raise an ``IOError`` if any data is not present\n" "This will raise a ``RuntimeError`` if the data can not be read for any other reason.\n"
	)

PyDoc_STRVAR() [17/23]

PyDoc_STRVAR	(	cp_SparseVirtualFileSystem_svf_time_read_docstring	,
		"time_read(self, id: str) -> typing.Optional\n\n" "Returns the timestamp of the last read from the Sparse Virtual File identified by the given id as a datetime.datetime." "or None if no read has taken place.\n" "This will raise an ``IndexError`` if the Sparse Virtual File of that id does not exist."	[datetime.datetime]
	)

PyDoc_STRVAR() [18/23]

PyDoc_STRVAR	(	cp_SparseVirtualFileSystem_svf_time_write_docstring	,
		"time_write(self, id: str) -> typing.Optional\n\n" "Returns the timestamp of the last write to the Sparse Virtual File identified by the given id as a datetime.datetime." "or None if no write has taken place.\n" "This will raise an ``IndexError`` if the Sparse Virtual File of that id does not exist."	[datetime.datetime]
	)

PyDoc_STRVAR() [19/23]

PyDoc_STRVAR	(	cp_SparseVirtualFileSystem_svf_write_docstring	,
		"write(self, id: str, file_position: int, data: bytes) -> None\n\n" "Writes the data to the Sparse Virtual File of the given ID at file_position and length.\n\n" "This takes a string as an	id,
		a file position and data as a bytes object.\n" "This will raise an ``IndexError`` if the SVF of that id does not exist.\n" "This will raise an ``IOError`` if the given data is different than that seen before and only\n" "new data up to this point will be written.\n" "This will raise a ``RuntimeError`` if the data can not be written for any other reason.\n"
	)

PyDoc_STRVAR() [20/23]

PyDoc_STRVAR	(	cp_SparseVirtualFileSystem_total_blocks_docstring	,
		"total_blocks(self) -> int\n\n" "Returns the total number of blocks of data held by the Sparse Virtual File System."
	)

PyDoc_STRVAR() [21/23]

PyDoc_STRVAR	(	cp_SparseVirtualFileSystem_total_bytes_docstring	,
		"total_bytes(self) -> int\n\n" "Returns the total number of file bytes held by the Sparse Virtual File System."
	)

PyDoc_STRVAR() [22/23]

PyDoc_STRVAR	(	cp_SparseVirtualFileSystem_total_size_of_docstring	,
		"total_size_of(self) -> int\n\n" "Returns the estimate of total memory usage of the Sparse Virtual File System."
	)

PyDoc_STRVAR() [23/23]

PyDoc_STRVAR	(	svfs_cSVFS_doc	,
		"This class implements a Sparse Virtual File System where Sparse Virtual Files are mapped to a key (a string).\n" "This can be constructed with an optional boolean overwrite flag that ensures in-memory data is overwritten" " on destruction of any SVF."
	)

SVFS_SVFS_METHOD_SIZE_T_WRAPPER() [1/7]

SVFS_SVFS_METHOD_SIZE_T_WRAPPER	(	bytes_read	,
		"Returns the count of the number of bytes read from the Sparse Virtual File identified by the given id." " This will raise an ``IndexError`` if the Sparse Virtual File of that id does not exist."
	)

SVFS_SVFS_METHOD_SIZE_T_WRAPPER() [2/7]

SVFS_SVFS_METHOD_SIZE_T_WRAPPER	(	bytes_write	,
		"Returns the count of the number of bytes writen to the Sparse Virtual File identified by the given id." " This will raise an ``IndexError`` if the Sparse Virtual File of that id does not exist."
	)

SVFS_SVFS_METHOD_SIZE_T_WRAPPER() [3/7]

SVFS_SVFS_METHOD_SIZE_T_WRAPPER	(	count_read	,
		"Returns the count of read operations on the Sparse Virtual File identified by the given id." " This will raise an ``IndexError`` if the Sparse Virtual File of that id does not exist."
	)

SVFS_SVFS_METHOD_SIZE_T_WRAPPER() [4/7]

SVFS_SVFS_METHOD_SIZE_T_WRAPPER	(	count_write	,
		"Returns the count of write operations on the Sparse Virtual File identified by the given id." " This will raise an ``IndexError`` if the Sparse Virtual File of that id does not exist."
	)

SVFS_SVFS_METHOD_SIZE_T_WRAPPER() [5/7]

SVFS_SVFS_METHOD_SIZE_T_WRAPPER	(	num_blocks	,
		"Returns the number of data blocks held by the Sparse Virtual File identified by the given id." " This will raise an ``IndexError`` if the Sparse Virtual File of that id does not exist."
	)

SVFS_SVFS_METHOD_SIZE_T_WRAPPER() [6/7]

SVFS_SVFS_METHOD_SIZE_T_WRAPPER	(	num_bytes	,
		"Returns the number of bytes of data held by the Sparse Virtual File identified by the given id." " This will raise an ``IndexError`` if the Sparse Virtual File of that id does not exist."
	)

SVFS_SVFS_METHOD_SIZE_T_WRAPPER() [7/7]

SVFS_SVFS_METHOD_SIZE_T_WRAPPER	(	size_of	,
		"Returns the best guess of total memory usage used by the Sparse Virtual File identified by the given id." " This will raise an ``IndexError`` if the Sparse Virtual File of that id does not exist."
	)

Variable Documentation

cp_SparseVirtualFileSystem_members

PyMemberDef cp_SparseVirtualFileSystem_members[]

static

Initial value:

= {
 
 
        {NULL, 0, 0, 0, NULL}  
}

Definition at line 1428 of file _cSVFS.cpp.

cp_SparseVirtualFileSystem_methods

PyMethodDef cp_SparseVirtualFileSystem_methods[]

static

Definition at line 1434 of file _cSVFS.cpp.

svfs_mapping_methods

PyMappingMethods svfs_mapping_methods

Initial value:

= {
        
        
        .mp_length = &cp_SparseVirtualFileSystem_mapping_length,
}

Definition at line 1581 of file _cSVFS.cpp.

svfsc_cSVFS

PyTypeObject svfsc_cSVFS

static

Initial value:

= {
        PyVarObject_HEAD_INIT(NULL, 0)
        .tp_name = "svfsc.cSVFS",
        .tp_basicsize = sizeof(cp_SparseVirtualFileSystem),
        .tp_itemsize = 0,
        .tp_dealloc = (destructor) cp_SparseVirtualFileSystem_dealloc,
        
        .tp_as_mapping = &svfs_mapping_methods,
        .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE,
        .tp_doc = svfs_cSVFS_doc,
        .tp_methods = cp_SparseVirtualFileSystem_methods,
        .tp_members = cp_SparseVirtualFileSystem_members,
        .tp_init = (initproc) cp_SparseVirtualFileSystem_init,
        .tp_new = cp_SparseVirtualFileSystem_new,
}

Definition at line 1598 of file _cSVFS.cpp.