From 8a0aa1f77f063159feabc6594c8a64ddfddf1149 Mon Sep 17 00:00:00 2001 From: Doug Coleman Date: Sat, 9 Dec 2017 13:47:37 -0600 Subject: [PATCH] forestdb: not a maintained db --- forestdb/ffi/authors.txt | 1 + forestdb/ffi/ffi.factor | 468 ++++++++++++++++++++++++++++++ forestdb/lib/authors.txt | 1 + forestdb/lib/lib-tests.factor | 381 ++++++++++++++++++++++++ forestdb/lib/lib.factor | 446 ++++++++++++++++++++++++++++ forestdb/paths/authors.txt | 1 + forestdb/paths/paths-tests.factor | 39 +++ forestdb/paths/paths.factor | 76 +++++ forestdb/utils/authors.txt | 1 + forestdb/utils/utils.factor | 69 +++++ 10 files changed, 1483 insertions(+) create mode 100644 forestdb/ffi/authors.txt create mode 100644 forestdb/ffi/ffi.factor create mode 100644 forestdb/lib/authors.txt create mode 100644 forestdb/lib/lib-tests.factor create mode 100644 forestdb/lib/lib.factor create mode 100644 forestdb/paths/authors.txt create mode 100644 forestdb/paths/paths-tests.factor create mode 100644 forestdb/paths/paths.factor create mode 100644 forestdb/utils/authors.txt create mode 100644 forestdb/utils/utils.factor diff --git a/forestdb/ffi/authors.txt b/forestdb/ffi/authors.txt new file mode 100644 index 0000000..7c1b2f2 --- /dev/null +++ b/forestdb/ffi/authors.txt @@ -0,0 +1 @@ +Doug Coleman diff --git a/forestdb/ffi/ffi.factor b/forestdb/ffi/ffi.factor new file mode 100644 index 0000000..28cf608 --- /dev/null +++ b/forestdb/ffi/ffi.factor @@ -0,0 +1,468 @@ +! Copyright (C) 2014 Doug Coleman. +! See http://factorcode.org/license.txt for BSD license. +USING: alien alien.c-types alien.libraries +alien.libraries.finder alien.syntax classes.struct kernel ; +IN: forestdb.ffi + +! Functions with LIBFDB_API are exported. + +<< "forestdb" dup find-library cdecl add-library >> + +LIBRARY: forestdb + +! Begin fdb_types.h + +CONSTANT: FDB_MAX_KEYLEN 3840 +CONSTANT: FDB_MAX_METALEN 65512 +CONSTANT: FDB_MAX_BODYLEN 4294967295 +CONSTANT: FDB_SNAPSHOT_INMEM -1 + +TYPEDEF: uint64_t fdb_seqnum_t +TYPEDEF: int64_t cs_off_t +TYPEDEF: long fdb_ssize_t ! XXX: platform dependent? + +TYPEDEF: void* fdb_custom_cmp_fixed +TYPEDEF: void* fdb_custom_cmp_variable +TYPEDEF: void* fdb_file_handle +TYPEDEF: void* fdb_kvs_handle +TYPEDEF: void* fdb_iterator +TYPEDEF: void* fdb_changes_callback_fn +TYPEDEF: void* voidref +TYPEDEF: void* fdb_handle_stats_cb + +ENUM: fdb_open_flags < uint32_t + { FDB_OPEN_FLAG_CREATE 1 } + { FDB_OPEN_FLAG_RDONLY 2 } + { FDB_OPEN_WITH_LEGACY_CRC 4 } ; + +ENUM: fdb_commit_opt_t < uint8_t + { FDB_COMMIT_NORMAL 0 } + { FDB_COMMIT_MANUAL_WAL_FLUSH 1 } ; + +ENUM: fdb_seqtree_opt_t < uint8_t + { FDB_SEQTREE_NOT_USE 0 } + { FDB_SEQTREE_USE 1 } ; + +ENUM: fdb_durability_opt_t < uint8_t + { FDB_DRB_NONE 0 } + { FDB_DRB_ODIRECT 1 } + { FDB_DRB_ASYNC 2 } + { FDB_DRB_ODIRECT_ASYNC 3 } ; + +ENUM: fdb_compaction_mode_t < uint8_t + { FDB_COMPACTION_MANUAL 0 } + { FDB_COMPACTION_AUTO 1 } ; + +ENUM: fdb_isolation_level_t < uint8_t + { FDB_ISOLATION_READ_COMMITTED 2 } + { FDB_ISOLATION_READ_UNCOMMITTED 3 } ; + +ENUM: fdb_iterator_opt_t < uint16_t + { FDB_ITR_NONE 0 } + { FDB_ITR_NO_DELETES 2 } + { FDB_ITR_SKIP_MIN_KEY 4 } + { FDB_ITR_SKIP_MAX_KEY 8 } + { FDB_ITR_NO_VALUES 0x10 } ; ! only keys and metadata for fdb_changes_since + +ENUM: fdb_changes_decision < int32_t + { FDB_CHANGES_PRESERVE 1 } + { FDB_CHANGES_CLEAN 0 } + { FDB_CHANGES_CANCEL -1 } ; + +ENUM: fdb_iterator_seek_opt_t < uint8_t + { FDB_ITR_SEEK_HIGHER 0 } + { FDB_ITR_SEEK_LOWER 1 } ; + +ENUM: fdb_compaction_status < uint32_t + { FDB_CS_BEGIN 0x1 } + { FDB_CS_MOVE_DOC 0x2 } + { FDB_CS_BATCH_MOVE 0x4 } + { FDB_CS_FLUSH_WAL 0x8 } + { FDB_CS_END 0x10 } + { FDB_CS_COMPLETE 0x20 } ; + +ENUM: fdb_compact_decision < int + { FDB_CS_KEEP_DOC 0 } + { FDB_CS_DROP_DOC 1 } ; + +ENUM: fdb_encryption_algorithm_t < int + { FDB_ENCRYPTION_NONE 0 } + { FDB_ENCRYPTION_AES256 1 } ; + +STRUCT: fdb_doc + { keylen size_t } + { metalen size_t } + { bodylen size_t } + { size_ondisk size_t } + { key void* } + { seqnum fdb_seqnum_t } + { offset uint64_t } + { meta void* } + { body void* } + { deleted bool } + { flags uint32_t } ; + +CALLBACK: void fdb_log_callback ( int err_code, char* err_msg, void* ctx_data ) +CALLBACK: void fdb_fatal_error_callback ( ) +CALLBACK: fdb_compact_decision fdb_compaction_callback ( + fdb_file_handle* fhandle, + fdb_compaction_status status, + char* kv_store_name, + fdb_doc* doc, + uint64_t last_oldfile_offset, + uint64_t last_newfile_offset, + void* ctx ) + +STRUCT: fdb_encryption_key + { algorithm fdb_encryption_algorithm_t } + { bytes uint8_t[32] } ; + +STRUCT: fdb_filemgr_ops_t + { constructor void* } + { open void* } + { pwrite void* } + { pread void* } + { close void* } + { goto_eof void* } + { file_size void* } + { fdatasync void* } + { sync void* } + { get_errno_str void* } + { mmap void* } + { munmap void* } + { aio_init void* } + { aio_prep_read void* } + { aio_submit void* } + { aio_getevents void* } + { aio_destroy void* } + { get_fs_type void* } + { copy_file_range void* } + { destructor void* } + { ctx void* } ; + +! cmp_fixed and cmp_variable have their own open() functions +STRUCT: fdb_config + { chunksize uint16_t } + { blocksize uint32_t } + { buffercache_size uint64_t } + { wal_threshold uint64_t } + { wal_flush_before_commit bool } + { auto_commit bool } + { purging_interval uint32_t } + { seqtree_opt fdb_seqtree_opt_t } + { durability_opt fdb_durability_opt_t } + { flags fdb_open_flags } + { compaction_buf_maxsize uint32_t } + { cleanup_cache_onclose bool } + { compress_document_body bool } + { compaction_mode fdb_compaction_mode_t } + { compaction_threshold uint8_t } + { compaction_minimum_filesize uint64_t } + { compactor_sleep_duration uint64_t } + { multi_kv_instances bool } + { prefetch_duration uint64_t } + { num_wal_partitions uint16_t } + { num_bcache_partitions uint16_t } + { compaction_cb fdb_compaction_callback } + { compaction_cb_mask uint32_t } + { compaction_cb_ctx void* } + { max_writer_lock_prob size_t } + { num_compactor_threads size_t } + { num_bgflusher_threads size_t } + { encryption_key fdb_encryption_key } + { block_reusing_threshold size_t } + { num_keeping_headers size_t } + { breakpad_minidump_dir char* } + { custom_file_ops fdb_filemgr_ops_t* } + { num_background_threads size_t } + { bcache_flush_limit size_t } ; + +STRUCT: fdb_kvs_config + { create_if_missing bool } + { custom_cmp fdb_custom_cmp_variable } ; + +! filename is a pointer to the handle's filename +! new_filename is a pointer to the handle's new_file + +STRUCT: fdb_file_info + { filename char* } + { new_filename char* } + { doc_count uint64_t } + { deleted_count uint64_t } + { space_used uint64_t } + { file_size uint64_t } + { num_kv_stores size_t } ; + +STRUCT: fdb_kvs_info + { name char* } + { last_seqnum fdb_seqnum_t } + { doc_count uint64_t } + { deleted_count uint64_t } + { space_used uint64_t } + { file fdb_file_handle* } ; + +STRUCT: fdb_kvs_name_list + { num_kvs_names size_t } + { kvs_names char** } ; + +STRUCT: fdb_kvs_ops_info + { num_sets uint64_t } + { num_dels uint64_t } + { num_commits uint64_t } + { num_compacts uint64_t } + { num_gets uint64_t } + { num_iterator_gets uint64_t } + { num_iterator_moves uint64_t } ; + +ENUM: fdb_latency_stat_type < uint8_t + { FDB_LATENCY_SETS 0 } + { FDB_LATENCY_GETS 1 } + { FDB_LATENCY_COMMITS 2 } + { FDB_LATENCY_SNAP_INMEM 3 } + { FDB_LATENCY_SNAP_DUR 4 } + { FDB_LATENCY_COMPACTS 5 } + { FDB_LATENCY_ITR_INIT 6 } + { FDB_LATENCY_ITR_SEQ_INIT 7 } + { FDB_LATENCY_ITR_NEXT 8 } + { FDB_LATENCY_ITR_PREV 9 } + { FDB_LATENCY_ITR_GET 10 } + { FDB_LATENCY_ITR_GET_META 11 } + { FDB_LATENCY_ITR_SEEK 12 } + { FDB_LATENCY_ITR_SEEK_MAX 13 } + { FDB_LATENCY_ITR_SEEK_MIN 14 } + { FDB_LATENCY_ITR_CLOSE 15 } + { FDB_LATENCY_OPEN 16 } + { FDB_LATENCY_KVS_OPEN 17 } + { FDB_LATENCY_SNAP_CLONE 18 } + { FDB_LATENCY_WAL_INS 19 } + { FDB_LATENCY_WAL_FIND 20 } + { FDB_LATENCY_WAL_COMMIT 21 } + { FDB_LATENCY_WAL_FLUSH 22 } + { FDB_LATENCY_WAL_RELEASE 23 } + { FDB_LATENCY_NUM_STATS 24 } ; + +STRUCT: fdb_latency_stat + { lat_count uint64_t } + { lat_min uint32_t } + { lat_max uint32_t } + { lat_avg uint32_t } ; + +STRUCT: fdb_kvs_commit_marker_t + { kv_store_name char* } + { seqnum fdb_seqnum_t } ; + +TYPEDEF: uint64_t fdb_snapshot_marker_t + +STRUCT: fdb_snapshot_info_t + { marker fdb_snapshot_marker_t } + { num_kvs_markers int64_t } + { kvs_markers fdb_kvs_commit_marker_t* } ; + +! end fdb_types.h + +! Begin fdb_errors.h +ENUM: fdb_status + { FDB_RESULT_SUCCESS 0 } + { FDB_RESULT_INVALID_ARGS -1 } + { FDB_RESULT_OPEN_FAIL -2 } + { FDB_RESULT_NO_SUCH_FILE -3 } + { FDB_RESULT_WRITE_FAIL -4 } + { FDB_RESULT_READ_FAIL -5 } + { FDB_RESULT_CLOSE_FAIL -6 } + { FDB_RESULT_COMMIT_FAIL -7 } + { FDB_RESULT_ALLOC_FAIL -8 } + { FDB_RESULT_KEY_NOT_FOUND -9 } + { FDB_RESULT_RONLY_VIOLATION -10 } + { FDB_RESULT_COMPACTION_FAIL -11 } + { FDB_RESULT_ITERATOR_FAIL -12 } + { FDB_RESULT_SEEK_FAIL -13 } + { FDB_RESULT_FSYNC_FAIL -14 } + { FDB_RESULT_CHECKSUM_ERROR -15 } + { FDB_RESULT_FILE_CORRUPTION -16 } + { FDB_RESULT_COMPRESSION_FAIL -17 } + { FDB_RESULT_NO_DB_INSTANCE -18 } + { FDB_RESULT_FAIL_BY_ROLLBACK -19 } + { FDB_RESULT_INVALID_CONFIG -20 } + { FDB_RESULT_MANUAL_COMPACTION_FAIL -21 } + { FDB_RESULT_INVALID_COMPACTION_MODE -22 } + { FDB_RESULT_FILE_IS_BUSY -23 } + { FDB_RESULT_FILE_REMOVE_FAIL -24 } + { FDB_RESULT_FILE_RENAME_FAIL -25 } + { FDB_RESULT_TRANSACTION_FAIL -26 } + { FDB_RESULT_FAIL_BY_TRANSACTION -27 } + { FDB_RESULT_FAIL_BY_COMPACTION -28 } + { FDB_RESULT_TOO_LONG_FILENAME -29 } + { FDB_RESULT_INVALID_HANDLE -30 } + { FDB_RESULT_KV_STORE_NOT_FOUND -31 } + { FDB_RESULT_KV_STORE_BUSY -32 } + { FDB_RESULT_INVALID_KV_INSTANCE_NAME -33 } + { FDB_RESULT_INVALID_CMP_FUNCTION -34 } + { FDB_RESULT_IN_USE_BY_COMPACTOR -35 } + { FDB_RESULT_FILE_NOT_OPEN -36 } + { FDB_RESULT_TOO_BIG_BUFFER_CACHE -37 } + { FDB_RESULT_NO_DB_HEADERS -38 } + { FDB_RESULT_HANDLE_BUSY -39 } + { FDB_RESULT_AIO_NOT_SUPPORTED -40 } + { FDB_RESULT_AIO_INIT_FAIL -41 } + { FDB_RESULT_AIO_SUBMIT_FAIL -42 } + { FDB_RESULT_AIO_GETEVENTS_FAIL -43 } + { FDB_RESULT_CRYPTO_ERROR -44 } + { FDB_RESULT_COMPACTION_CANCELLATION -45 } + { FDB_RESULT_SB_INIT_FAIL -46 } + { FDB_RESULT_SB_RACE_CONDITION -47 } + { FDB_RESULT_SB_READ_FAIL -48 } + { FDB_RESULT_FILE_VERSION_NOT_SUPPORTED -49 } + { FDB_RESULT_EPERM -50 } + { FDB_RESULT_EIO -51 } + { FDB_RESULT_ENXIO -52 } + { FDB_RESULT_EBADF -53 } + { FDB_RESULT_ENOMEM -54 } + { FDB_RESULT_EACCESS -55 } + { FDB_RESULT_EFAULT -56 } + { FDB_RESULT_EEXIST -57 } + { FDB_RESULT_ENODEV -58 } + { FDB_RESULT_ENOTDIR -59 } + { FDB_RESULT_EISDIR -60 } + { FDB_RESULT_EINVAL -61 } + { FDB_RESULT_ENFILE -62 } + { FDB_RESULT_EMFILE -63 } + { FDB_RESULT_EFBIG -64 } + { FDB_RESULT_ENOSPC -65 } + { FDB_RESULT_EROFS -66 } + { FDB_RESULT_EOPNOTSUPP -67 } + { FDB_RESULT_ENOBUFS -68 } + { FDB_RESULT_ELOOP -69 } + { FDB_RESULT_ENAMETOOLONG -70 } + { FDB_RESULT_EOVERFLOW -71 } + { FDB_RESULT_EAGAIN -72 } + { FDB_RESULT_CANCELLED -73 } + { FDB_RESULT_ENGINE_NOT_INSTANTIATED -74 } + { FDB_RESULT_LOG_FILE_NOT_FOUND -75 } + { FDB_RESULT_LOCK_FAIL -76 } + { FDB_RESULT_LAST -76 } + ; ! update this + +! End fdb_errors.h + +! Begin forestdb.h +FUNCTION: fdb_status fdb_init ( fdb_config* config ) +FUNCTION: fdb_config fdb_get_default_config ( ) +FUNCTION: fdb_kvs_config fdb_get_default_kvs_config ( ) + +FUNCTION: fdb_status fdb_open ( fdb_file_handle** ptr_fhandle, c-string filename, fdb_config* fconfig ) +FUNCTION: fdb_status fdb_open_custom_cmp ( fdb_file_handle** ptr_fhandle, c-string filename, fdb_config* fconfig, size_t num_functions, char** kvs_names, fdb_custom_cmp_variable* functions ) + +FUNCTION: fdb_status fdb_set_log_callback ( fdb_kvs_handle* handle, fdb_log_callback log_callback, void* ctx_data ) + +FUNCTION: void fdb_set_fatal_error_callback ( fdb_fatal_error_callback err_callback ) + +! doc is calloc'd +FUNCTION: fdb_status fdb_doc_create ( fdb_doc** doc, void* key, size_t keylen, void* meta, size_t metalen, void* body, size_t bodylen ) +FUNCTION: fdb_status fdb_doc_update ( fdb_doc** doc, void* meta, size_t metalen, void* body, size_t bodylen ) +FUNCTION: fdb_status fdb_doc_set_seqnum ( fdb_doc* doc, fdb_seqnum_t seqnum ) +FUNCTION: fdb_status fdb_doc_free ( fdb_doc* doc ) + +FUNCTION: fdb_status fdb_get ( fdb_kvs_handle* handle, fdb_doc* doc ) +FUNCTION: fdb_status fdb_get_metaonly ( fdb_kvs_handle* handle, fdb_doc* doc ) +FUNCTION: fdb_status fdb_get_byseq ( fdb_kvs_handle* handle, fdb_doc* doc ) +FUNCTION: fdb_status fdb_get_metaonly_byseq ( fdb_kvs_handle* handle, fdb_doc* doc ) +FUNCTION: fdb_status fdb_get_byoffset ( fdb_kvs_handle* handle, fdb_doc* doc ) + +FUNCTION: fdb_status fdb_set ( fdb_kvs_handle* handle, fdb_doc* doc ) +FUNCTION: fdb_status fdb_del ( fdb_kvs_handle* handle, fdb_doc* doc ) + +FUNCTION: fdb_status fdb_get_kv ( fdb_kvs_handle* handle, void* key, size_t keylen, void** value_out, size_t* valuelen_out ) +FUNCTION: fdb_status fdb_set_kv ( fdb_kvs_handle* handle, void* key, size_t keylen, void* value, size_t valuelen ) +FUNCTION: fdb_status fdb_del_kv ( fdb_kvs_handle* handle, void* key, size_t keylen ) +FUNCTION: fdb_status fdb_free_block ( void *ptr ) + +FUNCTION: fdb_status fdb_commit ( fdb_file_handle* fhandle, fdb_commit_opt_t opt ) +FUNCTION: fdb_status fdb_snapshot_open ( fdb_kvs_handle* handle_in, fdb_kvs_handle** handle_out, fdb_seqnum_t snapshot_seqnum ) +! Swaps out the handle for a new one +FUNCTION: fdb_status fdb_rollback ( fdb_kvs_handle** handle_ptr, fdb_seqnum_t rollback_seqnum ) +FUNCTION: fdb_status fdb_rollback_all ( fdb_file_handle* fhandle, fdb_snapshot_marker_t marker ) + +FUNCTION: fdb_status fdb_iterator_init ( fdb_kvs_handle* handle, fdb_iterator** iterator, void* min_key, size_t min_keylen, void* max_key, size_t max_keylen, fdb_iterator_opt_t opt ) +FUNCTION: fdb_status fdb_iterator_sequence_init ( fdb_kvs_handle* handle, fdb_iterator** iterator, fdb_seqnum_t min_seq, fdb_seqnum_t max_seq, fdb_iterator_opt_t opt ) + +FUNCTION: fdb_status fdb_iterator_prev ( fdb_iterator* iterator ) +FUNCTION: fdb_status fdb_iterator_next ( fdb_iterator* iterator ) +FUNCTION: fdb_status fdb_iterator_get ( fdb_iterator* iterator, fdb_doc **doc ) +FUNCTION: fdb_status fdb_iterator_get_metaonly ( fdb_iterator* iterator, fdb_doc **doc ) + +FUNCTION: fdb_status fdb_iterator_seek ( fdb_iterator* iterator, void* seek_key, size_t seek_keylen, fdb_iterator_seek_opt_t direction ) +FUNCTION: fdb_status fdb_iterator_seek_to_min ( fdb_iterator* iterator ) +FUNCTION: fdb_status fdb_iterator_seek_to_max ( fdb_iterator* iterator ) +FUNCTION: fdb_status fdb_iterator_close ( fdb_iterator* iterator ) + +FUNCTION: fdb_status fdb_changes_since ( fdb_kvs_handle *handle, + fdb_seqnum_t since, + fdb_iterator_opt_t opt, + fdb_changes_callback_fn callback, + void *ctx ) +FUNCTION: fdb_status fdb_compact ( fdb_file_handle* fhandle, c-string new_filename ) +FUNCTION: fdb_status fdb_compact_with_cow ( fdb_file_handle* fhandle, c-string new_filename ) +FUNCTION: fdb_status fdb_compact_upto ( fdb_file_handle* fhandle, c-string new_filename, fdb_snapshot_marker_t marker ) +FUNCTION: fdb_status fdb_compact_upto_with_cow ( fdb_file_handle* fhandle, c-string new_filename, fdb_snapshot_marker_t marker ) +FUNCTION: fdb_status fdb_cancel_compaction ( fdb_file_handle* fhandle ) +FUNCTION: fdb_status fdb_set_daemon_compaction_interval ( fdb_file_handle* fhandle, size_t interval ) +FUNCTION: fdb_status fdb_rekey ( fdb_file_handle* fhandle, fdb_encryption_key new_key ) +FUNCTION: size_t fdb_get_buffer_cache_used ( ) +FUNCTION: size_t fdb_estimate_space_used ( fdb_file_handle* fhandle ) +FUNCTION: size_t fdb_estimate_space_used_from ( fdb_file_handle* fhandle, fdb_snapshot_marker_t marker ) + +FUNCTION: fdb_status fdb_get_file_info ( fdb_file_handle* fhandle, fdb_file_info* info ) +FUNCTION: fdb_status fdb_get_kvs_info ( fdb_kvs_handle* handle, fdb_kvs_info* info ) +FUNCTION: fdb_status fdb_get_kvs_ops_info ( fdb_kvs_handle* handle, fdb_kvs_ops_info* info ) +FUNCTION: fdb_status fdb_get_latency_stats ( fdb_file_handle* fhandle, fdb_latency_stat* stats, fdb_latency_stat_type type ) +FUNCTION: fdb_status fdb_get_latency_histogram ( fdb_file_handle* fhandle, + char** stats, + size_t* stats_length, + fdb_latency_stat_type type ) +FUNCTION: c-string fdb_get_latency_stat_name ( fdb_latency_stat_type type ) +FUNCTION: fdb_status fdb_get_kvs_seqnum ( fdb_kvs_handle* handle, fdb_seqnum_t* seqnum ) +FUNCTION: fdb_status fdb_get_kvs_name_list ( fdb_kvs_handle* handle, fdb_kvs_name_list* kvs_name_list ) + +FUNCTION: fdb_status fdb_get_all_snap_markers ( + fdb_file_handle* fhandle, + fdb_snapshot_info_t** markers, + uint64_t* size ) + +FUNCTION: fdb_seqnum_t fdb_get_available_rollback_seq ( + fdb_kvs_handle* handle, + uint64_t request_seqno ) + +FUNCTION: fdb_status fdb_free_snap_markers ( fdb_snapshot_info_t* markers, uint64_t size ) +FUNCTION: fdb_status fdb_free_kvs_name_list ( fdb_kvs_name_list* kvs_name_list ) + +FUNCTION: fdb_status fdb_switch_compaction_mode ( fdb_file_handle* fhandle, fdb_compaction_mode_t mode, size_t new_threshold ) +FUNCTION: fdb_status fdb_close ( fdb_file_handle* fhandle ) + +FUNCTION: fdb_status fdb_destroy ( c-string filename, fdb_config* fconfig ) +FUNCTION: fdb_status fdb_shutdown ( ) + +FUNCTION: fdb_status fdb_begin_transaction ( fdb_file_handle* fhandle, fdb_isolation_level_t isolation_level ) +FUNCTION: fdb_status fdb_end_transaction ( fdb_file_handle* fhandle, fdb_commit_opt_t opt ) +FUNCTION: fdb_status fdb_abort_transaction ( fdb_file_handle* fhandle ) +FUNCTION: fdb_status fdb_kvs_open ( fdb_file_handle* fhandle, + fdb_kvs_handle** ptr_handle, + c-string kvs_name, + fdb_kvs_config* config ) + +FUNCTION: fdb_status fdb_kvs_open_default ( fdb_file_handle* fhandle, + fdb_kvs_handle** ptr_handle, + fdb_kvs_config* config ) + +FUNCTION: fdb_status fdb_kvs_close ( fdb_kvs_handle* handle ) + +FUNCTION: fdb_status fdb_kvs_remove ( fdb_file_handle* fhandle, c-string kvs_name ) + +FUNCTION: fdb_status fdb_set_block_reusing_params ( fdb_file_handle* fhandle, size_t block_reusing_threshold, size_t num_keeping_headers ) +FUNCTION: char* fdb_error_msg ( fdb_status err_code ) +FUNCTION: char* fdb_get_lib_version ( ) +FUNCTION: char* fdb_get_file_version ( fdb_file_handle* fhandle ) +FUNCTION: fdb_filemgr_ops_t* fdb_get_default_file_ops ( ) + +FUNCTION: fdb_status fdb_fetch_handle_stats ( fdb_kvs_handle *handle, + fdb_handle_stats_cb callback, + void *ctx ) diff --git a/forestdb/lib/authors.txt b/forestdb/lib/authors.txt new file mode 100644 index 0000000..7c1b2f2 --- /dev/null +++ b/forestdb/lib/authors.txt @@ -0,0 +1 @@ +Doug Coleman diff --git a/forestdb/lib/lib-tests.factor b/forestdb/lib/lib-tests.factor new file mode 100644 index 0000000..73a2a46 --- /dev/null +++ b/forestdb/lib/lib-tests.factor @@ -0,0 +1,381 @@ +! Copyright (C) 2014 Doug Coleman. +! See http://factorcode.org/license.txt for BSD license. +USING: accessors alien.strings arrays forestdb.lib forestdb.utils kernel +make multiline sequences tools.test ; +IN: forestdb.lib.tests + +! Get/set by key/value +{ "val123" } [ + [ + "test123" [ + "key123" "val123" fdb-set-kv + "key123" fdb-get-kv + ] with-kvs-name + ] with-forestdb-test-manual +] unit-test + +{ "val12345" } [ + [ + "test123" [ + "key123" "val12345" fdb-set-kv + "key123" fdb-get-kv + ] with-kvs-name + ] with-forestdb-test-manual +] unit-test + +! Get + +{ + { "key1" "val" } +} [ + [ + 5 set-kv-n + fdb-commit-normal + "key1" "meta" "val" [ + fdb_doc>doc [ key>> ] [ body>> ] bi 2array + ] with-create-doc + ] with-forestdb-test-manual +] unit-test + + +{ + { "key1" f "val1" } +} [ + [ + 5 set-kv-n + fdb-commit-normal + "key1" "no meta" "going away" [ + fdb-get + fdb_doc>doc [ key>> ] [ meta>> ] [ body>> ] tri 3array + ] with-create-doc + ] with-forestdb-test-manual +] unit-test + + +{ + { "key2" f "val2" } +} [ + [ + 5 set-kv-n + fdb-commit-normal + 2 [ + fdb-get-byseq fdb_doc>doc + [ key>> ] [ meta>> ] [ body>> ] tri 3array + ] with-doc + ] with-forestdb-test-manual +] unit-test + +{ + { "key2" f "val2" } +} [ + [ + 5 set-kv-n + fdb-commit-normal + 2 [ + fdb-get-byseq fdb_doc>doc + [ key>> ] [ meta>> ] [ body>> ] tri 3array + ] with-doc + ] with-forestdb-test-manual +] unit-test + +! Filename is only valid inside with-forestdb +{ f } [ + [ + fdb-get-info filename>> alien>native-string empty? + ] with-forestdb-test-manual +] unit-test + +! Test fdb_doc_create +{ 6 9 9 } [ + [ + "key123" "meta blah" "some body" [ + [ keylen>> ] [ metalen>> ] [ bodylen>> ] tri + ] with-create-doc + ] with-forestdb-test-manual +] unit-test + +{ 7 8 15 } [ + [ + "key1234" "meta blah" "some body" [ + [ "new meta" "some other body" fdb-doc-update ] + [ [ keylen>> ] [ metalen>> ] [ bodylen>> ] tri ] bi + ] with-create-doc + ] with-forestdb-test-manual +] unit-test + +{ 1 1 } [ + [ + 1 set-kv-n + fdb-commit-normal + fdb-get-kvs-info [ last_seqnum>> ] [ doc_count>> ] bi + ] with-forestdb-test-manual +] unit-test + +{ 6 5 } [ + [ + 5 set-kv-n + 5 set-kv-nth + fdb-commit-normal + fdb-get-kvs-info [ last_seqnum>> ] [ doc_count>> ] bi + ] with-forestdb-test-manual +] unit-test + +{ 5 5 } [ + [ + 5 set-kv-n + fdb-commit-normal + fdb-get-kvs-info [ last_seqnum>> ] [ doc_count>> ] bi + ] with-forestdb-test-manual +] unit-test + +! Snapshots + +/* +{ 5 5 } [ + [ + 5 set-kv-n + fdb-commit-normal + FDB_SNAPSHOT_INMEM [ + fdb-get-kvs-info [ last_seqnum>> ] [ doc_count>> ] bi + ] with-forestdb-snapshot + ] with-forestdb-test-manual +] unit-test +*/ + + +/* +! Snapshots can only occur on commits. If you commit five keys at once, +! and then try to open a snapshot on the second key, it should fail. + +! XXX: Buggy, fails in _fdb_open with FDB_RESULT_NO_DB_INSTANCE +[ + delete-test-db-1 + test-db-1 [ + 5 set-kv-n + fdb-commit-normal + FDB_SNAPSHOT_INMEM [ + fdb-get-kvs-info [ last_seqnum>> ] [ doc_count>> ] bi + ] with-forestdb-snapshot + ] with-forestdb-tester +] [ + T{ fdb-error { error FDB_RESULT_NO_DB_INSTANCE } } = +] must-fail-with + +! Test that we take two snapshots and their seqnums/doc counts are right. +! XXX: Buggy, want to see the first snapshot's document count at 5 too +{ + { 7 7 } + { 7 7 } +} [ + delete-test-db-1 + test-db-1 [ + 5 set-kv-n + fdb-commit-normal + + 6 7 set-kv-range + fdb-commit-normal + + FDB_SNAPSHOT_INMEM [ + fdb-get-kvs-info [ last_seqnum>> ] [ doc_count>> ] bi 2array + ] with-forestdb-snapshot + + FDB_SNAPSHOT_INMEM [ + fdb-get-kvs-info [ last_seqnum>> ] [ doc_count>> ] bi 2array + ] with-forestdb-snapshot + ] with-forestdb-tester +] unit-test + + +! Same test as above, but with buggy behavior for now so it passes +{ + 7 + 7 +} [ + delete-test-db-1 + test-db-1 [ + 5 set-kv-n + fdb-commit-normal + + 6 7 set-kv-range + fdb-commit-normal + + FDB_SNAPSHOT_INMEM [ + fdb-get-kvs-info last_seqnum>> + ] with-forestdb-snapshot + + FDB_SNAPSHOT_INMEM [ + fdb-get-kvs-info last_seqnum>> + ] with-forestdb-snapshot + ] with-forestdb-tester +] unit-test + + + + +! Rollback test +! Make sure the doc_count is correct after a rollback +{ + 7 + { 5 5 } +} [ + delete-test-db-1 + test-db-1 [ + 5 set-kv-n + fdb-commit-normal + + 6 7 set-kv-range + fdb-commit-normal + + FDB_SNAPSHOT_INMEM [ + fdb-get-kvs-info last_seqnum>> + ] with-forestdb-snapshot + + 5 fdb-rollback + + FDB_SNAPSHOT_INMEM [ + fdb-get-kvs-info [ last_seqnum>> ] [ doc_count>> ] bi 2array + ] with-forestdb-snapshot + ] with-forestdb-tester +] unit-test + +*/ + + +! Iterators test +! No matching keys +{ + { } +} [ + [ + 5 set-kv-n + fdb-commit-normal + [ + "omg" "nada" [ + fdb_doc>doc [ seqnum>> ] [ key>> ] [ body>> ] tri 3array , + ] with-fdb-normal-iterator + ] { } make + ] with-forestdb-test-manual +] unit-test + +! All the keys +{ + { + { 1 "key1" "val1" } + { 2 "key2" "val2" } + { 3 "key3" "val3" } + { 4 "key4" "val4" } + { 5 "key5" "val5" } + } +} [ + [ + 5 set-kv-n + fdb-commit-normal + [ + "key1" "key5" [ + fdb_doc>doc [ seqnum>> ] [ key>> ] [ body>> ] tri 3array , + ] with-fdb-normal-iterator + ] { } make + ] with-forestdb-test-manual +] unit-test + +! Test that keys at extremes get returned +{ + { + { 1 "key1" "val1" } + } +} [ + [ + 5 set-kv-n + fdb-commit-normal + [ + "key0" "key1" [ + fdb_doc>doc [ seqnum>> ] [ key>> ] [ body>> ] tri 3array , + ] with-fdb-normal-iterator + ] { } make + ] with-forestdb-test-manual +] unit-test + +{ + { + { 5 "key5" "val5" } + } +} [ + [ + 5 set-kv-n + fdb-commit-normal + [ + "key5" "key9" [ + fdb_doc>doc [ seqnum>> ] [ key>> ] [ body>> ] tri 3array , + ] with-fdb-normal-iterator + ] { } make + ] with-forestdb-test-manual +] unit-test + + +! Test byseq mapping +{ + V{ 1 2 3 4 5 } +} [ + [ + 5 set-kv-n + fdb-commit-normal + 0 10 [ + fdb_doc>doc + ] with-fdb-byseq-map + [ seqnum>> ] map + ] with-forestdb-test-manual +] unit-test + +! XXX: Behavior changed here +! No longer makes new docs that are deleted +! Deleting 5 keys gives you 5 new seqnums that are those docs, but deleted +! { + ! V{ { 6 t } { 7 t } { 8 t } { 9 t } { 10 t } } +! } [ + ! [ + ! 5 set-kv-n + ! 5 del-kv-n + ! fdb-commit-normal + ! 0 10 [ + ! fdb_doc>doc + ! ] with-fdb-byseq-map + ! [ [ seqnum>> ] [ deleted?>> ] bi 2array ] map + ! ] with-forestdb-test-manual +! ] unit-test + +! Test new behavior +{ + V{ } +} [ + [ + 5 set-kv-n + 5 del-kv-n + fdb-commit-normal + 0 10 [ + fdb_doc>doc + ] with-fdb-byseq-map + [ [ seqnum>> ] [ deleted?>> ] bi 2array ] map + ] with-forestdb-test-manual +] unit-test + +{ + { + { 1 "key1" } + { 2 "key2" } + { 3 "key3" } + { 4 "key4" } + { 5 "key5" } + } +} +[ + [ + 5 set-kv-n + fdb-commit-normal + [ + 0 10 [ + [ seqnum>> ] + [ [ key>> ] [ keylen>> ] bi alien/length>string ] bi 2array , + ] with-fdb-byseq-each + ] { } make + ] with-forestdb-test-manual +] unit-test diff --git a/forestdb/lib/lib.factor b/forestdb/lib/lib.factor new file mode 100644 index 0000000..50e7144 --- /dev/null +++ b/forestdb/lib/lib.factor @@ -0,0 +1,446 @@ +! Copyright (C) 2014 Doug Coleman. +! See http://factorcode.org/license.txt for BSD license. +USING: accessors alien.c-types alien.data alien.strings byte-arrays +classes.struct combinators constructors continuations destructors +forestdb.ffi fry generalizations io.directories io.encodings.string +io.encodings.utf8 io.pathnames kernel libc math multiline namespaces +sequences strings ; +QUALIFIED: sets +IN: forestdb.lib + +/* +! Issues +! Get byseq ignores seqnum and uses key instead if key is set +*/ + +ERROR: fdb-error error ; + +: fdb-check-error ( ret -- ) + dup FDB_RESULT_SUCCESS = [ drop ] [ fdb-error ] if ; + + +TUPLE: fdb-kvs-handle < disposable handle ; +: ( handle -- obj ) + fdb-kvs-handle new-disposable + swap >>handle ; inline + +M: fdb-kvs-handle dispose* + handle>> fdb_kvs_close fdb-check-error ; + + +TUPLE: fdb-file-handle < disposable handle ; +: ( handle -- obj ) + fdb-file-handle new-disposable + swap >>handle ; inline + +M: fdb-file-handle dispose* + handle>> fdb_close fdb-check-error ; + + +SYMBOL: current-fdb-file-handle +SYMBOL: current-fdb-kvs-handle + +: get-file-handle ( -- handle ) + current-fdb-file-handle get handle>> ; + +: get-kvs-handle ( -- handle ) + current-fdb-kvs-handle get handle>> ; + +GENERIC: encode-kv ( object -- bytes ) + +M: string encode-kv utf8 encode ; +M: byte-array encode-kv ; + +: fdb-set-kv ( key value -- ) + [ get-kvs-handle ] 2dip + [ encode-kv dup length ] bi@ fdb_set_kv fdb-check-error ; + +: ( key -- doc ) + fdb_doc malloc-struct + swap [ utf8 malloc-string >>key ] [ length >>keylen ] bi ; + +: ( seqnum -- doc ) + fdb_doc malloc-struct + swap >>seqnum ; + +! Fill in document by exemplar +: fdb-get ( doc -- doc ) + [ get-kvs-handle ] dip [ fdb_get fdb-check-error ] keep ; + +: fdb-get-metaonly ( doc -- doc ) + [ get-kvs-handle ] dip [ fdb_get_metaonly fdb-check-error ] keep ; + +: fdb-get-byseq ( doc -- doc ) + [ get-kvs-handle ] dip [ fdb_get_byseq fdb-check-error ] keep ; + +: fdb-get-metaonly-byseq ( doc -- doc ) + [ get-kvs-handle ] dip [ fdb_get_metaonly_byseq fdb-check-error ] keep ; + +: fdb-get-byoffset ( doc -- doc ) + [ get-kvs-handle ] dip [ fdb_get_byoffset fdb-check-error ] keep ; + + +! Set/delete documents +: fdb-set ( doc -- ) + [ get-kvs-handle ] dip fdb_set fdb-check-error ; + +: fdb-del ( doc -- ) + [ get-kvs-handle ] dip fdb_del fdb-check-error ; + +: ret>string ( void** len -- string ) + [ void* deref ] [ size_t deref ] bi* + memory>byte-array utf8 decode ; + +: fdb-get-kv ( key -- value/f ) + [ get-kvs-handle ] dip + utf8 encode dup length f void* 0 size_t + [ fdb_get_kv ] 2keep + rot { + { FDB_RESULT_SUCCESS [ ret>string ] } + { FDB_RESULT_KEY_NOT_FOUND [ 2drop f ] } + [ fdb-error ] + } case ; + +: fdb-del-kv ( key -- ) + [ get-kvs-handle ] dip + utf8 encode dup length fdb_del_kv fdb-check-error ; + +: fdb-doc-create ( key meta body -- doc ) + [ f void* ] 3dip + [ utf8 encode dup length ] tri@ + [ fdb_doc_create fdb-check-error ] 7 nkeep 6 ndrop + void* deref fdb_doc memory>struct ; + +: fdb-doc-update ( doc meta body -- ) + [ void* ] 2dip + [ utf8 encode dup length ] bi@ + fdb_doc_update fdb-check-error ; + +: fdb-doc-free ( doc -- ) + fdb_doc_free fdb-check-error ; + +: clear-doc-key ( doc -- doc ) + [ dup [ (free) f ] when ] change-key + 0 >>keylen ; + +: with-doc ( doc quot: ( doc -- ) -- ) + over '[ _ _ [ _ fdb-doc-free rethrow ] recover ] call ; inline + +: with-create-doc ( key meta body quot: ( doc -- ) -- ) + [ fdb-doc-create ] dip with-doc ; inline + +: fdb-get-info ( -- fdb_file_info ) + get-file-handle + fdb_file_info [ fdb_get_file_info fdb-check-error ] keep ; + +: fdb-get-kvs-info ( -- fdb_kvs_info ) + get-kvs-handle + fdb_kvs_info [ fdb_get_kvs_info fdb-check-error ] keep ; + +: fdb-commit ( fdb_commit_opt_t -- ) + [ get-file-handle ] dip fdb_commit fdb-check-error ; + +: fdb-maybe-commit ( fdb_commit_opt_t/f -- ) + [ fdb-commit ] when* ; + +: fdb-commit-normal ( -- ) FDB_COMMIT_NORMAL fdb-commit ; + +: fdb-commit-wal-flush ( -- ) FDB_COMMIT_MANUAL_WAL_FLUSH fdb-commit ; + +: fdb-compact-to-path ( new-path -- ) + [ get-file-handle ] dip absolute-path + fdb_compact fdb-check-error ; + +: fdb-compact ( -- ) + get-file-handle f fdb_compact fdb-check-error ; + +: fdb-compact-commit-to-path ( path -- ) + fdb-compact-to-path fdb-commit-wal-flush ; + +: fdb-compact-commit ( -- ) + fdb-compact fdb-commit-wal-flush ; + + +! Call from within with-foresdb +: fdb-open-snapshot ( seqnum -- handle ) + [ + get-kvs-handle + f void* + ] dip [ + fdb_snapshot_open fdb-check-error + ] 2keep drop void* deref ; + +! fdb_rollback returns a new handle, so we +! have to replace our current handle with that one +! XXX: can't call dispose on old handle, library handles that +: fdb-rollback ( seqnum -- ) + [ get-kvs-handle void* ] dip + [ fdb_rollback fdb-check-error ] 2keep drop + void* deref current-fdb-kvs-handle set ; + + +TUPLE: fdb-iterator < disposable handle ; + +: ( handle -- obj ) + fdb-iterator new-disposable + swap >>handle ; inline + +M: fdb-iterator dispose* + handle>> fdb_iterator_close fdb-check-error ; + +: fdb-iterator-init ( start-key end-key fdb_iterator_opt_t -- iterator ) + [ get-kvs-handle f void* ] 3dip + [ [ utf8 encode dup length ] bi@ ] dip + [ fdb_iterator_init fdb-check-error ] 7 nkeep 5 ndrop nip + void* deref ; + +: fdb-iterator-byseq-init ( start-seq end-seq fdb_iterator_opt_t -- iterator ) + [ get-kvs-handle f void* ] 3dip + [ fdb_iterator_sequence_init fdb-check-error ] 5 nkeep 3 ndrop nip + void* deref ; + +: fdb-iterator-init-none ( start-key end-key -- iterator ) + FDB_ITR_NONE fdb-iterator-init ; + +: fdb-iterator-no-deletes ( start-key end-key -- iterator ) + FDB_ITR_NO_DELETES fdb-iterator-init ; + +: check-iterate-result ( fdb_status -- ? ) + { + { FDB_RESULT_SUCCESS [ t ] } + { FDB_RESULT_ITERATOR_FAIL [ f ] } + [ throw ] + } case ; + +! fdb_doc key, meta, body only valid inside with-forestdb +! so make a helper word to preserve them outside +TUPLE: fdb-doc seqnum keylen key metalen meta bodylen body deleted? offset size-ondisk ; + +CONSTRUCTOR: fdb-doc ( seqnum keylen key metalen meta bodylen body deleted? offset size-ondisk -- obj ) ; + +TUPLE: fdb-info filename new-filename doc-count space-used file-size ; +CONSTRUCTOR: fdb-info ( filename new-filename doc-count space-used file-size -- obj ) ; + +/* +! Example fdb_doc and converted doc +S{ fdb_doc + { keylen 4 } { metalen 0 } { bodylen 4 } { size_ondisk 0 } + { key ALIEN: 1002f2f10 } { seqnum 5 } { offset 4256 } + { meta ALIEN: 1002dc790 } { body f } { deleted f } +} +T{ doc + { seqnum 5 } + { keylen 4 } { key "key5" } + { metalen 0 } { bodylen 4 } + { offset 4256 } { size-ondisk 0 } +} +*/ + +: alien/length>string ( alien n -- string/f ) + [ drop f ] [ + over [ + memory>byte-array utf8 decode + ] [ + 2drop f + ] if + ] if-zero ; + +: fdb_doc>doc ( fdb_doc -- doc ) + { + [ seqnum>> ] + [ keylen>> ] + [ [ key>> ] [ keylen>> ] bi alien/length>string ] + [ metalen>> ] + [ [ meta>> ] [ metalen>> ] bi alien/length>string ] + [ bodylen>> ] + [ [ body>> ] [ bodylen>> ] bi alien/length>string ] + [ deleted>> >boolean ] + [ offset>> ] + [ size_ondisk>> ] + } cleave ; + +: fdb_file_info>info ( fdb_doc -- doc ) + { + [ filename>> alien>native-string ] + [ new_filename>> alien>native-string ] + [ doc_count>> ] + [ space_used>> ] + [ file_size>> ] + } cleave ; + +: fdb-iterator-get ( iterator -- doc/f ) + f void* + [ fdb_iterator_get check-iterate-result ] keep swap + [ void* deref fdb_doc memory>struct ] + [ drop f ] if ; + +: fdb-iterator-seek ( iterator key seek-opt -- ) + [ dup length ] dip fdb_iterator_seek fdb-check-error ; + +: fdb-iterator-seek-lower ( iterator key -- ) + FDB_ITR_SEEK_LOWER fdb-iterator-seek ; + +: fdb-iterator-seek-higher ( iterator key -- ) + FDB_ITR_SEEK_HIGHER fdb-iterator-seek ; + +: with-fdb-iterator ( start-key end-key fdb_iterator_opt_t iterator-init iterator-advance quot: ( obj -- ) -- ) + [ execute ] 2dip + swap + '[ + _ &dispose handle>> [ + [ fdb-iterator-get ] keep swap + [ _ with-doc _ execute check-iterate-result ] + [ drop f ] if* + ] curry loop + ] with-destructors ; inline + + + +: get-kvs-default-config ( -- kvs-config ) + fdb_get_default_kvs_config ; + +: fdb-open ( path config -- file-handle ) + [ f void* ] 2dip + [ make-parent-directories ] dip + [ fdb_open fdb-check-error ] 3keep + 2drop void* deref ; + +: fdb-config-normal-commit ( -- config ) + fdb_get_default_config + FDB_SEQTREE_USE >>seqtree_opt ; + +: fdb-config-auto-commit ( -- config ) + fdb-config-normal-commit + FDB_COMPACTION_AUTO >>compaction_mode + 1 >>compactor_sleep_duration + t >>auto_commit ; + +! Make SEQTREES by default +: fdb-open-auto-commit ( path -- file-handle ) + fdb-config-auto-commit fdb-open ; + +: fdb-open-normal-commit ( path -- file-handle ) + fdb-config-normal-commit fdb-open ; + +: fdb-kvs-open-config ( name config -- kvs-handle ) + [ + current-fdb-file-handle get handle>> + f void* + ] 2dip + [ fdb_kvs_open fdb-check-error ] 3keep 2drop + void* deref ; + +: fdb-kvs-open-default-config ( name -- kvs-handle ) + get-kvs-default-config fdb-kvs-open-config ; + +: with-fdb-map ( start-key end-key fdb_iterator_opt_t iterator-init iterator-next quot: ( obj -- ) -- ) + [ execute ] 2dip + swap + '[ + _ &dispose handle>> [ + [ fdb-iterator-get ] keep swap + [ _ with-doc swap _ execute check-iterate-result ] + [ drop f f ] if* swap + ] curry collector-when [ loop ] dip + ] with-destructors ; inline + +: with-fdb-normal-iterator ( start-key end-key quot -- ) + [ FDB_ITR_NONE \ fdb-iterator-init \ fdb_iterator_next ] dip + with-fdb-iterator ; inline + +: with-fdb-byseq-each ( start-seq end-seq quot -- ) + [ FDB_ITR_NONE \ fdb-iterator-byseq-init \ fdb_iterator_next ] dip + with-fdb-iterator ; inline + +: with-fdb-byseq-map ( start-seq end-seq quot -- ) + [ FDB_ITR_NONE \ fdb-iterator-byseq-init \ fdb_iterator_next ] dip + with-fdb-map ; inline + +! : changes-cb ( handle doc ctx -- changes_decision ) +! ; + +! : fdb-changes-since ( seqnum iterator_opt cb ctx -- seq ) +! f 101 FDB_ITR_NONE fdb_changes_since ; + + +: with-kvs-name-config ( name config quot -- ) + '[ + _ _ fdb-kvs-open-config &dispose current-fdb-kvs-handle _ with-variable + ] with-destructors ; inline + +: with-kvs-name ( name quot -- ) + [ fdb_get_default_kvs_config ] dip with-kvs-name-config ; inline + + +: with-forestdb-file-handle ( path config quot -- ) + '[ + _ _ fdb-open &dispose current-fdb-file-handle _ with-variable + ] with-destructors ; inline + +: with-forestdb-path-config-kvs-name-config ( path config kvs-name kvs-config quot -- ) + '[ + _ _ with-kvs-name-config + ] with-forestdb-file-handle ; inline + +: with-forestdb-path-config-kvs-name ( path config kvs-name quot -- ) + '[ + _ _ with-kvs-name + ] with-forestdb-file-handle ; inline + +/* +! Do not try to commit here, as it will fail with FDB_RESULT_RONLY_VIOLATION +! fdb-current is weird, it gets replaced if you call fdb-rollback +! Therefore, only clean up fdb-current once, and clean it up at the end +: with-forestdb-handles ( file-handle handle quot fdb_commit_opt_t/f -- ) + '[ + _ current-fdb-file-handle [ + _ current-fdb-kvs-handle [ + [ + @ + _ fdb-maybe-commit + current-fdb-file-handle get &dispose drop + current-fdb-kvs-handle get &dispose drop + ] [ + [ + current-fdb-file-handle get &dispose drop + current-fdb-kvs-handle get &dispose drop + ] with-destructors + rethrow + ] recover + ] with-variable + ] with-variable + ] with-destructors ; inline + +! XXX: When you don't commit-wal at the end of with-forestdb, it won't +! persist to disk for next time you open the db. +: with-forestdb-handles-commit-normal ( file-handle handle quot commit -- ) + FDB_COMMIT_NORMAL with-forestdb-handles ; inline + +: with-forestdb-handles-commit-wal ( file-handle handle quot commit -- ) + FDB_COMMIT_MANUAL_WAL_FLUSH with-forestdb-handles ; inline + +: with-forestdb-snapshot ( n quot -- ) + [ fdb-open-snapshot ] dip '[ + _ current-fdb-kvs-handle [ + [ + @ + current-fdb-kvs-handle get &dispose drop + ] [ + current-fdb-kvs-handle get [ &dispose drop ] when* + rethrow + ] recover + ] with-variable + ] with-destructors ; inline + +: with-forestdb-path ( path quot -- ) + [ absolute-path fdb-open-default-config ] dip with-forestdb-handles-commit-wal ; inline + ! [ absolute-path fdb-open-default-config ] dip with-forestdb-handle-commit-normal ; inline +*/ diff --git a/forestdb/paths/authors.txt b/forestdb/paths/authors.txt new file mode 100644 index 0000000..7c1b2f2 --- /dev/null +++ b/forestdb/paths/authors.txt @@ -0,0 +1 @@ +Doug Coleman diff --git a/forestdb/paths/paths-tests.factor b/forestdb/paths/paths-tests.factor new file mode 100644 index 0000000..c433982 --- /dev/null +++ b/forestdb/paths/paths-tests.factor @@ -0,0 +1,39 @@ +! Copyright (C) 2014 Doug Coleman. +! See http://factorcode.org/license.txt for BSD license. +USING: forestdb.paths kernel tools.test ; +IN: forestdb.paths.tests + +{ "1.fq.0" } [ "0.fq.0" next-vnode-name ] unit-test +{ "1.fq.0" } [ "0.fq.1" next-vnode-name ] unit-test +{ "100.fq.0" } [ "99.fq.0" next-vnode-name ] unit-test +{ "100.fq.0" } [ "99.fq.1" next-vnode-name ] unit-test +{ "100.fq.0" } [ "99.fq.20" next-vnode-name ] unit-test +{ "100.fq.0" } [ "099.fq.20" next-vnode-name ] unit-test +{ "0100.fq.0" } [ "0099.fq.20" next-vnode-name ] unit-test + +{ "00001.fq.0" } [ "00000.fq.0" next-vnode-name ] unit-test +{ "001.fq.0" } [ "000.fq.1" next-vnode-name ] unit-test +{ "000100.fq.0" } [ "000099.fq.0" next-vnode-name ] unit-test +{ "00100.fq.0" } [ "00099.fq.1" next-vnode-name ] unit-test +{ "00000000100.fq.0" } [ "00000000099.fq.20" next-vnode-name ] unit-test + +{ "0.fq.0" } [ "00.fq.00" canonical-fdb-name ] unit-test +{ "1.fq.0" } [ "01.fq.00" canonical-fdb-name ] unit-test +{ "0.fq.1" } [ "00.fq.01" canonical-fdb-name ] unit-test +{ "100.fq.10" } [ "000100.fq.010" canonical-fdb-name ] unit-test + +{ "0.fq.1" } [ "0.fq.0" next-vnode-version-name ] unit-test +{ "0.fq.2" } [ "0.fq.1" next-vnode-version-name ] unit-test +{ "99.fq.1" } [ "99.fq.0" next-vnode-version-name ] unit-test +{ "99.fq.2" } [ "99.fq.1" next-vnode-version-name ] unit-test +{ "99.fq.21" } [ "99.fq.20" next-vnode-version-name ] unit-test + +[ "fq" ensure-fdb-filename drop ] [ not-an-fdb-filename? ] must-fail-with +[ "0.fq" ensure-fdb-filename drop ] [ not-an-fdb-filename? ] must-fail-with +[ "0.fq." ensure-fdb-filename drop ] [ not-an-fdb-filename? ] must-fail-with +[ ".fq.0" ensure-fdb-filename drop ] [ not-an-fdb-filename? ] must-fail-with +[ "1fq.0" ensure-fdb-filename drop ] [ not-an-fdb-filename? ] must-fail-with +[ "1fq0" ensure-fdb-filename drop ] [ not-an-fdb-filename? ] must-fail-with +[ "1.fq0" ensure-fdb-filename drop ] [ not-an-fdb-filename? ] must-fail-with +[ "1.fq.0.0" ensure-fdb-filename drop ] [ not-an-fdb-filename? ] must-fail-with +[ "1.fq.00." ensure-fdb-filename drop ] [ not-an-fdb-filename? ] must-fail-with diff --git a/forestdb/paths/paths.factor b/forestdb/paths/paths.factor new file mode 100644 index 0000000..5eb3352 --- /dev/null +++ b/forestdb/paths/paths.factor @@ -0,0 +1,76 @@ +! Copyright (C) 2014 Doug Coleman. +! See http://factorcode.org/license.txt for BSD license. +USING: arrays combinators.short-circuit combinators.smart +io.directories io.pathnames kernel math math.parser sequences +sorting sorting.human splitting sets ; +IN: forestdb.paths + +CONSTANT: fdb-filename-base "fq" + +: fdb-filename? ( path -- ? ) + "." split { + [ length 3 = ] + [ second fdb-filename-base = ] + [ first string>number ] + [ third string>number ] + } 1&& ; + +ERROR: not-an-fdb-filename string ; + +: ensure-fdb-filename ( string -- string ) + dup fdb-filename? [ not-an-fdb-filename ] unless ; + +ERROR: not-a-string-number string ; + +: ?string>number ( string -- n ) + dup string>number [ ] [ not-a-string-number ] ?if ; + +: change-string-number ( string quot -- string' ) + [ [ string>number ] dip call number>string ] 2keep drop + length CHAR: 0 pad-head ; inline + +: next-vnode-name ( string -- string' ) + [ + "." split + first [ 1 + ] change-string-number + "." fdb-filename-base ".0" + ] "" append-outputs-as ; + +: trim-head-zeros ( string -- string' ) + [ CHAR: 0 = ] trim-head 1 CHAR: 0 pad-head ; + +: canonical-fdb-name ( string -- string' ) + ensure-fdb-filename + "." split first3 + [ trim-head-zeros ] + [ ] + [ trim-head-zeros ] tri* 3array "." join ; + +: next-vnode-version-name ( string -- string' ) + "." split + [ but-last "." join ] + [ last [ 1 + ] change-string-number ] bi "." glue ; + +: path>next-vnode-path ( path -- path' ) + dup directory-files + [ fdb-filename? ] filter + [ human<=> ] sort [ + fdb-filename-base "0." ".0" surround append-path + ] [ + last "." split first [ 1 + ] change-string-number + ".fq.0" append append-path + ] if-empty ; + +: path-has-fdb? ( path -- ? ) + directory-files [ fdb-filename? ] filter length 0 > ; + +: path-only-fdb? ( path -- ? ) + directory-files + [ length ] + [ [ fdb-filename? ] filter length ] bi = ; + +: path-fdb-duplicates ( path -- seq ) + directory-files [ canonical-fdb-name ] map members ; + +! : path>next-vnode-version-name ( path -- path' ) + ! [ file-name ] diff --git a/forestdb/utils/authors.txt b/forestdb/utils/authors.txt new file mode 100644 index 0000000..7c1b2f2 --- /dev/null +++ b/forestdb/utils/authors.txt @@ -0,0 +1 @@ +Doug Coleman diff --git a/forestdb/utils/utils.factor b/forestdb/utils/utils.factor new file mode 100644 index 0000000..1963fc6 --- /dev/null +++ b/forestdb/utils/utils.factor @@ -0,0 +1,69 @@ +! Copyright (C) 2014 Doug Coleman. +! See http://factorcode.org/license.txt for BSD license. +USING: accessors assocs continuations forestdb.ffi forestdb.lib +fry io.directories io.files.temp io.files.unique +io.files.unique.private io.pathnames kernel locals math.parser +math.ranges namespaces sequences splitting ; +IN: forestdb.utils + +: fdb-test-config-seqtree-auto ( -- config ) + fdb_get_default_config + FDB_COMPACTION_AUTO >>compaction_mode + FDB_SEQTREE_USE >>seqtree_opt ; + +: fdb-test-config-seqtree-manual ( -- config ) + fdb_get_default_config + FDB_COMPACTION_MANUAL >>compaction_mode + FDB_SEQTREE_USE >>seqtree_opt ; + +! Manual naming scheme: foo.db +: do-forestdb-test-db-manual ( config quot -- ) + '[ + "forestdb-test-manual" ".db" [ + _ + "default" _ with-forestdb-path-config-kvs-name + ] cleanup-unique-file + ] with-temp-directory ; inline + +! Auto naming scheme: foo.db.0 foo.db.meta +: do-forestdb-test-db-auto ( config quot -- ) + '[ + "forestdb-test-auto" { ".db.0" ".db.meta" } [ + first ".0" ?tail drop + _ "default" _ with-forestdb-path-config-kvs-name + ] cleanup-unique-files + ] with-temp-directory ; inline + +: with-forestdb-test-db ( config quot -- ) + over [ + do-forestdb-test-db-manual + ] [ + do-forestdb-test-db-auto + ] if ; inline + +: with-forestdb-test-manual ( quot -- ) + [ fdb-test-config-seqtree-manual ] dip do-forestdb-test-db-manual ; inline + +: with-forestdb-test-auto ( quot -- ) + [ fdb-test-config-seqtree-auto ] dip do-forestdb-test-db-auto ; inline + +: make-kv-nth ( n -- key val ) + number>string [ "key" prepend ] [ "val" prepend ] bi ; + +: make-kv-n ( n -- seq ) + [1,b] [ make-kv-nth ] { } map>assoc ; + +: make-kv-range ( a b -- seq ) + [a,b] [ make-kv-nth ] { } map>assoc ; + +: set-kv-n ( n -- ) + make-kv-n [ fdb-set-kv ] assoc-each ; + +: del-kv-n ( n -- ) + make-kv-n keys [ fdb-del-kv ] each ; + +: set-kv-nth ( n -- ) + make-kv-nth fdb-set-kv ; + +: set-kv-range ( a b -- ) + make-kv-range [ fdb-set-kv ] assoc-each ; -- 2.34.1