當(dāng)前位置：首頁 > 编程资源 > 编程问答 >内容正文

编程问答

索引处理类的设计与实现

發(fā)布時間：2025/4/5 编程问答 13 豆豆

生活随笔收集整理的這篇文章主要介紹了索引处理类的设计与实现小編覺得挺不錯的,現(xiàn)在分享給大家,幫大家做個參考.

文章目錄

- 1 索引處理類的頭文件定義
- 2 構(gòu)造函數(shù)和析構(gòu)函數(shù)
- 3 create方法實(shí)現(xiàn)
- 4 load方法實(shí)現(xiàn)
- 5 remove和flush方法的實(shí)現(xiàn)
- 6 索引初始化測試
- 7 寫入文件實(shí)現(xiàn)
- - 7.1 哈希查找實(shí)現(xiàn)
  - 7.2 哈希插入實(shí)現(xiàn)
  - 7.3 更新塊信息
  - 7.4 寫入文件測試
- 8 讀文件實(shí)現(xiàn)
- - 8.1 讀MetaInfo
  - 8.2 讀流程實(shí)現(xiàn)
- 9 刪除文件實(shí)現(xiàn)
- - 9.1 刪除MetaInfo
  - 9.2 刪除文件流程實(shí)現(xiàn)

1 索引處理類的頭文件定義

#ifndef QINIU_LARGEFILE_INDEX_HANDLE_H_ #define QINIU_LARGEFILE_INDEX_HANDLE_H_#include "common.h" #include "mmap_file_op.h"namespace qiniu {namespace largefile{struct IndexHeader{public:IndexHeader(){memset(this, 0, sizeof(IndexHeader));}BlockInfo block_info_; //meta block infoint32_t bucket_size_; //hash bucket sizeint32_t data_file_offset_; //offset to write next data in blockint32_t index_file_size_; //offset after: index_header + all bucketsint32_t free_head_offset_;// free meta node list, for reuse};class IndexHandle{public:IndexHandle( const std::string& base_path, const uint32_t main_block_id);~IndexHandle();int create(const uint32_t logic_block_id, const int32_t bucket_size, const MMapOption map_option);int load(const uint32_t logic_block_id, const int32_t bucket_size, const MMapOption map_option);//remove index: unmmap and unlink fileint remove(const uint32_t logic_block_id);int flush();IndexHeader * index_header(){return reinterpret_cast<IndexHeader *>(file_op_->get_map_data());}int update_block_info(const OperType oper_type, const uint32_t modify_size);BlockInfo* block_info(){return reinterpret_cast<BlockInfo*> (file_op_->get_map_data());}int32_t* bucket_slot(){return reinterpret_cast<int32_t*> (reinterpret_cast<char*>(file_op_->get_map_data()) + sizeof(IndexHeader));}int32_t bucket_size() const {return reinterpret_cast<IndexHeader *>(file_op_->get_map_data())->bucket_size_;}int32_t get_block_data_offset() const{return reinterpret_cast<IndexHeader*>(file_op_->get_map_data())->data_file_offset_;}int32_t free_head_offset() const {return reinterpret_cast<IndexHeader*>(file_op_->get_map_data())->free_head_offset_;}void commit_block_data_offset(const int file_size){reinterpret_cast<IndexHeader*>(file_op_->get_map_data())->data_file_offset_ += file_size;}int32_t write_segment_meta(const uint64_t key, MetaInfo &meta);int32_t read_segment_meta(const uint64_t key, MetaInfo &meta);int32_t delete_segment_meta(const uint64_t key);int32_t hash_find(const uint64_t key, int32_t& current_offset, int32_t& previous_offset);int32_t hash_insert(const uint64_t key, int32_t previous_offset, MetaInfo &meta);private:bool hash_compare(const uint64_t left_key, const uint64_t right_key){return (left_key == right_key);}MMapFileOperation* file_op_;bool is_load_;};} } #endif //QINIU_LARGEFILE_INDEX_HANDLE_H_

2 構(gòu)造函數(shù)和析構(gòu)函數(shù)

IndexHandle::IndexHandle(const std::string& base_path, const uint32_t main_block_id){//create file_op_ handle objectstd::stringstream tmp_stream;tmp_stream << base_path<< INDEX_DIR_PREFIX<< main_block_id; // /root/martin/index/1std::string index_path;tmp_stream>>index_path;file_op_ = new MMapFileOperation(index_path, O_CREAT | O_RDWR | O_LARGEFILE);is_load_ = false;}IndexHandle::~IndexHandle(){if(file_op_){delete file_op_;file_op_ = NULL;}}

3 create方法實(shí)現(xiàn)

int IndexHandle::create(const uint32_t logic_block_id, const int32_t bucket_size, const MMapOption map_option) {int ret = TFS_SUCCESS;if(debug) printf(" create index , block id: %u , bucket size: %d, max_mmap_size: %d, first mmap size: %d, per mmap size: %d\n",logic_block_id, bucket_size, map_option.max_mmap_size_, map_option.first_mmap_size_, map_option.per_mmap_size_);if(is_load_){return EXIT_INDEX_ALREADY_LOADED_ERROR;}int64_t file_size = file_op_->get_file_size();if(file_size < 0){return TFS_ERROR;}else if(file_size == 0) //empty file{IndexHeader i_header;i_header.block_info_.block_id_ = logic_block_id;i_header.block_info_.seq_no_ = 1;i_header.bucket_size_ = bucket_size;i_header.index_file_size_ = sizeof(IndexHeader) + bucket_size * sizeof(int32_t);//index header + total bucketschar * init_data = new char[i_header.index_file_size_];memcpy(init_data, &i_header, sizeof(IndexHeader));memset(init_data + sizeof(IndexHeader), 0, i_header.index_file_size_ - sizeof(IndexHeader));//write index header and buckets into index file ret = file_op_->pwrite_file(init_data, i_header.index_file_size_, 0);delete[] init_data;init_data = NULL;if(ret != TFS_SUCCESS ){return ret;}ret = file_op_->flush_file();if(ret != TFS_SUCCESS ){return ret;}}else //file size > 0 , index already exist{return EXIT_META_UNEXPECT_FOUND_ERROR;}ret = file_op_->mmap_file(map_option);if(ret != TFS_SUCCESS){return ret;}is_load_ = true;if(debug) printf("init blockid: %d index successful. data file size: %d, index file size: %d, bucket_size: %d, free head offset: %d, seqno: %d, size: %d, filecount: %d, del_size: %d, del_file_count: %d version: %d\n",logic_block_id, index_header()->data_file_offset_, index_header()->index_file_size_,index_header()->bucket_size_, index_header()->free_head_offset_, block_info()->seq_no_, block_info()->size_,block_info()->file_count_, block_info()->del_size_, block_info()->del_file_count_, block_info()->version_);return TFS_SUCCESS; }

4 load方法實(shí)現(xiàn)

int IndexHandle::load(const uint32_t logic_block_id, const int32_t _bucket_size, const MMapOption map_option) {int ret = TFS_SUCCESS;if(is_load_){return EXIT_INDEX_ALREADY_LOADED_ERROR;}int64_t file_size = file_op_->get_file_size();if(file_size <0){return file_size;}else if(file_size == 0)//empty file {return EXIT_INDEX_CORRUPT_ERROR;}MMapOption tmp_map_option = map_option;if(file_size> tmp_map_option.first_mmap_size_ && file_size <= tmp_map_option.max_mmap_size_){tmp_map_option.first_mmap_size_ = file_size;}ret = file_op_->mmap_file(tmp_map_option);if( ret != TFS_SUCCESS){return ret;}if(debug) printf("IndexHandle::load - bucket_size(): %d, index_header()->bucket_size_: %d, block id: %d\n", bucket_size(), index_header()->bucket_size_, block_info()->block_id_);if(0 == bucket_size() || 0 == block_info()->block_id_){fprintf(stderr, "Index corrupt error. blockid: %u, bucket size: %d\n", block_info()->block_id_, bucket_size());return EXIT_INDEX_CORRUPT_ERROR;}//check file sizeint32_t index_file_size = sizeof(IndexHeader) + bucket_size() * sizeof(int32_t);if(file_size < index_file_size){fprintf(stderr, "Index corrupt error,blockid: %u, bucket size: %d, file size: %ld, index file size: %d\n", block_info()->block_id_, bucket_size(), file_size,index_file_size);return EXIT_INDEX_CORRUPT_ERROR;}//check block id if(logic_block_id != block_info()->block_id_){fprintf(stderr, "block id conflict. blockid: %u, index blockid： %u\n", logic_block_id, block_info()->block_id_);return EXIT_BLOCKID_CONFLICT_ERROR;}//check bucket_sizeif(_bucket_size != bucket_size()){fprintf(stderr, "Index configure error, old bucket size: %d, new bucket size: %d\n", bucket_size(), _bucket_size);return EXIT_BUCKET_CONFIGURE_ERROR;}is_load_ = true;if(debug) printf("load blockid: %d index successful. data file size: %d, index file size: %d, bucket_size: %d, free head offset: %d, seqno: %d, size: %d, filecount: %d, del_size: %d, del_file_count: %d version: %d\n",logic_block_id, index_header()->data_file_offset_, index_header()->index_file_size_,index_header()->bucket_size_, index_header()->free_head_offset_, block_info()->seq_no_, block_info()->size_,block_info()->file_count_, block_info()->del_size_, block_info()->del_file_count_, block_info()->version_);return TFS_SUCCESS; }

5 remove和flush方法的實(shí)現(xiàn)

int IndexHandle::remove(const uint32_t logic_block_id) {if(is_load_){if(logic_block_id != block_info()->block_id_){fprintf(stderr, "block id conflict. bolckid: %d, index blokid: %d\n", logic_block_id, block_info()->block_id_);return EXIT_BLOCKID_CONFLICT_ERROR;}}int ret = file_op_->munmap_file();if( TFS_SUCCESS != ret){return ret;}ret = file_op_->unlink_file();return ret; }int IndexHandle::flush() {int ret = file_op_->flush_file();if(TFS_SUCCESS != ret){fprintf(stderr, "index flush fail , ret: %d error desc: %s\n", ret, strerror(errno));}return ret; }

6 索引初始化測試

先來看一下塊初始化的流程：

測試代碼如下：

7 寫入文件實(shí)現(xiàn)

寫入文件的流程如下：

7.1 哈希查找實(shí)現(xiàn)

int IndexHandle::hash_find(const uint64_t key, int32_t& current_offset, int32_t& previous_offset) {int ret = TFS_SUCCESS;MetaInfo meta_info;current_offset = 0;previous_offset = 0;//1.確定key 存放的桶(slot)的位置int32_t slot = static_cast<uint32_t>(key) % bucket_size();//2.讀取桶首節(jié)點(diǎn)存儲的第一個節(jié)點(diǎn)的偏移量，如果偏移量為零，直接返回 EXIT_META_NOT_FOUND_ERROR//3.根據(jù)偏移量讀取存儲的metainfo //4.與key進(jìn)行比較，相等則設(shè)置current_offset 和previous_offset 并返回TFS_SUCCESS,否則繼續(xù)執(zhí)行5//5. 從metainfo 中取得下一個節(jié)點(diǎn)的在文件中的偏移量,如果偏移量為零，直接返回 EXIT_META_NOT_FOUND_ERROR,// 否則，跳轉(zhuǎn)至3繼續(xù)循環(huán)執(zhí)行int32_t pos = bucket_slot()[slot];for(;pos != 0; ){ret = file_op_->pread_file(reinterpret_cast<char*>(&meta_info), sizeof(MetaInfo), pos);if( TFS_SUCCESS != ret){return ret;}if( hash_compare( key, meta_info.get_key() ) ){current_offset = pos;return TFS_SUCCESS;}previous_offset = pos;pos = meta_info.get_next_meta_offset();}return EXIT_META_NOT_FOUND_ERROR;}

7.2 哈希插入實(shí)現(xiàn)

int32_t IndexHandle::hash_insert(const uint64_t key, int32_t previous_offset, MetaInfo &meta) {int ret = TFS_SUCCESS;MetaInfo tmp_meta_info;int32_t current_offset = 0;//1.確定key 存放的桶(slot)的位置int32_t slot = static_cast<uint32_t>(key) % bucket_size();//2.確定meta 節(jié)點(diǎn)存儲在文件中的偏移量if(free_head_offset() != 0){ret = file_op_->pread_file(reinterpret_cast<char*>(&tmp_meta_info), sizeof(MetaInfo), free_head_offset());if( TFS_SUCCESS != ret){return ret;}current_offset = index_header()->free_head_offset_;if(debug) printf("reuse metainfo, current_offset: %d\n", current_offset);index_header()->free_head_offset_ = tmp_meta_info.get_next_meta_offset();}else {current_offset = index_header()->index_file_size_;index_header()->index_file_size_ += sizeof(MetaInfo);}//3.將meta 節(jié)點(diǎn)寫入索引文件中meta.set_next_meta_offset(0);ret = file_op_->pwrite_file( reinterpret_cast<const char*>(&meta), sizeof(MetaInfo), current_offset);if( TFS_SUCCESS != ret){index_header()->index_file_size_ -= sizeof(MetaInfo);return ret;}//4. 將meta 節(jié)點(diǎn)插入到哈希鏈表中//當(dāng)前一個節(jié)點(diǎn)已經(jīng)存在if( 0 != previous_offset){ret = file_op_->pread_file(reinterpret_cast<char*>(&tmp_meta_info), sizeof(MetaInfo), previous_offset);if( TFS_SUCCESS != ret){index_header()->index_file_size_ -= sizeof(MetaInfo);return ret;}tmp_meta_info.set_next_meta_offset(current_offset);ret = file_op_->pwrite_file( reinterpret_cast<const char*>(&tmp_meta_info), sizeof(MetaInfo), previous_offset);if( TFS_SUCCESS != ret){index_header()->index_file_size_ -= sizeof(MetaInfo);return ret;}}else { //不存在前一個節(jié)點(diǎn)的情況bucket_slot()[slot] = current_offset;}return TFS_SUCCESS; }

7.3 更新塊信息

int IndexHandle::write_segment_meta(const uint64_t key, MetaInfo &meta) {int32_t current_offset = 0, previous_offset = 0;//思考？ key 存在嗎？存在=>處理？不存在=>處理？//1.從文件哈希表中查找key 是否存在 hash_find(key, current_offset, previous_offset);int ret = hash_find(key, current_offset, previous_offset);if( TFS_SUCCESS == ret){return EXIT_META_UNEXPECT_FOUND_ERROR;}else if( EXIT_META_NOT_FOUND_ERROR != ret){return ret;}//2.不存在就寫入meta 到文件哈希表中 hash_insert(key, previous_offset, meta);ret = hash_insert(key, previous_offset, meta);return ret; }int IndexHandle::update_block_info(const OperType oper_type, const uint32_t modify_size) {if( block_info()->block_id_ == 0){return EXIT_BLOCKID_ZERO_ERROR;}if( oper_type == C_OPER_INSERT ){++block_info()->version_;++block_info()->file_count_;++block_info()->seq_no_;block_info()->size_ += modify_size;}else if( oper_type == C_OPER_DELETE ){++block_info()->version_;--block_info()->file_count_;block_info()->size_ -= modify_size;++block_info()->del_file_count_;block_info()->del_size_ += modify_size;}if(debug) printf("update block info. blockid: %u, version: %u, file count: %u, size: %u, del file count: %u, del size: %u, seq no: %u, oper type: %d\n",block_info()->block_id_, block_info()->version_, block_info()->file_count_, block_info()->size_,block_info()->del_file_count_, block_info()->del_size_, block_info()->seq_no_, oper_type);return TFS_SUCCESS; }

7.4 寫入文件測試

#include "common.h" #include "file_op.h" #include "index_handle.h" #include <sstream>using namespace qiniu; using namespace std;const static largefile::MMapOption mmap_option = {1024000, 4096, 4096}; //內(nèi)存映射的參數(shù) const static uint32_t main_blocksize = 1024*1024*64; //主塊文件的大小 const static uint32_t bucket_size = 1000; //哈希桶的大小 static int32_t block_id = 1;static int debug = 1;int main(int argc, char **argv) //argv[0]="rm" argv[1]="-f" argv[2] ="a.out" {std::string mainblock_path;std::string index_path;int32_t ret = largefile::TFS_SUCCESS;cout<<" Type your bockid :"<<endl;cin>> block_id;if(block_id < 1){cerr<<" Invalid blockid, exit."<<endl;exit(-1);}//1. 加載索引文件largefile::IndexHandle *index_handle = new largefile::IndexHandle(".", block_id); //索引文件句柄if(debug) printf("load index ...\n");ret = index_handle->load(block_id, bucket_size, mmap_option);if(ret != largefile::TFS_SUCCESS){fprintf(stderr, "load index %d failed.\n", block_id);//delete mainblock;delete index_handle;exit(-2);}//2. 寫入文件到主塊文件中std::stringstream tmp_stream;tmp_stream<< "." << largefile::MAINBLOCK_DIR_PREFIX << block_id;tmp_stream >> mainblock_path;largefile::FileOperation *mainblock = new largefile::FileOperation(mainblock_path, O_RDWR | O_LARGEFILE | O_CREAT);char buffer[4096];memset(buffer, '6', sizeof(buffer));int32_t data_offset = index_handle->get_block_data_offset();uint32_t file_no = index_handle->block_info()->seq_no_;if( (ret = mainblock->pwrite_file(buffer, sizeof(buffer), data_offset)) != largefile::TFS_SUCCESS){fprintf(stderr, "write to main block failed. ret: %d, reason: %s\n", ret, strerror(errno));mainblock->close_file();delete mainblock;delete index_handle;exit(-3);}//3. 索引文件中寫入Metainfolargefile::MetaInfo meta;meta.set_file_id( file_no );meta.set_offset( data_offset );meta.set_size( sizeof(buffer) );ret = index_handle->write_segment_meta(meta.get_key(), meta);if(ret == largefile::TFS_SUCCESS){//1.更新索引頭部信息index_handle->commit_block_data_offset( sizeof(buffer) );//2. 更新塊信息index_handle->update_block_info(largefile::C_OPER_INSERT, sizeof(buffer) );ret = index_handle->flush();if(ret != largefile::TFS_SUCCESS){fprintf(stderr, "flush mainblock %d failed. file no: %u \n", block_id, file_no);}}else {fprintf(stderr, "write_segement_meta - mainblock %d failed. file no: %u\n", block_id, file_no);}if(ret != largefile::TFS_SUCCESS){fprintf(stderr,"write to mainblock %d failed. file no: %u\n", block_id, file_no);}else {if(debug) printf("write successfully. file_no : %u , block_id: %d\n", file_no, block_id);}mainblock->close_file();delete mainblock;delete index_handle;return 0; }

8 讀文件實(shí)現(xiàn)

8.1 讀MetaInfo

int32_t IndexHandle::read_segment_meta(const uint64_t key, MetaInfo &meta) {int32_t current_offset = 0, previous_offset = 0;//1.確定key 存放的桶(slot)的位置//int32_t slot = static_cast<uint32_t>(key) % bucket_size();int32_t ret = hash_find(key, current_offset, previous_offset);if( TFS_SUCCESS == ret) //exist{ret = file_op_->pread_file(reinterpret_cast<char *> (&meta), sizeof(MetaInfo), current_offset);return ret;}else {return ret;} }

8.2 讀流程實(shí)現(xiàn)

#include "common.h" #include "file_op.h" #include "index_handle.h" #include <sstream>using namespace qiniu; using namespace std;const static largefile::MMapOption mmap_option = {1024000, 4096, 4096}; //內(nèi)存映射的參數(shù) const static uint32_t main_blocksize = 1024*1024*64; //主塊文件的大小 const static uint32_t bucket_size = 1000; //哈希桶的大小 static int32_t block_id = 1;static int debug = 1;int main(int argc, char **argv) //argv[0]="rm" argv[1]="-f" argv[2] ="a.out" {std::string mainblock_path;std::string index_path;int32_t ret = largefile::TFS_SUCCESS;cout<<" Type your bockid :"<<endl;cin>> block_id;if(block_id < 1){cerr<<" Invalid blockid, exit."<<endl;exit(-1);}//1. 加載索引文件largefile::IndexHandle *index_handle = new largefile::IndexHandle(".", block_id); //索引文件句柄if(debug) printf("load index ...\n");ret = index_handle->load(block_id, bucket_size, mmap_option);if(ret != largefile::TFS_SUCCESS){fprintf(stderr, "load index %d failed.\n", block_id);//delete mainblock;delete index_handle;exit(-2);}//2. 讀取文件的meta info.uint64_t file_id = 0;cout<<" Type your file_id :"<<endl;cin>> file_id;if(file_id < 1){cerr<<" Invalid fileid, exit."<<endl;exit(-2);}largefile::MetaInfo meta;ret = index_handle->read_segment_meta( file_id, meta);if(ret != largefile::TFS_SUCCESS){fprintf(stderr, "read_segment_meta error. file_id: %lu, ret: %d\n", file_id, ret);exit(-3);}//3. 根據(jù)meta info 讀取文件std::stringstream tmp_stream;tmp_stream<< "." << largefile::MAINBLOCK_DIR_PREFIX << block_id;tmp_stream >> mainblock_path;largefile::FileOperation *mainblock = new largefile::FileOperation(mainblock_path, O_RDWR );char *buffer = new char[meta.get_size() + 1];ret = mainblock->pread_file(buffer, meta.get_size(), meta.get_offset());if(ret != largefile::TFS_SUCCESS){fprintf(stderr, "read from main block failed. ret: %d, reason: %s\n", ret, strerror(errno));mainblock->close_file();delete mainblock;delete index_handle;exit(-3);}buffer[meta.get_size()] = '\0';printf("read size %d, content: %s\n", meta.get_size(), buffer);mainblock->close_file();delete mainblock;delete index_handle;return 0; }

9 刪除文件實(shí)現(xiàn)

9.1 刪除MetaInfo

int32_t IndexHandle::delete_segment_meta(const uint64_t key) {int32_t current_offset = 0, previous_offset = 0;int32_t ret = hash_find(key, current_offset, previous_offset);if( ret != TFS_SUCCESS ){return ret;}MetaInfo meta_info;ret = file_op_->pread_file(reinterpret_cast<char *>(&meta_info), sizeof(MetaInfo), current_offset);if ( TFS_SUCCESS != ret ){return ret;}int32_t next_pos = meta_info.get_next_meta_offset();if(previous_offset == 0){int32_t slot = static_cast<uint32_t> (key) % bucket_size();bucket_slot()[slot] = next_pos;}else {MetaInfo pre_meta_info;ret = file_op_->pread_file(reinterpret_cast<char *>(&pre_meta_info), sizeof(MetaInfo), previous_offset);if ( TFS_SUCCESS != ret ){return ret;}pre_meta_info.set_next_meta_offset(next_pos);ret = file_op_->pwrite_file(reinterpret_cast<char *>(&pre_meta_info), sizeof(MetaInfo), previous_offset);if ( TFS_SUCCESS != ret ){return ret;}}//把刪除節(jié)點(diǎn)加入可重用節(jié)點(diǎn)鏈表（下一個小節(jié)實(shí)現(xiàn)）meta_info.set_next_meta_offset(free_head_offset());// index_header()->free_head_offset_;ret = file_op_->pwrite_file(reinterpret_cast<char *>(&meta_info), sizeof(MetaInfo), current_offset);if( TFS_SUCCESS != ret ){return ret;}index_header()->free_head_offset_ = current_offset;if(debug) printf("delete_segment_meta - reuse metainfo, current_offset: %d\n", current_offset);update_block_info(C_OPER_DELETE, meta_info.get_size());return TFS_SUCCESS;}

9.2 刪除文件流程實(shí)現(xiàn)

參考資料：

C/C++從入門到精通-高級程序員之路【奇牛學(xué)院】

總結(jié)

以上是生活随笔為你收集整理的索引处理类的设计与实现的全部內(nèi)容，希望文章能夠幫你解決所遇到的問題。

如果覺得生活随笔網(wǎng)站內(nèi)容還不錯，歡迎將生活随笔推薦給好友。

索引