From 5df79c53745fde5d6c3340a2979b1429cd5892c1 Mon Sep 17 00:00:00 2001 From: Henrik Rydberg Date: Sat, 8 Oct 2011 20:30:28 +0200 Subject: Initial import of htcd system 1.0 Signed-off-by: Henrik Rydberg --- src/hdb/README | 154 +++++++++++++++++++++++++++ src/hdb/buffer.h | 38 +++++++ src/hdb/comp.cc | 78 ++++++++++++++ src/hdb/cvt.cc | 83 +++++++++++++++ src/hdb/db.cc | 276 +++++++++++++++++++++++++++++++++++++++++++++++ src/hdb/db.h | 129 ++++++++++++++++++++++ src/hdb/filemap.cc | 302 ++++++++++++++++++++++++++++++++++++++++++++++++++++ src/hdb/filemap.h | 196 ++++++++++++++++++++++++++++++++++ src/hdb/hdbcvt.cc | 50 +++++++++ src/hdb/hdbfix.cc | 140 ++++++++++++++++++++++++ src/hdb/hdbtest.cc | 153 ++++++++++++++++++++++++++ src/hdb/sortable.cc | 81 ++++++++++++++ src/hdb/sortable.h | 138 ++++++++++++++++++++++++ src/hdb/test.cc | 52 +++++++++ 14 files changed, 1870 insertions(+) create mode 100644 src/hdb/README create mode 100644 src/hdb/buffer.h create mode 100644 src/hdb/comp.cc create mode 100644 src/hdb/cvt.cc create mode 100644 src/hdb/db.cc create mode 100644 src/hdb/db.h create mode 100644 src/hdb/filemap.cc create mode 100644 src/hdb/filemap.h create mode 100644 src/hdb/hdbcvt.cc create mode 100644 src/hdb/hdbfix.cc create mode 100644 src/hdb/hdbtest.cc create mode 100644 src/hdb/sortable.cc create mode 100644 src/hdb/sortable.h create mode 100644 src/hdb/test.cc (limited to 'src/hdb') diff --git a/src/hdb/README b/src/hdb/README new file mode 100644 index 0000000..0db7269 --- /dev/null +++ b/src/hdb/README @@ -0,0 +1,154 @@ +/************************************************************************* + * + * HTCd - Copyright (C) 1998-2006 Henrik Rydberg + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + */ + +HDB/1.2 FILE FORMAT SPECIFICATION Sat Feb 24 18:38:25 MET 2001 + +MT AND MP SAFE + +One single format utilizing memory maps, and which is designed to +work safely between processes and within a multi-threading process. +To accomplish this, there are two types of locking mechanisms - +one inter-process lock in the file header, and one MT lock implemented +the usual way. + +The format is robust within each thread, except for changes in the +table structure, which generates fatal error_t errors. + +The format includes a finite (32) number of tables, which should not +be a real problem. Besides, a possible change is to move the frame +pointer to a position inside, so that additional header could be +added if needed. + +FILEFORMAT in HDB/1.2 + +BYTES 1.2 NAME DESCRIPTION +32 IDENT Identification header +4 LOCK Inter-process lock +4 MINOR Minor modification state number +4 MAJOR Major modification state number +4 FATAL Fatal modification state number +4 FILESIZE Size of file +4 FILETOP Size of used part +4 TABLES Number of table name strings +4 INDEX Which column is the order column. +32 COMPNAME Comparison name +32*32 TABLENAMES Table names +4 FRAME Start of record data +* RECORDS Record list + 4 mem amount of memory for this record + 4 caps the capacity of this record - 0 means deleted + mem-8 data The data follows +0 ZEROS The rest of the file is padded with zeros. + +The records are always staying where they land; a minor change means +a change in-place, a major change means a change leading to an index +update, which could be a deletion, an insertion. Those are updated at +will, and not seen until wanted. A fatal change is a change in the +table structure, i.e., an insertion or deletion of a table. Also a +table name change counts as fatal. The filesize used to be (earlier +versions) a state needing attention as well, but not anymore. + +-- + +HDB/1.11 FILE FORMAT SPECIFICATION Mon Jan 29 03:51:41 MET 2001 + +STATIC/DYNAMIC mapping + +The header field determines whether the mapping is dynamic or static, +i.e., static mapping creates a shared mmap and edits the records +directly, whereas the dynamic mapping creates a private map and +stores the changes on save only. + +The format changed quite a lot after all... + +FILEFORMAT in HDB/1.11 + +BYTES 1.11 NAME DESCRIPTION +32 IDENT Identification header +4 MODNUM Modification number +4 LOCK Soft lock +4 FILESIZE Size of file +4 FILETOP Size of used part +4 FIXED Whether fixed mapping or not +4 TABLES Number of table name strings +4 INDEX Which column is the order column. +32 COMPNAME Comparison name +4 RECORDS Number of records in file. + tables + 4,* name table name + 4,* defaults default string + 4 caps default sizes + [4],* records TABLES[RECORDS] + +Size info [4] always refers to the stringref-size, but the number of +bytes actually written is as follows: for fixed maps, it is tabs[].caps, +and for dynamic maps, it is capsize(), which is defined as (n=stringref-size) +(4+n)&~3. + +-- + +HDB/1.10 FILE FORMAT SPECIFICATION Thu Jan 25 00:42:49 MET 2001 + +The HDB/1.0 format has been used widely in the software +for our group (MST) during the last three years of so. +The HDB/1.1 file format is binary compatible with 1.0, +and merely consists of a formalization, along with a +change in the internal manipulation scheme and a new +hdb editor. + +FILE FORMAT IN 1.0 and 1.1 + +BYTES 1.0/1.1 NAME DESCRIPTION +32/variable HEAD Identification header. + Used to be exactly 32 bytes, + but see no point in that. +4 TABLES Number of table name strings + 4 dim bytes in string + * data string data +4 DEFAULTS Number of default strings + 4 dim bytes in string + * data string data +4 ORDERBY Which column is the order column. +4 dim Number of bytes in compname. +* data string data +4 RECORDS Number of records in file. + 4 tables + 4 dim + * data +0/variable FOOTER File ends. Simple Footer. + +ADDITIONAL CHANGES BETWEEN 1.0 and 1.1 + +The major difference - and the rationale for 1.1 - is +the possiblity to edit files directly, as in mmap. +To be able to do this, there is a need to force a minimum +size to each field. This is done by padding the data +with trailing zeros. This is ok since only string data +is stored in the fields, and never binary data. Just think +of it; if it was binary data, it would not be replaced +by something the same size anyways, so thats forbidden +in the direct-editing mode. It makes no difference to +the normal operation. + +The default sizes for new fields are taken from the defaults +table, which again is completely compatible. The only +difference is that the zeros are stored as well, and hence +cannot be defined just by the default string, but need an editor +that specifies the sizes. + diff --git a/src/hdb/buffer.h b/src/hdb/buffer.h new file mode 100644 index 0000000..9383753 --- /dev/null +++ b/src/hdb/buffer.h @@ -0,0 +1,38 @@ +/************************************************************************* + * + * HTCd - Copyright (C) 1998-2006 Henrik Rydberg + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#ifndef BUFFERH +#define BUFFERH + +#include +#include + +inline void LoadBuffer(mstring& buffer,const mstring& path) + throw(merror_t) +{ + if(buffer.load(path.c_str())<0) THROW("file: Could not load "< +using namespace std; + +const mstring compat="(compatibility)"; +const int zkey=0; + +OP2_IMPL(lti) {out<<(atoi(L)=atoi(R));} +OP2_IMPL(gti) {out<<(atoi(L)>atoi(R));} + +void dbmap::setup() +{ + MLOCK(mutex); + if(M.empty()) { + opAddComp(M); + OP_ADDN(20,lti,lt.int,compat); + OP_ADDN(20,lei,le.int,compat); + OP_ADDN(20,gei,ge.int,compat); + OP_ADDN(20,gti,gt.int,compat); + + M.add(new op_t("lt.case",20,M.get(sref("lt"))->f2,compat)); + M.add(new op_t("gt.case",20,M.get(sref("gt"))->f2,compat)); + M.add(new op_t("lt.nocase",20,M.get(sref("ltn"))->f2,compat)); + M.add(new op_t("gt.nocase",20,M.get(sref("gtn"))->f2,compat)); + } +} + +const op_t* dbmap::CompOperator(const sref& s) +{ + if(M.empty()) setup(); + const op_t* p=M.get(s); + if(!p) THROW("dbmap: could not find operator "<head()->setcomp(index,compname); + remap_tabs(); + } +} + diff --git a/src/hdb/cvt.cc b/src/hdb/cvt.cc new file mode 100644 index 0000000..4788bf1 --- /dev/null +++ b/src/hdb/cvt.cc @@ -0,0 +1,83 @@ +/************************************************************************* + * + * HTCd - Copyright (C) 1998-2006 Henrik Rydberg + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include +using namespace std; + +const mstring HEAD10="HTC DB File Format version 1.0\n"; +const mstring HEAD11="HDB/1.1"; + +static fp_stream merr(stderr); + +static void Read(FILE* fp,mstring& s,int version) { + int n; fread(&n,4,1,fp); + switch(version) { + case 10: + s.resize(n); + fread(s.data(),1,n,fp); + break; + case 11: + s.resize(n-1); + fread(s.data(),1,n,fp); + s.resize(strlen(s.data())); + if(s.size()>=n) THROW("dbmap: assertion failed on dimension"); + break; + default: + THROW("read: bad HDB version"); + break; + } +} + +void Convert(dbmap& db,const char* path) +{ + FILE* fp=fopen(path,"rb"); + char line[1024]; + fgets(line,1000,fp); + int version=0; + if(sref(line)==HEAD10) { fgetc(fp); version=10; } + else if(First(sref(line))==HEAD11) { version=11; } + if(version==0) THROW("hdbcvt: unknown HDB format"); + int tables; + mstring s; + fread(&tables,4,1,fp); + for(int i=0;i +#include + +filemap* dbmap::flist; +mutex_t dbmap::mutex; +opmap dbmap::M; + +/////////////////////////////////////////////////////////////////////// + +dbmap::dbmap(): + wok(0),fatal(0),index(0), + compname(),file(0),tmap(),recs() +{ + minor=0; + major=0; +} + +dbmap::~dbmap() +{ + Close(); +} + +/////////////////////////////////////////////////////////////////////// + +void dbmap::remap_tabs() +{ + fatal=head()->fatal.get(); + tmap.clear(); + for(int i=0;itabs[i]),i); +} + +void dbmap::remap_recs() +{ + major=head()->major.get(); + recs.clear(); + int p=file->recs(); + while(file->size(p)) { + record_t* rec=(record_t*)file->at(p); + if(rec->caps.get()) + recs.push_back(idx_t(rec->at(index),rec)); + p+=rec->mem.get(); + } + recs.sort(CompOperator(compname)); +} + +int dbmap::remap() +{ + int state=0; + if(fatal!=head()->fatal.get()) { remap_tabs(); state|=4; } + if(major!=head()->major.get()) { remap_recs(); state|=2; } + if(minor!=head()->minor.get()) { minor=head()->minor.get(); state|=1; } + return state; +} + +/////////////////////////////////////////////////////////////////////// + +void Open(const mstring& path,int update=0,int prot=0600) throw(merror_t); +void Saveas(const mstring& path,int prot=0600) throw(merror_t); +void Close(); + +void dbmap::Open(const mstring& path,int update,int prot) throw(merror_t) +{ + Close(); + wok=update?1:0; + { + MLOCK(mutex); + for(filemap* p=flist;p;p=p->next) + if(p->Pathname()==path&&p->writable()==wok) { (file=p)->pushref(); break; } + if(!file) { + filemap* fm=new filemap(flist); + try { fm->open(path,wok,prot); fm->check(); file=flist=fm; } + catch(...) { delete fm; wok=0; throw; } + } + } + index=head()->index.get(); + compname=head()->compname; + remap_tabs(); + remap_recs(); + minor=head()->minor.get(); +} + +void dbmap::Saveas(const mstring& path,int prot) throw(merror_t) +{ + file->save(path,prot); +} + +void dbmap::Close() +{ + wok=0; + minor=0; + major=0; + fatal=0; + index=0; + compname.clear(); + recs.clear(); + tmap.clear(); + if(file) { + MLOCK(mutex); + if(file->reference()) file->popref(); + else { + if(flist==file) flist=file->next; + else for(filemap* p=flist;p;p=p->next) + if(p->next==file) { p->next=file->next; break; } + delete file; + } + file=0; + } +} + +void dbmap::Sync() +{ + MLOCK(mutex); + for(filemap* p=flist;p;p=p->next) { + if(p->writable()) p->sync(); + } +} + +/////////////////////////////////////////////////////////////////////// + +void dbmap::SetTable(int col,const sref& name) throw(merror_t) +{ + check(); + if(!wok) THROW("db: cannot set table - readonly"); + if(col<0||col>=Tables()) THROW("db: bad column"); + HEADLOCK(file); + head()->set(col,name); + remap(); +} + +void dbmap::InsertTable(int col,const sref& name) throw(merror_t) +{ + check(); + if(!wok) THROW("db: cannot insert table - readonly"); + if(Tables()+1>file->maxtabs) THROW("db: too many tables"); + if(col<0||col>Tables()) THROW("db: bad column"); + { + HEADLOCK(file); + head()->insert(col,name); + int p=file->recs(); + while(file->size(p)) { + record_t* rec=(record_t*)file->at(p); + if(rec->caps.get()) rec->insert(col); + p+=rec->mem.get(); + } + head()->major.add(1); + } + index=head()->index.get(); + compname=head()->compname; + remap(); +} + +void dbmap::RemTable(int col) throw(merror_t) +{ + check(); + if(!wok) THROW("db: cannot remove table - readonly"); + if(col<0||col>=Tables()) THROW("db: bad column"); + if(col==head()->index.get()) THROW("db: cannot remove index table"); + { + HEADLOCK(file); + head()->remove(col); + int p=file->recs(); + while(file->size(p)) { + record_t* rec=(record_t*)file->at(p); + if(rec->caps.get()) rec->remove(col); + p+=rec->mem.get(); + } + head()->major.add(1); + } + index=head()->index.get(); + compname=head()->compname; + remap(); +} + +/////////////////////////////////////////////////////////////////////// + +void dbmap::Get(int row,spile& list) throw(merror_t) +{ + check(); + if(row<0||row>=recs.size()) THROW("db: not a valid record"); + list.resize(Tables()); + recs[row].rec->get(list); +} + +void dbmap::Set(int row,int col,const sref& s) throw(merror_t) +{ + check(); + if(!wok) THROW("db: cannot set - readonly"); + if(row<0||row>=recs.size()) THROW("db: not a valid record"); + if(col==index) THROW("db: cannot set index"); + HEADLOCK(file); + record_t* rec=recs[row].rec; + int dim=rec->setreq(col,s); + if(dim>rec->caps.get()) { + rec=new(file->add(dim)) record_t(*rec); + file->rem(recs[row].rec); + recs[row].rec=rec; + head()->major.add(1); + major++; + } + rec->set(col,s); + head()->minor.add(1); + minor++; +} + +void dbmap::Set(int row,const spile& list) throw(merror_t) +{ + check(); + if(!wok) THROW("db: cannot set - readonly"); + if(row<0||row>=recs.size()) THROW("db: not a valid record"); + if(list.size()!=Tables()) + THROW("db: wrong number of tables - "<at(index)!=list[index]) THROW("db: cannot set index"); + int dim=record_t::req(list); + if(dim>rec->caps.get()) { + rec=new(file->add(dim)) record_t(*rec); + file->rem(recs[row].rec); + recs[row].rec=rec; + head()->major.add(1); + major++; + } + rec->set(list); + head()->minor.add(1); + minor++; +} + +/////////////////////////////////////////////////////////////////////// + +void dbmap::Rem(int row) throw(merror_t) +{ + if(!wok) THROW("db: cannot delete - readonly"); + if(row<0||row>=recs.size()) THROW("db: not a valid record"); + HEADLOCK(file); + file->rem(recs[row].rec); + recs.erase(recs.begin()+row); + head()->major.add(1); + major++; +} + +int dbmap::Insert(const spile& list,int unique) throw(merror_t) +{ + check(); + if(!wok) THROW("db: cannot insert - readonly"); + if(index!=head()->index.get()) + THROW("db: can only insert under normal order"); + HEADLOCK(file); + remap(); + int row=recs.get(list[index]); + if(row<0) { + record_t* rec=new(file->add(record_t::req(list))) record_t(list); + row=recs.add(rec->at(index),rec); + head()->major.add(1); + major++; + } + return row; +} diff --git a/src/hdb/db.h b/src/hdb/db.h new file mode 100644 index 0000000..6621944 --- /dev/null +++ b/src/hdb/db.h @@ -0,0 +1,129 @@ +/************************************************************************* + * + * HTCd - Copyright (C) 1998-2006 Henrik Rydberg + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#ifndef DBH +#define DBH + +#include +#include + +//////////////////////////////////////////////////// +// +// OK for MT and MP +// + +class dbmap { +public: + dbmap(); + ~dbmap(); + + filemap::head_t* head() { return file->head(); } + const filemap::head_t* head() const { return file->head(); } + + int remap(); + + int size() const { return recs.size(); } + int empty() const { return recs.empty(); } + + const sref operator()(int row,int col) const throw(merror_t) { + check(); return recs[row].rec->at(col); + } + const sref at(int row,int col) const throw(merror_t) { + check(); return recs[row].rec->at(col); + } + + /////////////////////////////////////////////////////// + + void Get(int row,spile& list) throw(merror_t); + void Set(int row,int col,const sref& s) throw(merror_t); + void Set(int row,const spile& list) throw(merror_t); + + /////////////////////////////////////////////////////// + + void Rem(int row) throw(merror_t); + int Insert(const spile& list,int unique) throw(merror_t); + + /////////////////////////////////////////////////////// + + int Tables() const { return head()->tables.get(); } + const sref Table(int col) const { check(); return head()->get(col); } + + /////////////////////////////////////////////////////// + + void SetTable(int col,const sref& name) throw(merror_t); + void RemTable(int col) throw(merror_t); + void InsertTable(int col,const sref& name) throw(merror_t); + + /////////////////////////////////////////////////////// + + int Index() const { return index; } + int FindIndex(const sref& tab) const { return tmap.get(tab); } + int Index(const sref& tab) const throw(merror_t) { + int p=FindIndex(tab); + if(p<0) THROW("db: table ["<Lastmod(); } + void Close(); + + static void Sync(); + +protected: + dbmap(const dbmap& db) {} + void operator=(const dbmap& db) {} + + void check() const { + if(fatal!=head()->fatal.get()) THROW("db: fatal state change") + } + + void remap_recs(); + void remap_tabs(); + + static void setup(); + static const op_t* CompOperator(const sref& s); + + static filemap* flist; + static mutex_t mutex; + static opmap M; +private: + int wok,minor,major,fatal,index; + mstring compname; + filemap* file; + rimap tmap; + indexmap recs; +}; + +void Convert(dbmap& db,const char* path); + +#endif + diff --git a/src/hdb/filemap.cc b/src/hdb/filemap.cc new file mode 100644 index 0000000..b40644f --- /dev/null +++ b/src/hdb/filemap.cc @@ -0,0 +1,302 @@ +/************************************************************************* + * + * HTCd - Copyright (C) 1998-2006 Henrik Rydberg + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include +#include +#include +#include +#include +#include +using namespace std; + +const int PAGESIZE=getpagesize(); +const filemap::char32 HEADNAME="HDB/1.2\n"; +const filemap::char32 COMPNAME="lt"; +const mstring dev_tag="/dev/"; + +const int TIMEOUT=20; + +/////////////////////////////////////////////////// + +filemap::head_t::head_t() +{ + memcpy(ident,HEADNAME,sizeof(char32)); + lock.set(0); + minor.set(0); + major.set(0); + fatal.set(0); + filesize.set(PAGESIZE); + filetop.set(sizeof(head_t)); + tables.set(0); + index.set(0); + memcpy(compname,COMPNAME,sizeof(char32)); + memset(tabs,0,maxtabs*sizeof(char32)); + frame.set(sizeof(head_t)); +} + +filemap::head_t::head_t(const head_t* hd,int mem) +{ + memcpy(ident,HEADNAME,sizeof(char32)); + lock.set(0); + minor.set(0); + major.set(0); + fatal.set(0); + filetop.set(sizeof(head_t)+mem); + filesize.set(PAGESIZE*int((filetop.get()+PAGESIZE)/PAGESIZE)); + tables=hd->tables; + index=hd->index; + memcpy(compname,hd->compname,sizeof(char32)); + memset(tabs,0,maxtabs*sizeof(char32)); + for(int i=0;itabs[i],sizeof(char32)); + frame.set(sizeof(head_t)); +} + +void filemap::head_t::setcomp(int idx,const sref& s) +{ + index.set(idx); + s.copyto(compname,sizeof(char32)); + fatal.add(1); +} + +void filemap::head_t::set(int col,const sref& s) +{ + s.copyto(tabs[col],sizeof(char32)); + fatal.add(1); +} + +void filemap::head_t::insert(int col,const sref& s) +{ + for(int i=maxtabs-1;i>col;i--) + memcpy(tabs[i],tabs[i-1],sizeof(char32)); + if(index.get()>=col&&index.get()col) index.set(index.get()-1); + tables.add(-1); + fatal.add(1); + } +} + +/////////////////////////////////////////////////////// + +filemap::chunk_t::chunk_t(chunk_t* p,int fd,int wok,int off,int size) +{ + next=p; + end=(begin=off)+size; + map=(char*)mmap(0,size,PROT_READ|(wok?PROT_WRITE:0),MAP_SHARED,fd,off); + if(map==MAP_FAILED) { map=0; THROW("filemap: could not map chunk") } +} + +filemap::chunk_t::~chunk_t() +{ + if(map) munmap(map,end-begin); +} + +int filemap::chunk_t::fill(int off) +{ + if(own(off)) { + at(off)->mem.set(end-off); + at(off)->caps.set(0); + return end; + } + else return off; +} + +void filemap::chunk_t::sync() +{ + msync(map,end-begin,MS_SYNC); +} + +/////////////////////////////////////////////////////// + +void filemap::remap() +{ + while(mem->endfilesize.get()) { + mem=new chunk_t(mem,fd,wok,mem->end,mem->size()<<1); + } +} + +filemap::rec_t* filemap::add(int n) +{ + int caps=sizeof(rec_t),dim=caps+n; + while(capsfilesize.get()end) { + ftruncate(fd,mem->end); + hd->filesize.set(mem->end); + } + while(hd->filetop.get()+caps>mem->end) { + hd->filetop.set(mem->fill(hd->filetop.get())); + mem=new chunk_t(mem,fd,wok,mem->end,mem->size()<<1); + ftruncate(fd,mem->end); + hd->filesize.set(mem->end); + } + rec_t* rec=at(hd->filetop.get()); + hd->filetop.add(caps); + rec->mem.set(caps); + rec->caps.set(n); + return rec; +} + +void filemap::rem(rec_t* rec) +{ + rec->caps.set(0); +} + +/////////////////////////////////////////////////////// + +filemap::filemap(filemap* p): + next(p),fd(-1),wok(0),refs(0),pathname(),hd(0),mem(0),nodelock(0) +{ + memset(&mutex,0,sizeof(mutex_t)); +} + +filemap::filemap(filemap* p,const mstring& path,int update,int prot): + next(p),fd(-1),wok(0),refs(0),pathname(),hd(0),mem(0),nodelock(0) +{ + memset(&mutex,0,sizeof(mutex_t)); + open(path,update,prot); +} + +filemap::~filemap() +{ + close(); +} + +/////////////////////////////////////////////////////// + +void filemap::open(const mstring& path,int update,int prot) throw(merror_t) +{ + close(); + if(update) { + if(path.left(5)!=dev_tag) nodelock=new nodelock_t(path); + else nodelock=0; + wok=1; + } + else { + nodelock=0; + wok=0; + } + fd=::open(path.c_str(),wok?O_RDWR|O_CREAT:O_RDONLY,prot); + if(fd<0) THROW("filemap: could not open "<map; + if(fsizeident))!=First(sref(HEADNAME))) THROW("filemap: bad version"); + remap(); + if(fsize!=hd->filesize.get()) + THROW("filemap: "<filetop.get(); + while(size(p)) { + rec_t* rec=at(p); + p+=rec->mem.get(); + } + if(pft. +} + +void filemap::save(const mstring& path,int prot) throw(merror_t) +{ + if(path==pathname) THROW("filemap: cannot overwrite current file"); + int mfd=::open(path.c_str(),O_RDWR|O_CREAT|O_TRUNC,prot); + if(mfd<0) THROW("filemap: could not open "<caps.get()) caps+=rec->mem.get(); + p+=rec->mem.get(); + } + head_t mhd(hd,caps); + ::write(mfd,&mhd,sizeof(head_t)); + p=recs(); + while(size(p)) { + rec_t* rec=at(p); + if(rec->caps.get()) ::write(mfd,rec,rec->mem.get()); + p+=rec->mem.get(); + } + ftruncate(mfd,mhd.filesize.get()); + ::close(mfd); +} + +void filemap::close() +{ + while(mem) { chunk_t* p=mem->next; delete mem; mem=p; } + hd=0; + wok=0; + refs=0; + if(fd>=0) { ::close(fd); fd=-1; } + delete nodelock; nodelock=0; + pathname.clear(); +} + +void filemap::sync() +{ + for(chunk_t* p=mem;p;p=p->next) p->sync(); +} + +/////////////////////////////////////////////////////// + +void filemap::lock() +{ + if(hd) { + MLOCK(mutex); + if((volatile long&)hd->lock) { + time_t end=time(0)+TIMEOUT; + while((volatile long&)hd->lock&&time(0)lock.set(1); + } +} + +void filemap::unlock() +{ + if(hd) hd->lock.set(0); +} + +/////////////////////////////////////////////////////// + +time_t filemap::Lastmod() const +{ + struct stat fs; fstat(fd,&fs); return fs.st_mtime; +} + diff --git a/src/hdb/filemap.h b/src/hdb/filemap.h new file mode 100644 index 0000000..d6af7eb --- /dev/null +++ b/src/hdb/filemap.h @@ -0,0 +1,196 @@ +/************************************************************************* + * + * HTCd - Copyright (C) 1998-2006 Henrik Rydberg + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#ifndef FILEMAPH +#define FILEMAPH + +#include +#include +#include +#include +#include + +////////////////////////////////////////////// +// +// NOTE that filemap is MT unsafe: +// it needs to be called from dbmap +// +// The exception is when filelocking; +// that's MT safe per filemap, and hence +// should not be locked further +// + +#define HEADLOCK(fm) headlock_t headlock(fm) + +////////////////////////////////////////////////////// + +// network byteorder +class byte4 { +public: + byte4() {} // must be uninitialized + explicit byte4(int h): n(htonl(h)) {} + void set(int h) { n=htonl(h); } + void add(int h) { n=htonl(ntohl(n)+h); } + int get() const { return ntohl(n); } +private: + long n; +}; + +////////////////////////////////////////////////////// + +class filemap { +public: + typedef char char32[32]; + static const int maxtabs=32; + + //////////////////////////////////////////////////// + + struct head_t { + char32 ident; + byte4 lock; + byte4 minor; + byte4 major; + byte4 fatal; + byte4 filesize; + byte4 filetop; + byte4 tables; + byte4 index; + char32 compname; + char32 tabs[maxtabs]; + byte4 frame; + + head_t(); + head_t(const head_t* hd,int mem); + + void setcomp(int idx, const sref& cp); + + sref get(int col) const { return sref(tabs[col]); } + void set(int col,const sref& s); + void insert(int col,const sref& s); + void remove(int col); + }; + + head_t* head() { return hd; } + const head_t* head() const { return hd; } + + /////////////////////////////////////////////////////// + + struct rec_t { + byte4 mem; + byte4 caps; + }; + + rec_t* at(int off) { + for(chunk_t* p=mem;p;p=p->next) if(p->own(off)) return p->at(off); return 0; + } + const rec_t* at(int off) const { + for(const chunk_t* p=mem;p;p=p->next) if(p->own(off)) return p->at(off); + return 0; + } + + int recs() const { return hd->frame.get(); } + int size(int off) const { + if(offfiletop.get()) { + for(const chunk_t* p=mem;p;p=p->next) + if(p->own(off)) return p->at(off)->mem.get(); + } + return 0; + } + + /////////////////////////////////////////////////////// + + void remap(); + + rec_t* add(int n); + void rem(rec_t* rec); + + /////////////////////////////////////////////////////// + + filemap(filemap* p); + filemap(filemap* p,const mstring& path,int update=0,int prot=0600); + ~filemap(); + + filemap* next; + + /////////////////////////////////////////////////////// + + void open(const mstring& path,int update=0,int prot=0600) throw(merror_t); + void save(const mstring& path,int prot=0600) throw(merror_t); + void check() throw(merror_t); + void close(); + void sync(); + + const mstring& Pathname() const { return pathname; } + time_t Lastmod() const; + + int reference() const { return refs; } + int writable() const { return wok; } + + void pushref() { refs++; } + void popref() { refs--; } + + /////////////////////////////////////////////////////// + + void lock(); + void unlock(); + + /////////////////////////////////////////////////////// + +protected: + struct chunk_t { + chunk_t* next; + long begin; + long end; + char* map; + + int size() const { return end-begin; } + + chunk_t(chunk_t* p,int fd,int wok,int off,int size); + ~chunk_t(); + + int own(int off) const { return off>=begin&&offlock(); } + ~headlock_t() { if(fm) fm->unlock(); } +}; + +#endif + diff --git a/src/hdb/hdbcvt.cc b/src/hdb/hdbcvt.cc new file mode 100644 index 0000000..ece70f0 --- /dev/null +++ b/src/hdb/hdbcvt.cc @@ -0,0 +1,50 @@ +/************************************************************************* + * + * HTCd - Copyright (C) 1998-2006 Henrik Rydberg + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include +#include + +static fp_stream mout(stdout),merr(stderr); + +main(int argc,char* argv[]) throw(merror_t) try +{ + if(argc<2) { + merr<<"Usage: "< [ ]\n"; + return -1; + } + mstring newfile=argv[1]; + mstring oldfile=newfile+sref(".bak"); + rename(newfile.c_str(),oldfile.c_str()); + dbmap db; + try { + db.Open(oldfile); + if(argc>3) db.Sort(sref(argv[2]),sref(argv[3])); + db.Saveas(newfile); + } + catch(const merror_t& e) { + merr<<"Opened as newfile, old version\n"; + db.Open(newfile,1); + Convert(db,oldfile.c_str()); + } + return 0; +} +catch(const merror_t& e) { + merr< + +static fp_stream mout(stdout),merr(stderr); + +main(int argc,char* argv[]) try +{ + if(argc<2) { + mout<<"Usage: "<\n"; + return -1; + } + mstring path; int silent; + if(argc>2) { silent=1; path=argv[2]; } + else { silent=0; path=argv[1]; } + mstring map; if(map.load(path)<0) THROW("hdbfix: could not open "<mem.get()<=0) { + if(!silent&&rec->mem.get()<0) + mout<<"ending mem: "<mem.get()<<"\n"; + break; + } + last=top; + top+=rec->mem.get(); + } + if(top==head.filetop.get()&&head.filesize.get()==map.size()) { + if(!silent) mout<=last;p-=4){ + int val=ntohl(*(long*)(map.data()+p)); + if(p+val==end) { nend++; end=p; } + } + int gap=end-top; + + mout<<"GAP DATA\n"; + mout<<"==============================\n"; + for(int i=top;i0) { + mout<<"Do you want to mark the gap as deleted?\n"; + getline(answer,stdin); answer.tolower(); + if(*answer=='y') { + filemap::rec_t* rec=(filemap::rec_t*)(map.data()+top); + rec->mem.set(gap); + rec->caps.set(0); + save=1; + mout<<"Done.\n"; + } + else { + mout<<"No change - quitting.\n"; + return 0; + } + } + } + } + + if(save) map.save(path); + + return 0; +} +catch(const merror_t& e) { + merr< + +static fp_stream mout(stdout),merr(stderr); + +main(int argc,char* argv[]) try +{ + if(argc<2) { + mout<<"Usage: "<\n"; + return -1; + } + mstring path; int silent; + if(argc>2) { silent=atoi(argv[1]); path=argv[2]; } + else { silent=0; path=argv[1]; } + mstring map; if(map.load(path)<0) THROW("hdbfix: could not open "<mem.get()<=0) break; + if(silent) { + if(silent==top) { + mout<mem.get(); + } + if(top==head.filetop.get()&&head.filesize.get()==map.size()) { + if(!silent) mout<=last;p-=4){ + unsigned val=*(unsigned*)(map.data()+p); + if(p+val==end) { nend++; end=p; } + } + int gap=end-top; + + mout<<"GAP DATA\n"; + mout<<"==============================\n"; + for(int i=top;i0) { + mout<<"Do you want to mark the gap as deleted?\n"; + getline(answer,stdin); answer.tolower(); + if(*answer=='y') { + filemap::rec_t* rec=(filemap::rec_t*)(map.data()+top); + rec->mem.set(gap); + rec->caps.set(0); + save=1; + mout<<"Done.\n"; + } + else { + mout<<"No change - quitting.\n"; + return 0; + } + } + } + } + + if(save) map.save(path); + + return 0; +} +catch(const merror_t& e) { + merr< +#include +using namespace std; + +int int_stream::appc(char c) { return val=c,1; } +int int_stream::appi(int i) { return val=i,1; } +int int_stream::appf(double f) { return val=(int)f,1; } +int int_stream::apps(const sref& s) { return val=atoi(s),s.size(); } + +/////////////////////////////////////////////////// + +record_t::record_t(const spile& list) +{ + pos[0].set(0); + for(int i=0;i p=equal_range(begin(),end(),idx_t(t,0),comp); + return p.first!=p.second?p.first-begin():-1; +} + +int indexmap::add(const sref& t,record_t* rec) +{ + idx_t tmp(t,rec); + pair p=equal_range(begin(),end(),tmp,comp); + if(p.first!=p.second) return &(*p.first=tmp)-begin(); + else return insert(p.second,tmp)-begin(); +} + +void indexmap::sort(const op_t* op) +{ + comp.func=op->f2; + ::sort(begin(),end(),comp); +} diff --git a/src/hdb/sortable.h b/src/hdb/sortable.h new file mode 100644 index 0000000..7de0aa8 --- /dev/null +++ b/src/hdb/sortable.h @@ -0,0 +1,138 @@ +/************************************************************************* + * + * HTCd - Copyright (C) 1998-2006 Henrik Rydberg + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#ifndef SORTABLEH +#define SORTABLEH + +#include +#include +#include + +//////////////////////////////////////////////////// +// +// NOTE that these classes are not MT safe: +// need to be called from dbamp +// + +struct record_t : public filemap::rec_t { + static const int maxtabs=filemap::maxtabs; + static const int recsize=(maxtabs+1)*sizeof(long); + + byte4 pos[maxtabs+1]; + char data[sizeof(byte4)]; + + int size() const { return pos[maxtabs].get()-pos[0].get(); } + int size(int i) const { return pos[i+1].get()-pos[i].get(); } + + sref at(int i) const { return sref(data+pos[i].get(),size(i)); } + + ///////////////////////////////////////////////// + + record_t(const record_t& rec) { + for(int i=0;i<=maxtabs;i++) pos[i]=rec.pos[i]; + memcpy(data,rec.data,rec.caps.get()); + } + record_t(const spile& list); + + ///////////////////////////////////////////////// + + void get(spile& list) const { + for(int i=0;icol;i--) pos[i]=pos[i-1]; + } + void remove(int col) { + resize(col,0); + for(int i=col;i { +public: + typedef mrvec::iterator iterator; + typedef mrvec::const_iterator const_iterator; + + struct comp_t { + op_t::op2_t func; + comp_t() : func(0) {} + comp_t(const op_t* op) : func(op->f2) {} + bool operator()(const idx_t& a,const idx_t& b) const { + int_stream res; + op_env env; + func(res,env,a.key,b.key); + return res.val; + } + }; + + int get(const sref& t) const; + int add(const sref& t,record_t* rec); + void sort(const op_t* op); +private: + comp_t comp; +}; + +#endif diff --git a/src/hdb/test.cc b/src/hdb/test.cc new file mode 100644 index 0000000..b8d335b --- /dev/null +++ b/src/hdb/test.cc @@ -0,0 +1,52 @@ +/************************************************************************* + * + * HTCd - Copyright (C) 1998-2006 Henrik Rydberg + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include + +static fp_stream merr(stderr); + +main(int argc,char* argv[]) throw(merror_t) try +{ + if(argc<2) { + merr<<"Usage: "<\n"; + return -1; + } + dbmap m; + m.Open(argv[1]); + + merr<filesize.get()<<"\n"; + merr<filetop.get()<<"\n"; + merr<frame.get()<<"\n"; + + for(int j=0;j