o2on svn commit
o2on-****@lists*****
2008年 7月 21日 (月) 16:24:31 JST
Revision: 104
http://svn.sourceforge.jp/cgi-bin/viewcvs.cgi?root=o2on&view=rev&rev=104
Author: electrolysis
Date: 2008-07-21 16:24:31 +0900 (Mon, 21 Jul 2008)
Log Message:
-----------
DBå†æ§‹ç¯‰ã®æ–¹æ³•ã‚’変更
荅括完:
・dat.db.rebuild上ã§å†æ§‹ç¯‰ã—ã€å®Œäº†æ™‚ã«å·®ã—替ãˆ
・datフォルダã®èµ°æŸ»ã¨å†æ§‹ç¯‰ã‚’åŒæœŸã—ã¦è¡Œã†
Modified Paths:
--------------
branches/BRANCH_0043/o2on/src.o2on/O2DatDB.cpp
branches/BRANCH_0043/o2on/src.o2on/O2DatDB.h
branches/BRANCH_0043/o2on/src.o2on/O2DatIO.cpp
branches/BRANCH_0043/o2on/src.o2on/O2DatIO.h
branches/BRANCH_0043/o2on/src.o2on/main.cpp
Modified: branches/BRANCH_0043/o2on/src.o2on/O2DatDB.cpp
===================================================================
--- branches/BRANCH_0043/o2on/src.o2on/O2DatDB.cpp 2008-06-26 11:53:50 UTC (rev 103)
+++ branches/BRANCH_0043/o2on/src.o2on/O2DatDB.cpp 2008-07-21 07:24:31 UTC (rev 104)
@@ -47,6 +47,7 @@
, UpdateThreadHandle(NULL)
, UpdateThreadLoop(false)
{
+ dbfilename_to_rebuild = dbfilename + L".rebuild";
}
@@ -173,17 +174,53 @@
+bool
+O2DatDB::
+check_queue_size(O2DatRecList &reclist)
+{
+ return (reclist.size() < MAX_UPDATE_QUEUE_SIZE ? true : false);
+}
+
+
+
bool
O2DatDB::
-create_table(void)
+before_rebuild(void)
{
+ if (!DeleteFile(dbfilename_to_rebuild.c_str()) && GetLastError() != ERROR_FILE_NOT_FOUND)
+ return false;
+ if (!create_table(true))
+ return false;
+
+ return true;
+}
+
+bool
+O2DatDB::
+after_rebuild(void)
+{
+ if (!MoveFileEx(dbfilename.c_str(), (dbfilename + L".old").c_str(), MOVEFILE_REPLACE_EXISTING | MOVEFILE_WRITE_THROUGH))
+ return false;
+ if (!MoveFileEx(dbfilename_to_rebuild.c_str(), dbfilename.c_str(), MOVEFILE_REPLACE_EXISTING | MOVEFILE_WRITE_THROUGH))
+ return false;
+
+ return true;
+}
+
+
+
+
+bool
+O2DatDB::
+create_table(bool to_rebuild)
+{
#if TRACE_SQL_EXEC_TIME
stopwatch sw("create table/index and analyze");
#endif
sqlite3 *db = NULL;
- int err = sqlite3_open16(dbfilename.c_str(), &db);
+ int err = sqlite3_open16(to_rebuild ? dbfilename_to_rebuild.c_str() : dbfilename.c_str(), &db);
if (err != SQLITE_OK)
goto error;
@@ -959,6 +996,90 @@
+void
+O2DatDB::
+insert(O2DatRecList &in, bool to_rebuild)
+{
+#if TRACE_SQL_EXEC_TIME
+ stopwatch sw("insert by reclist");
+#endif
+
+ sqlite3 *db = NULL;
+ sqlite3_stmt *stmt_insert = NULL;
+ O2DatRec org;
+
+ int err = sqlite3_open16(to_rebuild ? dbfilename_to_rebuild.c_str() : dbfilename.c_str(), &db);
+ if (err != SQLITE_OK)
+ goto error;
+ sqlite3_busy_timeout(db, 5000);
+
+ sqlite3_exec(db, "pragma synchronous = OFF;", NULL, NULL, NULL);
+
+ wchar_t *sql_insert =
+ L"insert or replace into dat ("
+ COLUMNS
+ L") values ("
+ L"?,?,?,?,?,?,?,?,?,?,?"
+ L");";
+ err = sqlite3_prepare16_v2(db, sql_insert, wcslen(sql_insert)*2, &stmt_insert, NULL);
+ if (err != SQLITE_OK)
+ goto error;
+
+ //
+ // Loop
+ //
+ sqlite3_exec(db, "begin;", NULL, NULL, NULL);
+ for (O2DatRecListIt it = in.begin(); it != in.end(); it++) {
+ sqlite3_reset(stmt_insert);
+ if (!bind(db, stmt_insert, 1, it->hash))
+ goto error;
+ if (!bind(db, stmt_insert, 2, it->domain))
+ goto error;
+ if (!bind(db, stmt_insert, 3, it->bbsname))
+ goto error;
+ if (!bind(db, stmt_insert, 4, it->datname))
+ goto error;
+ if (!bind(db, stmt_insert, 5, it->size))
+ goto error;
+ if (!bind(db, stmt_insert, 6, it->disksize))
+ goto error;
+ if (!bind(db, stmt_insert, 7, it->url))
+ goto error;
+ if (!bind(db, stmt_insert, 8, it->title))
+ goto error;
+ if (!bind(db, stmt_insert, 9, it->res))
+ goto error;
+ if (!bind(db, stmt_insert, 10, time(NULL)))
+ goto error;
+ if (!bind(db, stmt_insert, 11, (uint64)0))
+ goto error;
+
+ err = sqlite3_step(stmt_insert);
+ if (err != SQLITE_ROW && err != SQLITE_DONE)
+ goto error;
+ Sleep(1);
+ }
+ sqlite3_exec(db, "commit;", NULL, NULL, NULL);
+
+ sqlite3_finalize(stmt_insert);
+ stmt_insert = NULL;
+
+ err = sqlite3_close(db);
+ if (err != SQLITE_OK)
+ goto error;
+
+ return;
+
+error:
+ log(db);
+ if (stmt_insert) sqlite3_finalize(stmt_insert);
+ if (db) sqlite3_close(db);
+ return;
+}
+
+
+
+
#if 0
bool
O2DatDB::
@@ -1368,6 +1489,7 @@
O2DatDB::
StartUpdateThread(void)
{
+ Logger->AddLog(O2LT_INFO, L"UpdateThread", 0, 0, L"ŠJŽn");
if (UpdateThreadHandle)
return;
@@ -1380,6 +1502,7 @@
O2DatDB::
StopUpdateThread(void)
{
+ Logger->AddLog(O2LT_INFO, L"UpdateThread", 0, 0, L"’âŽ~");
if (!UpdateThreadHandle)
return;
UpdateThreadLoop = false;
Modified: branches/BRANCH_0043/o2on/src.o2on/O2DatDB.h
===================================================================
--- branches/BRANCH_0043/o2on/src.o2on/O2DatDB.h 2008-06-26 11:53:50 UTC (rev 103)
+++ branches/BRANCH_0043/o2on/src.o2on/O2DatDB.h 2008-07-21 07:24:31 UTC (rev 104)
@@ -56,6 +56,7 @@
protected:
O2Logger *Logger;
wstring dbfilename;
+ wstring dbfilename_to_rebuild;
O2DatRecList UpdateQueue;
Mutex UpdateQueueLock;
@@ -77,7 +78,12 @@
O2DatDB(O2Logger *lgr, const wchar_t *filename);
~O2DatDB();
- bool create_table(void);
+ bool check_queue_size(O2DatRecList &reclist);
+
+ bool before_rebuild(void);
+ bool after_rebuild(void);
+
+ bool create_table(bool to_rebuild);
bool reindex(const char *target);
bool analyze(void);
@@ -94,6 +100,8 @@
uint64 select_totaldisksize(void);
uint64 select_publishcount(time_t publish_tt);
+ void insert(O2DatRecList &in, bool to_rebuild);
+
// bool update(O2DatRec &in, bool is_origurl);
void update(O2DatRecList &in);
// bool update_lastpublish(const hashT &hash);
Modified: branches/BRANCH_0043/o2on/src.o2on/O2DatIO.cpp
===================================================================
--- branches/BRANCH_0043/o2on/src.o2on/O2DatIO.cpp 2008-06-26 11:53:50 UTC (rev 103)
+++ branches/BRANCH_0043/o2on/src.o2on/O2DatIO.cpp 2008-07-21 07:24:31 UTC (rev 104)
@@ -43,7 +43,6 @@
, RebuildDBThreadHandle(NULL)
, ReindexThreadHandle(NULL)
, LoopRebuildDB(false)
- , EnumDatThreadNum(0)
, AnalyzeThreadHandle(0)
{
@@ -878,8 +877,24 @@
O2DatIO *me = (O2DatIO*)data;
CoInitialize(NULL);
- me->RebuildDBThread(me->Profile->GetCacheRootW(), 0);
- me->DatDB->analyze();
+ me->DatDB->StopUpdateThread();
+ O2DatRecList reclist;
+ if (me->DatDB->before_rebuild()) {
+ me->RebuildDBThread(me->Profile->GetCacheRootW(), 0, reclist);
+ bool manualstop = !me->LoopRebuildDB;
+ // Žè“®‚Å’âŽ~‚µ‚½ê‡‚Í·‚µ‘Ö‚¦‚È‚Ç‚ð‚µ‚È‚¢
+ if (!manualstop) {
+ if (me->DatDB->after_rebuild())
+ me->DatDB->analyze();
+ else
+ me->Logger->AddLog(O2LT_ERROR, L"DBÄ\’z", 0, 0, "Ä\’zÏ‚ÝDB·‚µ‘Ö‚¦Ž¸”s");
+ }
+ }
+ else {
+ me->Logger->AddLog(O2LT_ERROR, L"DBÄ\’z", 0, 0, "Ä\’z—pDB쬎¸”s");
+ me->ProgressInfo->Reset(false, false);
+ }
+ me->DatDB->StartUpdateThread();
CoUninitialize();
CloseHandle(me->RebuildDBThreadHandle);
@@ -891,11 +906,14 @@
void
O2DatIO::
-RebuildDBThread(const wchar_t *dir, uint level)
+RebuildDBThread(const wchar_t *dir, uint level, O2DatRecList &reclist)
{
if (level == 0) {
ProgressInfo->Reset(true, true);
}
+ if (level == 3) {
+ ProgressInfo->SetMessage(dir+wcslen(Profile->GetCacheRootW()));
+ }
WIN32_FIND_DATAW wfd;
HANDLE handle;
@@ -913,18 +931,55 @@
do {
if (wfd.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) {
- if (wfd.cFileName[0] != L'.')
- dirs.push_back(wfd.cFileName);
+ if (wfd.cFileName[0] != L'.') {
+ if (level == 3)
+ Logger->AddLog(O2LT_WARNING, L"DBÄ\’z", 0, 0,
+ L"—]Œv‚ȃfƒBƒŒƒNƒgƒŠ‚ª‚ ‚é‚æ(%s\\%s)", dir, wfd.cFileName);
+ else
+ dirs.push_back(wfd.cFileName);
+ }
}
else {
if (wcscmp(wfd.cFileName, L".index") == 0) {
swprintf_s(path, MAX_PATH, L"%s\\%s", dir, wfd.cFileName);
DeleteFile(path);
+ continue;
}
- else {
+ else if (level != 3) {
Logger->AddLog(O2LT_WARNING, L"DBÄ\’z", 0, 0,
L"•Ï‚ȃtƒ@ƒCƒ‹‚ª‚ ‚é‚æ(%s\\%s)", dir, wfd.cFileName);
+ continue;
}
+
+ if (level != 3) continue;
+
+ wsplit(dir+wcslen(Profile->GetCacheRootW()), L"\\", paths);
+ if (!datpath.set(paths[0].c_str(), paths[1].c_str(), wfd.cFileName)) {
+ Logger->AddLog(O2LT_WARNING, L"DBÄ\’z", 0, 0,
+ L"dat‚¶‚á‚È‚¢ƒtƒ@ƒCƒ‹H(%s\\%s)", dir, wfd.cFileName);
+ continue;
+ }
+ if (wfd.dwFileAttributes & FILE_ATTRIBUTE_READONLY) {
+ swprintf_s(path, MAX_PATH, L"%s\\%s", dir, wfd.cFileName);
+ wfd.dwFileAttributes ^= FILE_ATTRIBUTE_READONLY;
+ SetFileAttributes(path, wfd.dwFileAttributes);
+ }
+
+ datpath.gethash(rec.hash);
+ rec.domain = paths[0];
+ rec.bbsname = paths[1];
+ rec.datname = wfd.cFileName;
+ rec.size = ((uint64)wfd.nFileSizeHigh << 32) | (uint64)wfd.nFileSizeLow;
+ rec.disksize = GetDiskFileSize(rec.size);
+ datpath.geturl(rec.url);
+ GetTitle(datpath);
+ datpath.gettitle(rec.title);
+ rec.res = 0;
+ reclist.push_back(rec);
+ if (!DatDB->check_queue_size(reclist)) {
+ DatDB->insert(reclist, true);
+ reclist.clear();
+ }
}
} while (LoopRebuildDB && FindNextFileW(handle, &wfd));
FindClose(handle);
@@ -983,133 +1038,25 @@
continue;
swprintf_s(path, MAX_PATH, L"%s\\%s", dir, s);
+ RebuildDBThread(path, level+1, reclist); //Ä‹A
- if (level == 2) {
- while (1) {
- EnumDatThreadNumLock.Lock();
- if (EnumDatThreadNum < 10) {
- EnumDatThreadNum++;
-
- ThreadData *param = new ThreadData;;
- param->me = this;
- param->dir = path;
-
- HANDLE thandle = (HANDLE)_beginthreadex(
- NULL, 0, StaticEnumDatThread, (void*)param, 0, NULL);
- CloseHandle(thandle);
- EnumDatThreadNumLock.Unlock();
- break;
- }
- EnumDatThreadNumLock.Unlock();
- Sleep(1000);
- }
- }
- else {
- RebuildDBThread(path, level+1); //Ä‹A
- }
-
if (level == 1)
ProgressInfo->AddPos(1);
}
}
if (level == 0) {
- while (EnumDatThreadNum)
- Sleep(1000);
+ if (LoopRebuildDB && !reclist.empty()) {
+ DatDB->insert(reclist, true);
+ reclist.clear();
+ }
ProgressInfo->Reset(false, true);
CLEAR_WORKSET;
}
}
-uint WINAPI
-O2DatIO::
-StaticEnumDatThread(void *data)
-{
- ThreadData *param = (ThreadData*)data;
- O2DatIO *me = param->me;
- wstring dir = param->dir;
- delete param;
-
- CoInitialize(NULL);
- me->EnumDatThread(dir.c_str());
- CoUninitialize();
-
- me->EnumDatThreadNumLock.Lock();
- me->EnumDatThreadNum--;
- me->EnumDatThreadNumLock.Unlock();
-
- //_endthreadex(0);
- return (0);
-}
-
void
O2DatIO::
-EnumDatThread(const wchar_t *dir)
-{
- ProgressInfo->SetMessage(dir+wcslen(Profile->GetCacheRootW()));
-
- wchar_t findpath[MAX_PATH];
- swprintf_s(findpath, MAX_PATH, L"%s\\*.*", dir);
-
- wstrarray paths;
- wsplit(dir+wcslen(Profile->GetCacheRootW()), L"\\", paths);
-
- O2DatRec rec;
- rec.domain = paths[0];
- rec.bbsname = paths[1];
-
- O2DatPath datpath;
- wchar_t path[MAX_PATH];
-
- WIN32_FIND_DATAW wfd;
- HANDLE handle = FindFirstFileW(findpath, &wfd);
- if (handle == INVALID_HANDLE_VALUE)
- return;
-
- do {
- if (wfd.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) {
- if (wfd.cFileName[0] != L'.') {
- Logger->AddLog(O2LT_WARNING, L"DBÄ\’z", 0, 0,
- L"—]Œv‚ȃfƒBƒŒƒNƒgƒŠ‚ª‚ ‚é‚æ(%s\\%s)", dir, wfd.cFileName);
- }
- continue;
- }
-
- if (wcscmp(wfd.cFileName, L".index") == 0) {
- swprintf_s(path, MAX_PATH, L"%s\\%s", dir, wfd.cFileName);
- DeleteFile(path);
- continue;
- }
- if (!datpath.set(rec.domain.c_str(), rec.bbsname.c_str(), wfd.cFileName)) {
- Logger->AddLog(O2LT_WARNING, L"DBÄ\’z", 0, 0,
- L"dat‚¶‚á‚È‚¢ƒtƒ@ƒCƒ‹H(%s\\%s)", dir, wfd.cFileName);
- continue;
- }
- if (wfd.dwFileAttributes & FILE_ATTRIBUTE_READONLY) {
- swprintf_s(path, MAX_PATH, L"%s\\%s", dir, wfd.cFileName);
- wfd.dwFileAttributes ^= FILE_ATTRIBUTE_READONLY;
- SetFileAttributes(path, wfd.dwFileAttributes);
- }
-
- datpath.gethash(rec.hash);
- rec.datname = wfd.cFileName;
- rec.size = ((uint64)wfd.nFileSizeHigh << 32) | (uint64)wfd.nFileSizeLow;
- rec.disksize = GetDiskFileSize(rec.size);
- datpath.geturl(rec.url);
- GetTitle(datpath);
- datpath.gettitle(rec.title);
- rec.res = 0;
-
- DatDB->AddUpdateQueue(rec);
- } while (LoopRebuildDB && FindNextFileW(handle, &wfd));
- FindClose(handle);
-}
-
-
-
-
-void
-O2DatIO::
Reindex(void)
{
if (ReindexThreadHandle)
Modified: branches/BRANCH_0043/o2on/src.o2on/O2DatIO.h
===================================================================
--- branches/BRANCH_0043/o2on/src.o2on/O2DatIO.h 2008-06-26 11:53:50 UTC (rev 103)
+++ branches/BRANCH_0043/o2on/src.o2on/O2DatIO.h 2008-07-21 07:24:31 UTC (rev 104)
@@ -38,18 +38,9 @@
HANDLE ReindexThreadHandle;
HANDLE AnalyzeThreadHandle;
bool LoopRebuildDB;
- uint EnumDatThreadNum;
- Mutex EnumDatThreadNumLock;
- struct ThreadData {
- O2DatIO *me;
- wstring dir;
- };
-
protected:
uint64 GetDiskFileSize(uint64 size);
- static uint WINAPI StaticEnumDatThread(void *data);
- void EnumDatThread(const wchar_t *dir);
public:
O2DatIO(O2DatDB *db, O2Logger *lgr, O2Profile *prof, O2ProgressInfo *proginfo);
@@ -80,7 +71,7 @@
void RebuildDB(void);
void StopRebuildDB(void);
static uint WINAPI StaticRebuildDBThread(void *data);
- void RebuildDBThread(const wchar_t *dir, uint level);
+ void RebuildDBThread(const wchar_t *dir, uint level, O2DatRecList &reclist);
void Reindex(void);
static uint WINAPI StaticReindexThread(void *data);
Modified: branches/BRANCH_0043/o2on/src.o2on/main.cpp
===================================================================
--- branches/BRANCH_0043/o2on/src.o2on/main.cpp 2008-06-26 11:53:50 UTC (rev 103)
+++ branches/BRANCH_0043/o2on/src.o2on/main.cpp 2008-07-21 07:24:31 UTC (rev 104)
@@ -379,7 +379,7 @@
wstring dbfilename(Profile->GetDBDirW());
dbfilename += L"\\dat.db";
DatDB = new O2DatDB(Logger, dbfilename.c_str());
- if (!DatDB->create_table()) {
+ if (!DatDB->create_table(false)) {
MessageBox(NULL,
L"DBƒI[ƒvƒ“‚ÉŽ¸”s‚µ‚Ü‚µ‚½\n‹N“®‚𒆎~‚µ‚Ü‚·",
NULL, MB_ICONERROR | MB_OK);
@@ -1060,6 +1060,10 @@
break;
case ID_REBUILDDB:
if (!Active && !hwndProgressDlg) {
+ if (MessageBox(hwnd,
+ _T("ƒf[ƒ^ƒx[ƒX‚ðÄ\’z‚µ‚Ü‚·"),
+ _T("DBÄ\’z"), MB_OKCANCEL|MB_ICONINFORMATION) == IDCANCEL)
+ break;
CreateProgressDialog(_T("DBÄ\’z..."));
DatIO->RebuildDB();
}